Merge remote-tracking branch 'john-ericson/enum-FileIngestionMethod' into no-stringly-typed-derivation-output

18 files changed, 1613 insertions, 194 deletions

diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh
new file mode 100644
index 000000000..8ae07b092
--- /dev/null
+++ b/src/libutil/ansicolor.hh
@@ -0,0 +1,15 @@
+#pragma once
+
+namespace nix {
+
+/* Some ANSI escape sequences. */
+#define ANSI_NORMAL "\e[0m"
+#define ANSI_BOLD "\e[1m"
+#define ANSI_FAINT "\e[2m"
+#define ANSI_ITALIC "\e[3m"
+#define ANSI_RED "\e[31;1m"
+#define ANSI_GREEN "\e[32;1m"
+#define ANSI_YELLOW "\e[33;1m"
+#define ANSI_BLUE "\e[34;1m"
+
+}
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index ba15ea571..f829415d1 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -3,16 +3,14 @@
 
 namespace nix {
 
-Args::FlagMaker Args::mkFlag()
-{
-    return FlagMaker(*this);
-}
-
-Args::FlagMaker::~FlagMaker()
+void Args::addFlag(Flag && flag_)
 {
+    auto flag = std::make_shared<Flag>(std::move(flag_));
+    if (flag->handler.arity != ArityAny)
+        assert(flag->handler.arity == flag->labels.size());
     assert(flag->longName != "");
-    args.longFlags[flag->longName] = flag;
-    if (flag->shortName) args.shortFlags[flag->shortName] = flag;
+    longFlags[flag->longName] = flag;
+    if (flag->shortName) shortFlags[flag->shortName] = flag;
 }
 
 void Args::parseCmdline(const Strings & _cmdline)
@@ -61,7 +59,7 @@ void Args::parseCmdline(const Strings & _cmdline)
 
 void Args::printHelp(const string & programName, std::ostream & out)
 {
-    std::cout << "Usage: " << programName << " <FLAGS>...";
+    std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName);
     for (auto & exp : expectedArgs) {
         std::cout << renderLabels({exp.label});
         // FIXME: handle arity > 1
@@ -72,11 +70,11 @@ void Args::printHelp(const string & programName, std::ostream & out)
 
     auto s = description();
     if (s != "")
-        std::cout << "\nSummary: " << s << ".\n";
+        std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n";
 
     if (longFlags.size()) {
         std::cout << "\n";
-        std::cout << "Flags:\n";
+        std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n";
         printFlags(out);
     }
 }
@@ -101,15 +99,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
     auto process = [&](const std::string & name, const Flag & flag) -> bool {
         ++pos;
         std::vector<std::string> args;
-        for (size_t n = 0 ; n < flag.arity; ++n) {
+        for (size_t n = 0 ; n < flag.handler.arity; ++n) {
             if (pos == end) {
-                if (flag.arity == ArityAny) break;
-                throw UsageError(format("flag '%1%' requires %2% argument(s)")
-                    % name % flag.arity);
+                if (flag.handler.arity == ArityAny) break;
+                throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
             }
             args.push_back(*pos++);
         }
-        flag.handler(std::move(args));
+        flag.handler.fun(std::move(args));
         return true;
     };
 
@@ -157,17 +154,18 @@ bool Args::processArgs(const Strings & args, bool finish)
     return res;
 }
 
-Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht)
+Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
 {
-    arity(1);
-    label("type");
-    description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')");
-    handler([ht](std::string s) {
-        *ht = parseHashType(s);
-        if (*ht == htUnknown)
-            throw UsageError("unknown hash type '%1%'", s);
-    });
-    return *this;
+    return Flag {
+        .longName = std::move(longName),
+        .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')",
+        .labels = {"hash-algo"},
+        .handler = {[ht](std::string s) {
+            *ht = parseHashType(s);
+            if (*ht == htUnknown)
+                throw UsageError("unknown hash type '%1%'", s);
+        }}
+    };
 }
 
 Strings argvToStrings(int argc, char * * argv)
@@ -183,7 +181,7 @@ std::string renderLabels(const Strings & labels)
     std::string res;
     for (auto label : labels) {
         for (auto & c : label) c = std::toupper(c);
-        res += " <" + label + ">";
+        res += " " ANSI_ITALIC + label + ANSI_NORMAL;
     }
     return res;
 }
@@ -192,10 +190,10 @@ void printTable(std::ostream & out, const Table2 & table)
 {
     size_t max = 0;
     for (auto & row : table)
-        max = std::max(max, row.first.size());
+        max = std::max(max, filterANSIEscapes(row.first, true).size());
     for (auto & row : table) {
         out << "  " << row.first
-            << std::string(max - row.first.size() + 2, ' ')
+            << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ')
             << row.second << "\n";
     }
 }
@@ -206,8 +204,7 @@ void Command::printHelp(const string & programName, std::ostream & out)
 
     auto exs = examples();
     if (!exs.empty()) {
-        out << "\n";
-        out << "Examples:\n";
+        out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n";
         for (auto & ex : exs)
             out << "\n"
                 << "  " << ex.description << "\n" // FIXME: wrap
@@ -223,49 +220,55 @@ MultiCommand::MultiCommand(const Commands & commands)
         auto i = commands.find(ss[0]);
         if (i == commands.end())
             throw UsageError("'%s' is not a recognised command", ss[0]);
-        command = i->second();
-        command->_name = ss[0];
+        command = {ss[0], i->second()};
     }});
+
+    categories[Command::catDefault] = "Available commands";
 }
 
 void MultiCommand::printHelp(const string & programName, std::ostream & out)
 {
     if (command) {
-        command->printHelp(programName + " " + command->name(), out);
+        command->second->printHelp(programName + " " + command->first, out);
         return;
     }
 
-    out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n";
+    out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName);
 
-    out << "\n";
-    out << "Common flags:\n";
+    out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n";
     printFlags(out);
 
-    out << "\n";
-    out << "Available commands:\n";
+    std::map<Command::Category, std::map<std::string, ref<Command>>> commandsByCategory;
 
-    Table2 table;
-    for (auto & i : commands) {
-        auto command = i.second();
-        command->_name = i.first;
-        auto descr = command->description();
-        if (!descr.empty())
-            table.push_back(std::make_pair(command->name(), descr));
+    for (auto & [name, commandFun] : commands) {
+        auto command = commandFun();
+        commandsByCategory[command->category()].insert_or_assign(name, command);
+    }
+
+    for (auto & [category, commands] : commandsByCategory) {
+        out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]);
+
+        Table2 table;
+        for (auto & [name, command] : commands) {
+            auto descr = command->description();
+            if (!descr.empty())
+                table.push_back(std::make_pair(name, descr));
+        }
+        printTable(out, table);
     }
-    printTable(out, table);
 }
 
 bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end)
 {
     if (Args::processFlag(pos, end)) return true;
-    if (command && command->processFlag(pos, end)) return true;
+    if (command && command->second->processFlag(pos, end)) return true;
     return false;
 }
 
 bool MultiCommand::processArgs(const Strings & args, bool finish)
 {
     if (command)
-        return command->processArgs(args, finish);
+        return command->second->processArgs(args, finish);
     else
         return Args::processArgs(args, finish);
 }
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 967efbe1c..1932e6a8a 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -32,13 +32,59 @@ protected:
     struct Flag
     {
         typedef std::shared_ptr<Flag> ptr;
+
+        struct Handler
+        {
+            std::function<void(std::vector<std::string>)> fun;
+            size_t arity;
+
+            Handler() {}
+
+            Handler(std::function<void(std::vector<std::string>)> && fun)
+                : fun(std::move(fun))
+                , arity(ArityAny)
+            { }
+
+            Handler(std::function<void()> && handler)
+                : fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); })
+                , arity(0)
+            { }
+
+            Handler(std::function<void(std::string)> && handler)
+                : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
+                    handler(std::move(ss[0]));
+                  })
+                , arity(1)
+            { }
+
+            Handler(std::function<void(std::string, std::string)> && handler)
+                : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
+                    handler(std::move(ss[0]), std::move(ss[1]));
+                  })
+                , arity(2)
+            { }
+
+            template<class T>
+            Handler(T * dest)
+                : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
+                , arity(1)
+            { }
+
+            template<class T>
+            Handler(T * dest, const T & val)
+                : fun([=](std::vector<std::string> ss) { *dest = val; })
+                , arity(0)
+            { }
+        };
+
         std::string longName;
         char shortName = 0;
         std::string description;
-        Strings labels;
-        size_t arity = 0;
-        std::function<void(std::vector<std::string>)> handler;
         std::string category;
+        Strings labels;
+        Handler handler;
+
+        static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
     };
 
     std::map<std::string, Flag::ptr> longFlags;
@@ -65,49 +111,7 @@ protected:
 
 public:
 
-    class FlagMaker
-    {
-        Args & args;
-        Flag::ptr flag;
-        friend class Args;
-        FlagMaker(Args & args) : args(args), flag(std::make_shared<Flag>()) { }
-    public:
-        ~FlagMaker();
-        FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; }
-        FlagMaker & shortName(char s) { flag->shortName = s; return *this; }
-        FlagMaker & description(const std::string & s) { flag->description = s; return *this; }
-        FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; }
-        FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; }
-        FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; }
-        FlagMaker & handler(std::function<void(std::vector<std::string>)> handler) { flag->handler = handler; return *this; }
-        FlagMaker & handler(std::function<void()> handler) { flag->handler = [handler](std::vector<std::string>) { handler(); }; return *this; }
-        FlagMaker & handler(std::function<void(std::string)> handler) {
-            flag->arity = 1;
-            flag->handler = [handler](std::vector<std::string> ss) { handler(std::move(ss[0])); };
-            return *this;
-        }
-        FlagMaker & category(const std::string & s) { flag->category = s; return *this; }
-
-        template<class T>
-        FlagMaker & dest(T * dest)
-        {
-            flag->arity = 1;
-            flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; };
-            return *this;
-        }
-
-        template<class T>
-        FlagMaker & set(T * dest, const T & val)
-        {
-            flag->arity = 0;
-            flag->handler = [=](std::vector<std::string> ss) { *dest = val; };
-            return *this;
-        }
-
-        FlagMaker & mkHashTypeFlag(HashType * ht);
-    };
-
-    FlagMaker mkFlag();
+    void addFlag(Flag && flag);
 
     /* Helper functions for constructing flags / positional
        arguments. */
@@ -116,13 +120,13 @@ public:
         const std::string & label, const std::string & description,
         std::function<void(std::string)> fun)
     {
-        mkFlag()
-            .shortName(shortName)
-            .longName(longName)
-            .labels({label})
-            .description(description)
-            .arity(1)
-            .handler([=](std::vector<std::string> ss) { fun(ss[0]); });
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .labels = {label},
+            .handler = {[=](std::string s) { fun(s); }}
+        });
     }
 
     void mkFlag(char shortName, const std::string & name,
@@ -135,11 +139,12 @@ public:
     void mkFlag(char shortName, const std::string & longName, const std::string & description,
         T * dest, const T & value)
     {
-        mkFlag()
-            .shortName(shortName)
-            .longName(longName)
-            .description(description)
-            .handler([=](std::vector<std::string> ss) { *dest = value; });
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .handler = {[=]() { *dest = value; }}
+        });
     }
 
     template<class I>
@@ -155,18 +160,18 @@ public:
     void mkFlag(char shortName, const std::string & longName,
         const std::string & description, std::function<void(I)> fun)
     {
-        mkFlag()
-            .shortName(shortName)
-            .longName(longName)
-            .labels({"N"})
-            .description(description)
-            .arity(1)
-            .handler([=](std::vector<std::string> ss) {
+        addFlag({
+            .longName = longName,
+            .shortName = shortName,
+            .description = description,
+            .labels = {"N"},
+            .handler = {[=](std::string s) {
                 I n;
-                if (!string2Int(ss[0], n))
+                if (!string2Int(s, n))
                     throw UsageError("flag '--%s' requires a integer argument", longName);
                 fun(n);
-            });
+            }}
+        });
     }
 
     /* Expect a string argument. */
@@ -192,17 +197,10 @@ public:
    run() method. */
 struct Command : virtual Args
 {
-private:
-    std::string _name;
-
     friend class MultiCommand;
 
-public:
-
     virtual ~Command() { }
 
-    std::string name() { return _name; }
-
     virtual void prepare() { };
     virtual void run() = 0;
 
@@ -216,6 +214,12 @@ public:
 
     virtual Examples examples() { return Examples(); }
 
+    typedef int Category;
+
+    static constexpr Category catDefault = 0;
+
+    virtual Category category() { return catDefault; }
+
     void printHelp(const string & programName, std::ostream & out) override;
 };
 
@@ -228,7 +232,10 @@ class MultiCommand : virtual Args
 public:
     Commands commands;
 
-    std::shared_ptr<Command> command;
+    std::map<Command::Category, std::string> categories;
+
+    // Selected command, if any.
+    std::optional<std::pair<std::string, ref<Command>>> command;
 
     MultiCommand(const Commands & commands);
 
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 7551d97d1..f03e444ec 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -177,12 +177,13 @@ void BaseSetting<T>::toJSON(JSONPlaceholder & out)
 template<typename T>
 void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag()
-        .longName(name)
-        .description(description)
-        .arity(1)
-        .handler([=](std::vector<std::string> ss) { overriden = true; set(ss[0]); })
-        .category(category);
+    args.addFlag({
+        .longName = name,
+        .description = description,
+        .category = category,
+        .labels = {"value"},
+        .handler = {[=](std::string s) { overriden = true; set(s); }},
+    });
 }
 
 template<> void BaseSetting<std::string>::set(const std::string & str)
@@ -227,16 +228,18 @@ template<> std::string BaseSetting<bool>::to_string() const
 
 template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag()
-        .longName(name)
-        .description(description)
-        .handler([=](std::vector<std::string> ss) { override(true); })
-        .category(category);
-    args.mkFlag()
-        .longName("no-" + name)
-        .description(description)
-        .handler([=](std::vector<std::string> ss) { override(false); })
-        .category(category);
+    args.addFlag({
+        .longName = name,
+        .description = description,
+        .category = category,
+        .handler = {[=]() { override(true); }}
+    });
+    args.addFlag({
+        .longName = "no-" + name,
+        .description = description,
+        .category = category,
+        .handler = {[=]() { override(false); }}
+    });
 }
 
 template<> void BaseSetting<Strings>::set(const std::string & str)
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
new file mode 100644
index 000000000..a5571d4ec
--- /dev/null
+++ b/src/libutil/error.cc
@@ -0,0 +1,146 @@
+#include "error.hh"
+
+#include <iostream>
+#include <optional>
+
+namespace nix
+{
+
+std::optional<string> ErrorInfo::programName = std::nullopt;
+
+std::ostream& operator<<(std::ostream &os, const hintformat &hf)
+{
+    return os << hf.str();
+}
+
+string showErrPos(const ErrPos &errPos)
+{
+    if (errPos.column > 0) {
+        return fmt("(%1%:%2%)", errPos.lineNumber, errPos.column);
+    } else {
+        return fmt("(%1%)", errPos.lineNumber);
+    };
+}
+
+void printCodeLines(const string &prefix, const NixCode &nixCode)
+{
+    // previous line of code.
+    if (nixCode.prevLineOfCode.has_value()) {
+        std::cout << fmt("%1% %|2$5d|| %3%",
+                         prefix,
+                         (nixCode.errPos.lineNumber - 1),
+                         *nixCode.prevLineOfCode)
+                  << std::endl;
+    }
+
+    // line of code containing the error.%2$+5d%
+    std::cout << fmt("%1% %|2$5d|| %3%",
+                     prefix,
+                     (nixCode.errPos.lineNumber),
+                     nixCode.errLineOfCode)
+              << std::endl;
+
+    // error arrows for the column range.
+    if (nixCode.errPos.column > 0) {
+        int start = nixCode.errPos.column;
+        std::string spaces;
+        for (int i = 0; i < start; ++i) {
+            spaces.append(" ");
+        }
+
+        std::string arrows("^");
+
+        std::cout << fmt("%1%      |%2%" ANSI_RED "%3%" ANSI_NORMAL,
+                         prefix,
+                         spaces,
+                         arrows) << std::endl;
+    }
+
+    // next line of code.
+    if (nixCode.nextLineOfCode.has_value()) {
+        std::cout << fmt("%1% %|2$5d|| %3%",
+                         prefix,
+                         (nixCode.errPos.lineNumber + 1),
+                         *nixCode.nextLineOfCode)
+                  << std::endl;
+    }
+}
+
+void printErrorInfo(const ErrorInfo &einfo)
+{
+    int errwidth = 80;
+    string prefix = "    ";
+
+    string levelString;
+    switch (einfo.level) {
+    case ErrLevel::elError: {
+        levelString = ANSI_RED;
+        levelString += "error:";
+        levelString += ANSI_NORMAL;
+        break;
+    }
+    case ErrLevel::elWarning: {
+        levelString = ANSI_YELLOW;
+        levelString += "warning:";
+        levelString += ANSI_NORMAL;
+        break;
+    }
+    default: {
+        levelString = fmt("invalid error level: %1%", einfo.level);
+        break;
+    }
+    }
+
+    int ndl = prefix.length() + levelString.length() + 3 + einfo.name.length() + einfo.programName.value_or("").length();
+    int dashwidth = ndl > (errwidth - 3) ? 3 : errwidth - ndl;
+
+    string dashes;
+    for (int i = 0; i < dashwidth; ++i)
+        dashes.append("-");
+
+    // divider.
+    std::cout << fmt("%1%%2%" ANSI_BLUE " %3% %4% %5% %6%" ANSI_NORMAL,
+                     prefix,
+                     levelString,
+                     "---",
+                     einfo.name,
+                     dashes,
+                     einfo.programName.value_or(""))
+              << std::endl;
+
+    // filename.
+    if (einfo.nixCode.has_value()) {
+        if (einfo.nixCode->errPos.nixFile != "") {
+            string eline = einfo.nixCode->errLineOfCode != ""
+                           ? string(" ") + showErrPos(einfo.nixCode->errPos)
+                           : "";
+
+            std::cout << fmt("%1%in file: " ANSI_BLUE "%2%%3%" ANSI_NORMAL,
+                             prefix, 
+                             einfo.nixCode->errPos.nixFile,
+                             eline) << std::endl;
+            std::cout << prefix << std::endl;
+        } else {
+            std::cout << fmt("%1%from command line argument", prefix) << std::endl;
+            std::cout << prefix << std::endl;
+        }
+    }
+
+    // description
+    std::cout << prefix << einfo.description << std::endl;
+    std::cout << prefix << std::endl;
+
+    // lines of code.
+    if (einfo.nixCode->errLineOfCode != "") {
+        printCodeLines(prefix, *einfo.nixCode);
+        std::cout << prefix << std::endl;
+    }
+
+    // hint
+    if (einfo.hint.has_value()) {
+        std::cout << prefix << *einfo.hint << std::endl;
+        std::cout << prefix << std::endl;
+    }
+}
+
+}
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
new file mode 100644
index 000000000..f402b692e
--- /dev/null
+++ b/src/libutil/error.hh
@@ -0,0 +1,121 @@
+#ifndef error_hh
+#define error_hh
+
+#include "ansicolor.hh"
+#include <string>
+#include <optional>
+#include <iostream>
+#include "types.hh"
+
+namespace nix
+{
+
+typedef enum {
+    elWarning,
+    elError
+} ErrLevel;
+
+struct ErrPos
+{
+    int lineNumber;
+    int column;
+    string nixFile;
+
+    template <class P>
+    ErrPos& operator=(const P &pos)
+    {
+        lineNumber = pos.line;
+        column = pos.column;
+        nixFile = pos.file;
+        return *this;
+    }
+
+    template <class P>
+    ErrPos(const P &p)
+    {
+      *this = p;
+    }
+};
+
+struct NixCode
+{
+    ErrPos errPos;
+    std::optional<string> prevLineOfCode;
+    string errLineOfCode;
+    std::optional<string> nextLineOfCode;
+};
+
+// ----------------------------------------------------------------
+// format function for hints.  same as fmt, except templated values
+// are always in yellow.
+
+template <class T>
+struct yellowify
+{
+    yellowify(T &s) : value(s) {}
+    T &value;
+};
+
+template <class T>
+std::ostream& operator<<(std::ostream &out, const yellowify<T> &y)
+{
+    return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
+}
+
+class hintformat
+{
+public:
+    hintformat(string format) :fmt(format)
+    {
+        fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
+    }
+    template<class T>
+    hintformat& operator%(const T &value)
+    {
+        fmt % yellowify(value);
+        return *this;
+    }
+
+    std::string str() const
+    {
+        return fmt.str();
+    }
+
+    template <typename U>
+    friend class AddHint;
+private:
+    format fmt;
+};
+
+std::ostream& operator<<(std::ostream &os, const hintformat &hf);
+
+template<typename... Args>
+inline hintformat hintfmt(const std::string & fs, const Args & ... args)
+{
+    hintformat f(fs);
+    formatHelper(f, args...);
+    return f;
+}
+
+// -------------------------------------------------
+// ErrorInfo.
+struct ErrorInfo
+{
+    ErrLevel level;
+    string name;
+    string description;
+    std::optional<hintformat> hint;
+    std::optional<NixCode> nixCode;
+
+    static std::optional<string> programName;
+};
+
+// --------------------------------------------------------
+// error printing
+
+// just to cout for now.
+void printErrorInfo(const ErrorInfo &einfo);
+
+}
+
+#endif
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index fa5c84a27..3cc4ef8f1 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -3,6 +3,7 @@
 
 #include <atomic>
 #include <nlohmann/json.hpp>
+#include <iostream>
 
 namespace nix {
 
@@ -24,6 +25,11 @@ void Logger::warn(const std::string & msg)
     log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg);
 }
 
+void Logger::writeToStdout(std::string_view s)
+{
+    std::cout << s << "\n";
+}
+
 class SimpleLogger : public Logger
 {
 public:
@@ -198,7 +204,7 @@ bool handleJSONLogMessage(const std::string & msg,
 
         if (action == "start") {
             auto type = (ActivityType) json["type"];
-            if (trusted || type == actDownload)
+            if (trusted || type == actFileTransfer)
                 activities.emplace(std::piecewise_construct,
                     std::forward_as_tuple(json["id"]),
                     std::forward_as_tuple(*logger, (Verbosity) json["level"], type,
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index beb5e6b64..18c24d508 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -17,7 +17,7 @@ typedef enum {
 typedef enum {
     actUnknown = 0,
     actCopyPath = 100,
-    actDownload = 101,
+    actFileTransfer = 101,
     actRealise = 102,
     actCopyPaths = 103,
     actBuilds = 104,
@@ -78,6 +78,16 @@ public:
     virtual void stopActivity(ActivityId act) { };
 
     virtual void result(ActivityId act, ResultType type, const Fields & fields) { };
+
+    virtual void writeToStdout(std::string_view s);
+
+    template<typename... Args>
+    inline void stdout(const std::string & fs, const Args & ... args)
+    {
+        boost::format f(fs);
+        formatHelper(f, args...);
+        writeToStdout(f.str());
+    }
 };
 
 ActivityId getCurActivity();
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 5780c93a6..a04118512 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -148,6 +148,9 @@ struct StringSink : Sink
 {
     ref<std::string> s;
     StringSink() : s(make_ref<std::string>()) { };
+    explicit StringSink(const size_t reservedSize) : s(make_ref<std::string>()) {
+      s->reserve(reservedSize);
+    };
     StringSink(ref<std::string> s) : s(s) { };
     void operator () (const unsigned char * data, size_t len) override;
 };
diff --git a/src/libutil/tests/libutil-tests b/src/libutil/tests/libutil-tests
new file mode 100755
index 000000000..ae871b466
--- /dev/null
+++ b/src/libutil/tests/libutil-tests
diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk
new file mode 100644
index 000000000..a297edb64
--- /dev/null
+++ b/src/libutil/tests/local.mk
@@ -0,0 +1,15 @@
+check: libutil-tests_RUN
+
+programs += libutil-tests
+
+libutil-tests_DIR := $(d)
+
+libutil-tests_INSTALL_DIR :=
+
+libutil-tests_SOURCES := $(wildcard $(d)/*.cc)
+
+libutil-tests_CXXFLAGS += -I src/libutil
+
+libutil-tests_LIBS = libutil
+
+libutil-tests_LDFLAGS := $(GTEST_LIBS)
diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
new file mode 100644
index 000000000..8e77ccbe1
--- /dev/null
+++ b/src/libutil/tests/tests.cc
@@ -0,0 +1,589 @@
+#include "util.hh"
+#include "types.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+/* ----------- tests for util.hh ------------------------------------------------*/
+
+    /* ----------------------------------------------------------------------------
+     * absPath
+     * --------------------------------------------------------------------------*/
+
+    TEST(absPath, doesntChangeRoot) {
+        auto p = absPath("/");
+
+        ASSERT_EQ(p, "/");
+    }
+
+
+
+
+    TEST(absPath, turnsEmptyPathIntoCWD) {
+        char cwd[PATH_MAX+1];
+        auto p = absPath("");
+
+        ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX));
+    }
+
+    TEST(absPath, usesOptionalBasePathWhenGiven) {
+        char _cwd[PATH_MAX+1];
+        char* cwd = getcwd((char*)&_cwd, PATH_MAX);
+
+        auto p = absPath("", cwd);
+
+        ASSERT_EQ(p, cwd);
+    }
+
+    TEST(absPath, isIdempotent) {
+        char _cwd[PATH_MAX+1];
+        char* cwd = getcwd((char*)&_cwd, PATH_MAX);
+        auto p1 = absPath(cwd);
+        auto p2 = absPath(p1);
+
+        ASSERT_EQ(p1, p2);
+    }
+
+
+    TEST(absPath, pathIsCanonicalised) {
+        auto path = "/some/path/with/trailing/dot/.";
+        auto p1 = absPath(path);
+        auto p2 = absPath(p1);
+
+        ASSERT_EQ(p1, "/some/path/with/trailing/dot");
+        ASSERT_EQ(p1, p2);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * canonPath
+     * --------------------------------------------------------------------------*/
+
+    TEST(canonPath, removesTrailingSlashes) {
+        auto path = "/this/is/a/path//";
+        auto p = canonPath(path);
+
+        ASSERT_EQ(p, "/this/is/a/path");
+    }
+
+    TEST(canonPath, removesDots) {
+        auto path = "/this/./is/a/path/./";
+        auto p = canonPath(path);
+
+        ASSERT_EQ(p, "/this/is/a/path");
+    }
+
+    TEST(canonPath, removesDots2) {
+        auto path = "/this/a/../is/a////path/foo/..";
+        auto p = canonPath(path);
+
+        ASSERT_EQ(p, "/this/is/a/path");
+    }
+
+    TEST(canonPath, requiresAbsolutePath) {
+        ASSERT_ANY_THROW(canonPath("."));
+        ASSERT_ANY_THROW(canonPath(".."));
+        ASSERT_ANY_THROW(canonPath("../"));
+        ASSERT_DEATH({ canonPath(""); }, "path != \"\"");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * dirOf
+     * --------------------------------------------------------------------------*/
+
+    TEST(dirOf, returnsEmptyStringForRoot) {
+        auto p = dirOf("/");
+
+        ASSERT_EQ(p, "/");
+    }
+
+    TEST(dirOf, returnsFirstPathComponent) {
+        auto p1 = dirOf("/dir/");
+        ASSERT_EQ(p1, "/dir");
+        auto p2 = dirOf("/dir");
+        ASSERT_EQ(p2, "/");
+        auto p3 = dirOf("/dir/..");
+        ASSERT_EQ(p3, "/dir");
+        auto p4 = dirOf("/dir/../");
+        ASSERT_EQ(p4, "/dir/..");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * baseNameOf
+     * --------------------------------------------------------------------------*/
+
+    TEST(baseNameOf, emptyPath) {
+        auto p1 = baseNameOf("");
+        ASSERT_EQ(p1, "");
+    }
+
+    TEST(baseNameOf, pathOnRoot) {
+        auto p1 = baseNameOf("/dir");
+        ASSERT_EQ(p1, "dir");
+    }
+
+    TEST(baseNameOf, relativePath) {
+        auto p1 = baseNameOf("dir/foo");
+        ASSERT_EQ(p1, "foo");
+    }
+
+    TEST(baseNameOf, pathWithTrailingSlashRoot) {
+        auto p1 = baseNameOf("/");
+        ASSERT_EQ(p1, "");
+    }
+
+    TEST(baseNameOf, trailingSlash) {
+        auto p1 = baseNameOf("/dir/");
+        ASSERT_EQ(p1, "dir");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * isInDir
+     * --------------------------------------------------------------------------*/
+
+    TEST(isInDir, trivialCase) {
+        auto p1 = isInDir("/foo/bar", "/foo");
+        ASSERT_EQ(p1, true);
+    }
+
+    TEST(isInDir, notInDir) {
+        auto p1 = isInDir("/zes/foo/bar", "/foo");
+        ASSERT_EQ(p1, false);
+    }
+
+    // XXX: hm, bug or feature? :) Looking at the implementation
+    // this might be problematic.
+    TEST(isInDir, emptyDir) {
+        auto p1 = isInDir("/zes/foo/bar", "");
+        ASSERT_EQ(p1, true);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * isDirOrInDir
+     * --------------------------------------------------------------------------*/
+
+    TEST(isDirOrInDir, trueForSameDirectory) {
+        ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true);
+        ASSERT_EQ(isDirOrInDir("/", "/"), true);
+    }
+
+    TEST(isDirOrInDir, trueForEmptyPaths) {
+        ASSERT_EQ(isDirOrInDir("", ""), true);
+    }
+
+    TEST(isDirOrInDir, falseForDisjunctPaths) {
+        ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false);
+    }
+
+    TEST(isDirOrInDir, relativePaths) {
+        ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true);
+    }
+
+    // XXX: while it is possible to use "." or ".." in the
+    // first argument this doesn't seem to work in the second.
+    TEST(isDirOrInDir, DISABLED_shouldWork) {
+        ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true);
+
+    }
+
+    /* ----------------------------------------------------------------------------
+     * pathExists
+     * --------------------------------------------------------------------------*/
+
+    TEST(pathExists, rootExists) {
+        ASSERT_TRUE(pathExists("/"));
+    }
+
+    TEST(pathExists, cwdExists) {
+        ASSERT_TRUE(pathExists("."));
+    }
+
+    TEST(pathExists, bogusPathDoesNotExist) {
+        ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes"));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * concatStringsSep
+     * --------------------------------------------------------------------------*/
+
+    TEST(concatStringsSep, buildCommaSeparatedString) {
+        Strings strings;
+        strings.push_back("this");
+        strings.push_back("is");
+        strings.push_back("great");
+
+        ASSERT_EQ(concatStringsSep(",", strings), "this,is,great");
+    }
+
+    TEST(concatStringsSep, buildStringWithEmptySeparator) {
+        Strings strings;
+        strings.push_back("this");
+        strings.push_back("is");
+        strings.push_back("great");
+
+        ASSERT_EQ(concatStringsSep("", strings), "thisisgreat");
+    }
+
+    TEST(concatStringsSep, buildSingleString) {
+        Strings strings;
+        strings.push_back("this");
+
+        ASSERT_EQ(concatStringsSep(",", strings), "this");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * hasPrefix
+     * --------------------------------------------------------------------------*/
+
+    TEST(hasPrefix, emptyStringHasNoPrefix) {
+        ASSERT_FALSE(hasPrefix("", "foo"));
+    }
+
+    TEST(hasPrefix, emptyStringIsAlwaysPrefix) {
+        ASSERT_TRUE(hasPrefix("foo", ""));
+        ASSERT_TRUE(hasPrefix("jshjkfhsadf", ""));
+    }
+
+    TEST(hasPrefix, trivialCase) {
+        ASSERT_TRUE(hasPrefix("foobar", "foo"));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * hasSuffix
+     * --------------------------------------------------------------------------*/
+
+    TEST(hasSuffix, emptyStringHasNoSuffix) {
+        ASSERT_FALSE(hasSuffix("", "foo"));
+    }
+
+    TEST(hasSuffix, trivialCase) {
+        ASSERT_TRUE(hasSuffix("foo", "foo"));
+        ASSERT_TRUE(hasSuffix("foobar", "bar"));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * base64Encode
+     * --------------------------------------------------------------------------*/
+
+    TEST(base64Encode, emptyString) {
+        ASSERT_EQ(base64Encode(""), "");
+    }
+
+    TEST(base64Encode, encodesAString) {
+        ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0=");
+    }
+
+    TEST(base64Encode, encodeAndDecode) {
+        auto s = "quod erat demonstrandum";
+        auto encoded = base64Encode(s);
+        auto decoded = base64Decode(encoded);
+
+        ASSERT_EQ(decoded, s);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * base64Decode
+     * --------------------------------------------------------------------------*/
+
+    TEST(base64Decode, emptyString) {
+        ASSERT_EQ(base64Decode(""), "");
+    }
+
+    TEST(base64Decode, decodeAString) {
+        ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * toLower
+     * --------------------------------------------------------------------------*/
+
+    TEST(toLower, emptyString) {
+        ASSERT_EQ(toLower(""), "");
+    }
+
+    TEST(toLower, nonLetters) {
+        auto s = "!@(*$#)(@#=\\234_";
+        ASSERT_EQ(toLower(s), s);
+    }
+
+    // std::tolower() doesn't handle unicode characters. In the context of
+    // store paths this isn't relevant but doesn't hurt to record this behavior
+    // here.
+    TEST(toLower, umlauts) {
+        auto s = "ÄÖÜ";
+        ASSERT_EQ(toLower(s), "ÄÖÜ");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * string2Float
+     * --------------------------------------------------------------------------*/
+
+    TEST(string2Float, emptyString) {
+        double n;
+        ASSERT_EQ(string2Float("", n), false);
+    }
+
+    TEST(string2Float, trivialConversions) {
+        double n;
+        ASSERT_EQ(string2Float("1.0", n), true);
+        ASSERT_EQ(n, 1.0);
+
+        ASSERT_EQ(string2Float("0.0", n), true);
+        ASSERT_EQ(n, 0.0);
+
+        ASSERT_EQ(string2Float("-100.25", n), true);
+        ASSERT_EQ(n, (-100.25));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * string2Int
+     * --------------------------------------------------------------------------*/
+
+    TEST(string2Int, emptyString) {
+        double n;
+        ASSERT_EQ(string2Int("", n), false);
+    }
+
+    TEST(string2Int, trivialConversions) {
+        double n;
+        ASSERT_EQ(string2Int("1", n), true);
+        ASSERT_EQ(n, 1);
+
+        ASSERT_EQ(string2Int("0", n), true);
+        ASSERT_EQ(n, 0);
+
+        ASSERT_EQ(string2Int("-100", n), true);
+        ASSERT_EQ(n, (-100));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * statusOk
+     * --------------------------------------------------------------------------*/
+
+    TEST(statusOk, zeroIsOk) {
+        ASSERT_EQ(statusOk(0), true);
+        ASSERT_EQ(statusOk(1), false);
+    }
+
+
+    /* ----------------------------------------------------------------------------
+     * rewriteStrings
+     * --------------------------------------------------------------------------*/
+
+    TEST(rewriteStrings, emptyString) {
+        StringMap rewrites;
+        rewrites["this"] = "that";
+
+        ASSERT_EQ(rewriteStrings("", rewrites), "");
+    }
+
+    TEST(rewriteStrings, emptyRewrites) {
+        StringMap rewrites;
+
+        ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that");
+    }
+
+    TEST(rewriteStrings, successfulRewrite) {
+        StringMap rewrites;
+        rewrites["this"] = "that";
+
+        ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that");
+    }
+
+    TEST(rewriteStrings, doesntOccur) {
+        StringMap rewrites;
+        rewrites["foo"] = "bar";
+
+        ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * replaceStrings
+     * --------------------------------------------------------------------------*/
+
+    TEST(replaceStrings, emptyString) {
+        ASSERT_EQ(replaceStrings("", "this", "that"), "");
+        ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that");
+    }
+
+    TEST(replaceStrings, successfulReplace) {
+        ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that");
+    }
+
+    TEST(replaceStrings, doesntOccur) {
+        ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * trim
+     * --------------------------------------------------------------------------*/
+
+    TEST(trim, emptyString) {
+        ASSERT_EQ(trim(""), "");
+    }
+
+    TEST(trim, removesWhitespace) {
+        ASSERT_EQ(trim("foo"), "foo");
+        ASSERT_EQ(trim("     foo "), "foo");
+        ASSERT_EQ(trim("     foo bar baz"), "foo bar baz");
+        ASSERT_EQ(trim("     \t foo bar baz\n"), "foo bar baz");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * chomp
+     * --------------------------------------------------------------------------*/
+
+    TEST(chomp, emptyString) {
+        ASSERT_EQ(chomp(""), "");
+    }
+
+    TEST(chomp, removesWhitespace) {
+        ASSERT_EQ(chomp("foo"), "foo");
+        ASSERT_EQ(chomp("foo "), "foo");
+        ASSERT_EQ(chomp(" foo "), " foo");
+        ASSERT_EQ(chomp(" foo bar baz  "), " foo bar baz");
+        ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz");
+    }
+
+    /* ----------------------------------------------------------------------------
+     * quoteStrings
+     * --------------------------------------------------------------------------*/
+
+    TEST(quoteStrings, empty) {
+        Strings s = { };
+        Strings expected = { };
+
+        ASSERT_EQ(quoteStrings(s), expected);
+    }
+
+    TEST(quoteStrings, emptyStrings) {
+        Strings s = { "", "", "" };
+        Strings expected = { "''", "''", "''" };
+        ASSERT_EQ(quoteStrings(s), expected);
+
+    }
+
+    TEST(quoteStrings, trivialQuote) {
+        Strings s = { "foo", "bar", "baz" };
+        Strings expected = { "'foo'", "'bar'", "'baz'" };
+
+        ASSERT_EQ(quoteStrings(s), expected);
+    }
+
+    TEST(quoteStrings, quotedStrings) {
+        Strings s = { "'foo'", "'bar'", "'baz'" };
+        Strings expected = { "''foo''", "''bar''", "''baz''" };
+
+        ASSERT_EQ(quoteStrings(s), expected);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * tokenizeString
+     * --------------------------------------------------------------------------*/
+
+    TEST(tokenizeString, empty) {
+        Strings expected = { };
+
+        ASSERT_EQ(tokenizeString<Strings>(""), expected);
+    }
+
+    TEST(tokenizeString, tokenizeSpacesWithDefaults) {
+        auto s = "foo bar baz";
+        Strings expected = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s), expected);
+    }
+
+    TEST(tokenizeString, tokenizeTabsWithDefaults) {
+        auto s = "foo\tbar\tbaz";
+        Strings expected = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s), expected);
+    }
+
+    TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) {
+        auto s = "foo\t bar\t baz";
+        Strings expected = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s), expected);
+    }
+
+    TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) {
+        auto s = "foo\t\n bar\t\n baz";
+        Strings expected = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s), expected);
+    }
+
+    TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) {
+        auto s = "foo\t\n\r bar\t\n\r baz";
+        Strings expected = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s), expected);
+
+        auto s2 = "foo \t\n\r bar \t\n\r baz";
+        Strings expected2 = { "foo", "bar", "baz" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s2), expected2);
+    }
+
+    TEST(tokenizeString, tokenizeWithCustomSep) {
+        auto s = "foo\n,bar\n,baz\n";
+        Strings expected = { "foo\n", "bar\n", "baz\n" };
+
+        ASSERT_EQ(tokenizeString<Strings>(s, ","), expected);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * get
+     * --------------------------------------------------------------------------*/
+
+    TEST(get, emptyContainer) {
+        StringMap s = { };
+        auto expected = std::nullopt;
+
+        ASSERT_EQ(get(s, "one"), expected);
+    }
+
+    TEST(get, getFromContainer) {
+        StringMap s;
+        s["one"] = "yi";
+        s["two"] = "er";
+        auto expected = "yi";
+
+        ASSERT_EQ(get(s, "one"), expected);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * filterANSIEscapes
+     * --------------------------------------------------------------------------*/
+
+    TEST(filterANSIEscapes, emptyString) {
+        auto s = "";
+        auto expected = "";
+
+        ASSERT_EQ(filterANSIEscapes(s), expected);
+    }
+
+    TEST(filterANSIEscapes, doesntChangePrintableChars) {
+        auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f";
+
+        ASSERT_EQ(filterANSIEscapes(s), s);
+    }
+
+    TEST(filterANSIEscapes, filtersColorCodes) {
+        auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m";
+
+        ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" );
+        ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " );
+        ASSERT_EQ(filterANSIEscapes(s, true, 4), " A  " );
+        ASSERT_EQ(filterANSIEscapes(s, true, 5), " A  B" );
+        ASSERT_EQ(filterANSIEscapes(s, true, 8), " A  B  C" );
+    }
+
+    TEST(filterANSIEscapes, expandsTabs) {
+        auto s = "foo\tbar\tbaz";
+
+        ASSERT_EQ(filterANSIEscapes(s, true), "foo     bar     baz" );
+    }
+}
diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc
new file mode 100644
index 000000000..80646ad3e
--- /dev/null
+++ b/src/libutil/tests/url.cc
@@ -0,0 +1,266 @@
+#include "url.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+/* ----------- tests for url.hh --------------------------------------------------*/
+
+    string print_map(std::map<string, string> m) {
+        std::map<string, string>::iterator it;
+        string s = "{ ";
+        for (it = m.begin(); it != m.end(); ++it) {
+            s += "{ ";
+            s += it->first;
+            s += " = ";
+            s += it->second;
+            s += " } ";
+        }
+        s += "}";
+        return s;
+    }
+
+
+    std::ostream& operator<<(std::ostream& os, const ParsedURL& p) {
+        return os << "\n"
+                  << "url:       " << p.url << "\n"
+                  << "base:      " << p.base << "\n"
+                  << "scheme:    " << p.scheme << "\n"
+                  << "authority: " << p.authority.value() << "\n"
+                  << "path:      " << p.path << "\n"
+                  << "query:     " << print_map(p.query) << "\n"
+                  << "fragment:  " << p.fragment << "\n";
+    }
+
+    TEST(parseURL, parsesSimpleHttpUrl) {
+        auto s = "http://www.example.org/file.tar.gz";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://www.example.org/file.tar.gz",
+            .base = "http://www.example.org/file.tar.gz",
+            .scheme = "http",
+            .authority = "www.example.org",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parsesSimpleHttpsUrl) {
+        auto s = "https://www.example.org/file.tar.gz";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "https://www.example.org/file.tar.gz",
+            .base = "https://www.example.org/file.tar.gz",
+            .scheme = "https",
+            .authority = "www.example.org",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parsesSimpleHttpUrlWithQueryAndFragment) {
+        auto s = "https://www.example.org/file.tar.gz?download=fast&when=now#hello";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "https://www.example.org/file.tar.gz",
+            .base = "https://www.example.org/file.tar.gz",
+            .scheme = "https",
+            .authority = "www.example.org",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { { "download", "fast" }, { "when", "now" } },
+            .fragment = "hello",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment) {
+        auto s = "http://www.example.org/file.tar.gz?field=value#?foo=bar%23";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://www.example.org/file.tar.gz",
+            .base = "http://www.example.org/file.tar.gz",
+            .scheme = "http",
+            .authority = "www.example.org",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { { "field", "value" } },
+            .fragment = "?foo=bar#",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+
+    TEST(parseURL, parseIPv4Address) {
+        auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://127.0.0.1:8080/file.tar.gz",
+            .base = "https://127.0.0.1:8080/file.tar.gz",
+            .scheme = "http",
+            .authority = "127.0.0.1:8080",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { { "download", "fast" }, { "when", "now" } },
+            .fragment = "hello",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parseIPv6Address) {
+        auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
+            .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
+            .scheme = "http",
+            .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080",
+            .path = "",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+
+    }
+
+    TEST(parseURL, parseEmptyQueryParams) {
+        auto s = "http://127.0.0.1:8080/file.tar.gz?&&&&&";
+        auto parsed = parseURL(s);
+        ASSERT_EQ(parsed.query, (StringMap) { });
+    }
+
+    TEST(parseURL, parseUserPassword) {
+        auto s = "http://user:pass@www.example.org:8080/file.tar.gz";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "http://user:pass@www.example.org/file.tar.gz",
+            .base = "http://user:pass@www.example.org/file.tar.gz",
+            .scheme = "http",
+            .authority = "user:pass@www.example.org:8080",
+            .path = "/file.tar.gz",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parseFileURLWithQueryAndFragment) {
+        auto s = "file:///none/of/your/business";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "",
+            .base = "",
+            .scheme = "file",
+            .authority = "",
+            .path = "/none/of/your/business",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+
+    }
+
+    TEST(parseURL, parsedUrlsIsEqualToItself) {
+        auto s = "http://www.example.org/file.tar.gz";
+        auto url = parseURL(s);
+
+        ASSERT_TRUE(url == url);
+    }
+
+    TEST(parseURL, parseFTPUrl) {
+        auto s = "ftp://ftp.nixos.org/downloads/nixos.iso";
+        auto parsed = parseURL(s);
+
+        ParsedURL expected {
+            .url = "ftp://ftp.nixos.org/downloads/nixos.iso",
+            .base = "ftp://ftp.nixos.org/downloads/nixos.iso",
+            .scheme = "ftp",
+            .authority = "ftp.nixos.org",
+            .path = "/downloads/nixos.iso",
+            .query = (StringMap) { },
+            .fragment = "",
+        };
+
+        ASSERT_EQ(parsed, expected);
+    }
+
+    TEST(parseURL, parsesAnythingInUriFormat) {
+        auto s = "whatever://github.com/NixOS/nixpkgs.git";
+        auto parsed = parseURL(s);
+    }
+
+    TEST(parseURL, parsesAnythingInUriFormatWithoutDoubleSlash) {
+        auto s = "whatever:github.com/NixOS/nixpkgs.git";
+        auto parsed = parseURL(s);
+    }
+
+    TEST(parseURL, emptyStringIsInvalidURL) {
+        ASSERT_THROW(parseURL(""), Error);
+    }
+
+    /* ----------------------------------------------------------------------------
+     * decodeQuery
+     * --------------------------------------------------------------------------*/
+
+    TEST(decodeQuery, emptyStringYieldsEmptyMap) {
+        auto d = decodeQuery("");
+        ASSERT_EQ(d, (StringMap) { });
+    }
+
+    TEST(decodeQuery, simpleDecode) {
+        auto d = decodeQuery("yi=one&er=two");
+        ASSERT_EQ(d, ((StringMap) { { "yi", "one" }, { "er", "two" } }));
+    }
+
+    TEST(decodeQuery, decodeUrlEncodedArgs) {
+        auto d = decodeQuery("arg=%3D%3D%40%3D%3D");
+        ASSERT_EQ(d, ((StringMap) { { "arg", "==@==" } }));
+    }
+
+    TEST(decodeQuery, decodeArgWithEmptyValue) {
+        auto d = decodeQuery("arg=");
+        ASSERT_EQ(d, ((StringMap) { { "arg", ""} }));
+    }
+
+    /* ----------------------------------------------------------------------------
+     * percentDecode
+     * --------------------------------------------------------------------------*/
+
+    TEST(percentDecode, decodesUrlEncodedString) {
+        string s = "==@==";
+        string d = percentDecode("%3D%3D%40%3D%3D");
+        ASSERT_EQ(d, s);
+    }
+
+    TEST(percentDecode, multipleDecodesAreIdempotent) {
+        string once = percentDecode("%3D%3D%40%3D%3D");
+        string twice = percentDecode(once);
+
+        ASSERT_EQ(once, twice);
+    }
+
+    TEST(percentDecode, trailingPercent) {
+        string s = "==@==%";
+        string d = percentDecode("%3D%3D%40%3D%3D%25");
+
+        ASSERT_EQ(d, s);
+    }
+
+}
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 20b96a85c..250c9581d 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -41,7 +41,8 @@ struct FormatOrString
 {
     string s;
     FormatOrString(const string & s) : s(s) { };
-    FormatOrString(const format & f) : s(f.str()) { };
+    template<class F>
+    FormatOrString(const F & f) : s(f.str()) { };
     FormatOrString(const char * s) : s(s) { };
 };
 
@@ -51,12 +52,13 @@ struct FormatOrString
    ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion
    takes place). */
 
-inline void formatHelper(boost::format & f)
+template<class F>
+inline void formatHelper(F & f)
 {
 }
 
-template<typename T, typename... Args>
-inline void formatHelper(boost::format & f, const T & x, const Args & ... args)
+template<class F, typename T, typename... Args>
+inline void formatHelper(F & f, const T & x, const Args & ... args)
 {
     formatHelper(f % x, args...);
 }
@@ -157,4 +159,12 @@ typedef list<Path> Paths;
 typedef set<Path> PathSet;
 
 
+/* Helper class to run code at startup. */
+template<typename T>
+struct OnStartup
+{
+    OnStartup(T && t) { t(); }
+};
+
+
 }
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
new file mode 100644
index 000000000..5d5328e5d
--- /dev/null
+++ b/src/libutil/url.cc
@@ -0,0 +1,137 @@
+#include "url.hh"
+#include "util.hh"
+
+namespace nix {
+
+std::regex refRegex(refRegexS, std::regex::ECMAScript);
+std::regex revRegex(revRegexS, std::regex::ECMAScript);
+std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript);
+
+ParsedURL parseURL(const std::string & url)
+{
+    static std::regex uriRegex(
+        "((" + schemeRegex + "):"
+        + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))"
+        + "(?:\\?(" + queryRegex + "))?"
+        + "(?:#(" + queryRegex + "))?",
+        std::regex::ECMAScript);
+
+    std::smatch match;
+
+    if (std::regex_match(url, match, uriRegex)) {
+        auto & base = match[1];
+        std::string scheme = match[2];
+        auto authority = match[3].matched
+            ? std::optional<std::string>(match[3]) : std::nullopt;
+        std::string path = match[4].matched ? match[4] : match[5];
+        auto & query = match[6];
+        auto & fragment = match[7];
+
+        auto isFile = scheme.find("file") != std::string::npos;
+
+        if (authority && *authority != "" && isFile)
+            throw Error("file:// URL '%s' has unexpected authority '%s'",
+                url, *authority);
+
+        if (isFile && path.empty())
+            path = "/";
+
+        return ParsedURL{
+            .url = url,
+            .base = base,
+            .scheme = scheme,
+            .authority = authority,
+            .path = path,
+            .query = decodeQuery(query),
+            .fragment = percentDecode(std::string(fragment))
+        };
+    }
+
+    else
+        throw BadURL("'%s' is not a valid URL", url);
+}
+
+std::string percentDecode(std::string_view in)
+{
+    std::string decoded;
+    for (size_t i = 0; i < in.size(); ) {
+        if (in[i] == '%') {
+            if (i + 2 >= in.size())
+                throw BadURL("invalid URI parameter '%s'", in);
+            try {
+                decoded += std::stoul(std::string(in, i + 1, 2), 0, 16);
+                i += 3;
+            } catch (...) {
+                throw BadURL("invalid URI parameter '%s'", in);
+            }
+        } else
+            decoded += in[i++];
+    }
+    return decoded;
+}
+
+std::map<std::string, std::string> decodeQuery(const std::string & query)
+{
+    std::map<std::string, std::string> result;
+
+    for (auto s : tokenizeString<Strings>(query, "&")) {
+        auto e = s.find('=');
+        if (e != std::string::npos)
+            result.emplace(
+                s.substr(0, e),
+                percentDecode(std::string_view(s).substr(e + 1)));
+    }
+
+    return result;
+}
+
+std::string percentEncode(std::string_view s)
+{
+    std::string res;
+    for (auto & c : s)
+        if ((c >= 'a' && c <= 'z')
+            || (c >= 'A' && c <= 'Z')
+            || (c >= '0' && c <= '9')
+            || strchr("-._~!$&'()*+,;=:@", c))
+            res += c;
+        else
+            res += fmt("%%%02x", (unsigned int) c);
+    return res;
+}
+
+std::string encodeQuery(const std::map<std::string, std::string> & ss)
+{
+    std::string res;
+    bool first = true;
+    for (auto & [name, value] : ss) {
+        if (!first) res += '&';
+        first = false;
+        res += percentEncode(name);
+        res += '=';
+        res += percentEncode(value);
+    }
+    return res;
+}
+
+std::string ParsedURL::to_string() const
+{
+    return
+        scheme
+        + ":"
+        + (authority ? "//" + *authority : "")
+        + path
+        + (query.empty() ? "" : "?" + encodeQuery(query))
+        + (fragment.empty() ? "" : "#" + percentEncode(fragment));
+}
+
+bool ParsedURL::operator ==(const ParsedURL & other) const
+{
+    return
+        scheme == other.scheme
+        && authority == other.authority
+        && path == other.path
+        && query == other.query
+        && fragment == other.fragment;
+}
+
+}
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
new file mode 100644
index 000000000..1503023a2
--- /dev/null
+++ b/src/libutil/url.hh
@@ -0,0 +1,62 @@
+#pragma once
+
+#include "types.hh"
+
+#include <regex>
+
+namespace nix {
+
+struct ParsedURL
+{
+    std::string url;
+    std::string base; // URL without query/fragment
+    std::string scheme;
+    std::optional<std::string> authority;
+    std::string path;
+    std::map<std::string, std::string> query;
+    std::string fragment;
+
+    std::string to_string() const;
+
+    bool operator ==(const ParsedURL & other) const;
+};
+
+MakeError(BadURL, Error);
+
+std::string percentDecode(std::string_view in);
+
+std::map<std::string, std::string> decodeQuery(const std::string & query);
+
+ParsedURL parseURL(const std::string & url);
+
+// URI stuff.
+const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
+const static std::string schemeRegex = "(?:[a-z+]+)";
+const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
+const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
+const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
+const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
+const static std::string hostRegex = "(?:" + ipv6AddressRegex + "|" + hostnameRegex + ")";
+const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|:)*)";
+const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?";
+const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])";
+const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*";
+const static std::string segmentRegex = "(?:" + pcharRegex + "+)";
+const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
+const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
+
+// A Git ref (i.e. branch or tag name).
+const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
+extern std::regex refRegex;
+
+// A Git revision (a SHA-1 commit hash).
+const static std::string revRegexS = "[0-9a-fA-F]{40}";
+extern std::regex revRegex;
+
+// A ref or revision, or a ref followed by a revision.
+const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))";
+
+const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*";
+extern std::regex flakeIdRegex;
+
+}
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 097ff210a..71db92d77 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -268,16 +268,13 @@ bool isLink(const Path & path)
 }
 
 
-DirEntries readDirectory(const Path & path)
+DirEntries readDirectory(DIR *dir, const Path & path)
 {
     DirEntries entries;
     entries.reserve(64);
 
-    AutoCloseDir dir(opendir(path.c_str()));
-    if (!dir) throw SysError(format("opening directory '%1%'") % path);
-
     struct dirent * dirent;
-    while (errno = 0, dirent = readdir(dir.get())) { /* sic */
+    while (errno = 0, dirent = readdir(dir)) { /* sic */
         checkInterrupt();
         string name = dirent->d_name;
         if (name == "." || name == "..") continue;
@@ -294,6 +291,14 @@ DirEntries readDirectory(const Path & path)
     return entries;
 }
 
+DirEntries readDirectory(const Path & path)
+{
+    AutoCloseDir dir(opendir(path.c_str()));
+    if (!dir) throw SysError(format("opening directory '%1%'") % path);
+
+    return readDirectory(dir.get(), path);
+}
+
 
 unsigned char getFileType(const Path & path)
 {
@@ -311,19 +316,16 @@ string readFile(int fd)
     if (fstat(fd, &st) == -1)
         throw SysError("statting file");
 
-    std::vector<unsigned char> buf(st.st_size);
-    readFull(fd, buf.data(), st.st_size);
-
-    return string((char *) buf.data(), st.st_size);
+    return drainFD(fd, true, st.st_size);
 }
 
 
-string readFile(const Path & path, bool drain)
+string readFile(const Path & path)
 {
     AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
     if (!fd)
         throw SysError(format("opening file '%1%'") % path);
-    return drain ? drainFD(fd.get()) : readFile(fd.get());
+    return readFile(fd.get());
 }
 
 
@@ -389,12 +391,14 @@ void writeLine(int fd, string s)
 }
 
 
-static void _deletePath(const Path & path, unsigned long long & bytesFreed)
+static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed)
 {
     checkInterrupt();
 
+    string name(baseNameOf(path));
+
     struct stat st;
-    if (lstat(path.c_str(), &st) == -1) {
+    if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) {
         if (errno == ENOENT) return;
         throw SysError(format("getting status of '%1%'") % path);
     }
@@ -406,20 +410,45 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
         /* Make the directory accessible. */
         const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR;
         if ((st.st_mode & PERM_MASK) != PERM_MASK) {
-            if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1)
+            if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1)
                 throw SysError(format("chmod '%1%'") % path);
         }
 
-        for (auto & i : readDirectory(path))
-            _deletePath(path + "/" + i.name, bytesFreed);
+        int fd = openat(parentfd, path.c_str(), O_RDONLY);
+        if (!fd)
+            throw SysError(format("opening directory '%1%'") % path);
+        AutoCloseDir dir(fdopendir(fd));
+        if (!dir)
+            throw SysError(format("opening directory '%1%'") % path);
+        for (auto & i : readDirectory(dir.get(), path))
+            _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed);
     }
 
-    if (remove(path.c_str()) == -1) {
+    int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0;
+    if (unlinkat(parentfd, name.c_str(), flags) == -1) {
         if (errno == ENOENT) return;
         throw SysError(format("cannot unlink '%1%'") % path);
     }
 }
 
+static void _deletePath(const Path & path, unsigned long long & bytesFreed)
+{
+    Path dir = dirOf(path);
+    if (dir == "")
+        dir = "/";
+
+    AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY));
+    if (!dirfd) {
+        // This really shouldn't fail silently, but it's left this way
+        // for backwards compatibility.
+        if (errno == ENOENT) return;
+
+        throw SysError(format("opening directory '%1%'") % path);
+    }
+
+    _deletePath(dirfd.get(), path, bytesFreed);
+}
+
 
 void deletePath(const Path & path)
 {
@@ -478,6 +507,17 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
 }
 
 
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
+{
+    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    // Strictly speaking, this is UB, but who cares...
+    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
+    if (!fd)
+        throw SysError("creating temporary file '%s'", tmpl);
+    return {std::move(fd), tmpl};
+}
+
+
 std::string getUserName()
 {
     auto pw = getpwuid(geteuid());
@@ -622,9 +662,9 @@ void writeFull(int fd, const string & s, bool allowInterrupts)
 }
 
 
-string drainFD(int fd, bool block)
+string drainFD(int fd, bool block, const size_t reserveSize)
 {
-    StringSink sink;
+    StringSink sink(reserveSize);
     drainFD(fd, sink, block);
     return std::move(*sink.s);
 }
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 7c3a30242..a63ee05b3 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -2,6 +2,7 @@
 
 #include "types.hh"
 #include "logging.hh"
+#include "ansicolor.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -16,6 +17,7 @@
 #include <sstream>
 #include <optional>
 #include <future>
+#include <iterator>
 
 #ifndef HAVE_STRUCT_DIRENT_D_TYPE
 #define DT_UNKNOWN 0
@@ -56,12 +58,12 @@ Path canonPath(const Path & path, bool resolveSymlinks = false);
 
 /* Return the directory part of the given canonical path, i.e.,
    everything before the final `/'.  If the path is the root or an
-   immediate child thereof (e.g., `/foo'), this means an empty string
-   is returned. */
+   immediate child thereof (e.g., `/foo'), this means `/'
+   is returned.*/
 Path dirOf(const Path & path);
 
 /* Return the base name of the given canonical path, i.e., everything
-   following the final `/'. */
+   following the final `/' (trailing slashes are removed). */
 std::string_view baseNameOf(std::string_view path);
 
 /* Check whether 'path' is a descendant of 'dir'. */
@@ -101,7 +103,7 @@ unsigned char getFileType(const Path & path);
 
 /* Read the contents of a file into a string. */
 string readFile(int fd);
-string readFile(const Path & path, bool drain = false);
+string readFile(const Path & path);
 void readFile(const Path & path, Sink & sink);
 
 /* Write a string to a file. */
@@ -122,10 +124,6 @@ void deletePath(const Path & path);
 
 void deletePath(const Path & path, unsigned long long & bytesFreed);
 
-/* Create a temporary directory. */
-Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
-    bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
-
 std::string getUserName();
 
 /* Return $HOME or the user's home directory from /etc/passwd. */
@@ -164,7 +162,7 @@ MakeError(EndOfFile, Error);
 
 
 /* Read a file descriptor until EOF occurs. */
-string drainFD(int fd, bool block = true);
+string drainFD(int fd, bool block = true, const size_t reserveSize=0);
 
 void drainFD(int fd, Sink & sink, bool block = true);
 
@@ -205,6 +203,14 @@ public:
 };
 
 
+/* Create a temporary directory. */
+Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
+    bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
+
+/* Create a temporary file, returning a file handle and its path. */
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix = "nix");
+
+
 class Pipe
 {
 public:
@@ -383,17 +389,6 @@ string replaceStrings(const std::string & s,
 std::string rewriteStrings(const std::string & s, const StringMap & rewrites);
 
 
-/* If a set contains 'from', remove it and insert 'to'. */
-template<typename T>
-void replaceInSet(std::set<T> & set, const T & from, const T & to)
-{
-    auto i = set.find(from);
-    if (i == set.end()) return;
-    set.erase(i);
-    set.insert(to);
-}
-
-
 /* Convert the exit status of a child as returned by wait() into an
    error string. */
 string statusToString(int status);
@@ -441,15 +436,6 @@ std::string shellEscape(const std::string & s);
 void ignoreException();
 
 
-/* Some ANSI escape sequences. */
-#define ANSI_NORMAL "\e[0m"
-#define ANSI_BOLD "\e[1m"
-#define ANSI_FAINT "\e[2m"
-#define ANSI_RED "\e[31;1m"
-#define ANSI_GREEN "\e[32;1m"
-#define ANSI_YELLOW "\e[33;1m"
-#define ANSI_BLUE "\e[34;1m"
-
 
 /* Tree formatting. */
 constexpr char treeConn[] = "├───";