Merge remote-tracking branch 'origin/master' into substitute-other-storedir

13 files changed, 405 insertions, 271 deletions

diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh
index a38c2d798..ae741f867 100644
--- a/src/libutil/ansicolor.hh
+++ b/src/libutil/ansicolor.hh
@@ -11,7 +11,7 @@ namespace nix {
 #define ANSI_GREEN "\e[32;1m"
 #define ANSI_YELLOW "\e[33;1m"
 #define ANSI_BLUE "\e[34;1m"
-#define ANSI_MAGENTA "\e[35m;1m"
-#define ANSI_CYAN "\e[36m;1m"
+#define ANSI_MAGENTA "\e[35;1m"
+#define ANSI_CYAN "\e[36;1m"
 
 }
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 51c88537e..ce7cf9754 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -150,17 +150,17 @@ static void skipGeneric(Source & source)
 
 static void parseContents(ParseSink & sink, Source & source, const Path & path)
 {
-    unsigned long long size = readLongLong(source);
+    uint64_t size = readLongLong(source);
 
     sink.preallocateContents(size);
 
-    unsigned long long left = size;
+    uint64_t left = size;
     std::vector<unsigned char> buf(65536);
 
     while (left) {
         checkInterrupt();
         auto n = buf.size();
-        if ((unsigned long long)n > left) n = left;
+        if ((uint64_t)n > left) n = left;
         source(buf.data(), n);
         sink.receiveContents(buf.data(), n);
         left -= n;
@@ -323,7 +323,7 @@ struct RestoreSink : ParseSink
             throw SysError("fchmod");
     }
 
-    void preallocateContents(unsigned long long len)
+    void preallocateContents(uint64_t len)
     {
 #if HAVE_POSIX_FALLOCATE
         if (len) {
@@ -338,7 +338,7 @@ struct RestoreSink : ParseSink
 #endif
     }
 
-    void receiveContents(unsigned char * data, unsigned int len)
+    void receiveContents(unsigned char * data, size_t len)
     {
         writeFull(fd.get(), data, len);
     }
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 768fe2536..5665732d2 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -57,17 +57,35 @@ struct ParseSink
 
     virtual void createRegularFile(const Path & path) { };
     virtual void isExecutable() { };
-    virtual void preallocateContents(unsigned long long size) { };
-    virtual void receiveContents(unsigned char * data, unsigned int len) { };
+    virtual void preallocateContents(uint64_t size) { };
+    virtual void receiveContents(unsigned char * data, size_t len) { };
 
     virtual void createSymlink(const Path & path, const string & target) { };
 };
 
-struct TeeSink : ParseSink
+/* If the NAR archive contains a single file at top-level, then save
+   the contents of the file to `s'.  Otherwise barf. */
+struct RetrieveRegularNARSink : ParseSink
 {
-    TeeSource source;
+    bool regular = true;
+    Sink & sink;
 
-    TeeSink(Source & source) : source(source) { }
+    RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
+
+    void createDirectory(const Path & path)
+    {
+        regular = false;
+    }
+
+    void receiveContents(unsigned char * data, size_t len)
+    {
+        sink(data, len);
+    }
+
+    void createSymlink(const Path & path, const string & target)
+    {
+        regular = false;
+    }
 };
 
 void parseDump(ParseSink & sink, Source & source);
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 1f0e4f803..986c5d1cd 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -1,6 +1,8 @@
 #include "args.hh"
 #include "hash.hh"
 
+#include <glob.h>
+
 namespace nix {
 
 void Args::addFlag(Flag && flag_)
@@ -13,6 +15,20 @@ void Args::addFlag(Flag && flag_)
     if (flag->shortName) shortFlags[flag->shortName] = flag;
 }
 
+bool pathCompletions = false;
+std::shared_ptr<std::set<std::string>> completions;
+
+std::string completionMarker = "___COMPLETE___";
+
+std::optional<std::string> needsCompletion(std::string_view s)
+{
+    if (!completions) return {};
+    auto i = s.find(completionMarker);
+    if (i != std::string::npos)
+        return std::string(s.begin(), i);
+    return {};
+}
+
 void Args::parseCmdline(const Strings & _cmdline)
 {
     Strings pendingArgs;
@@ -20,6 +36,14 @@ void Args::parseCmdline(const Strings & _cmdline)
 
     Strings cmdline(_cmdline);
 
+    if (auto s = getEnv("NIX_GET_COMPLETIONS")) {
+        size_t n = std::stoi(*s);
+        assert(n > 0 && n <= cmdline.size());
+        *std::next(cmdline.begin(), n - 1) += completionMarker;
+        completions = std::make_shared<decltype(completions)::element_type>();
+        verbosity = lvlError;
+    }
+
     for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
 
         auto arg = *pos;
@@ -63,7 +87,7 @@ void Args::printHelp(const string & programName, std::ostream & out)
     for (auto & exp : expectedArgs) {
         std::cout << renderLabels({exp.label});
         // FIXME: handle arity > 1
-        if (exp.arity == 0) std::cout << "...";
+        if (exp.handler.arity == ArityAny) std::cout << "...";
         if (exp.optional) std::cout << "?";
     }
     std::cout << "\n";
@@ -99,18 +123,32 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
     auto process = [&](const std::string & name, const Flag & flag) -> bool {
         ++pos;
         std::vector<std::string> args;
+        bool anyCompleted = false;
         for (size_t n = 0 ; n < flag.handler.arity; ++n) {
             if (pos == end) {
                 if (flag.handler.arity == ArityAny) break;
                 throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
             }
+            if (flag.completer)
+                if (auto prefix = needsCompletion(*pos)) {
+                    anyCompleted = true;
+                    flag.completer(n, *prefix);
+                }
             args.push_back(*pos++);
         }
-        flag.handler.fun(std::move(args));
+        if (!anyCompleted)
+            flag.handler.fun(std::move(args));
         return true;
     };
 
     if (string(*pos, 0, 2) == "--") {
+        if (auto prefix = needsCompletion(*pos)) {
+            for (auto & [name, flag] : longFlags) {
+                if (!hiddenCategories.count(flag->category)
+                    && hasPrefix(name, std::string(*prefix, 2)))
+                    completions->insert("--" + name);
+            }
+        }
         auto i = longFlags.find(string(*pos, 2));
         if (i == longFlags.end()) return false;
         return process("--" + i->first, *i->second);
@@ -123,6 +161,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
         return process(std::string("-") + c, *i->second);
     }
 
+    if (auto prefix = needsCompletion(*pos)) {
+        if (prefix == "-") {
+            completions->insert("--");
+            for (auto & [flag, _] : shortFlags)
+                completions->insert(std::string("-") + flag);
+        }
+    }
+
     return false;
 }
 
@@ -138,12 +184,17 @@ bool Args::processArgs(const Strings & args, bool finish)
 
     bool res = false;
 
-    if ((exp.arity == 0 && finish) ||
-        (exp.arity > 0 && args.size() == exp.arity))
+    if ((exp.handler.arity == ArityAny && finish) ||
+        (exp.handler.arity != ArityAny && args.size() == exp.handler.arity))
     {
         std::vector<std::string> ss;
-        for (auto & s : args) ss.push_back(s);
-        exp.handler(std::move(ss));
+        for (const auto &[n, s] : enumerate(args)) {
+            ss.push_back(s);
+            if (exp.completer)
+                if (auto prefix = needsCompletion(s))
+                    exp.completer(n, *prefix);
+        }
+        exp.handler.fun(ss);
         expectedArgs.pop_front();
         res = true;
     }
@@ -154,6 +205,13 @@ bool Args::processArgs(const Strings & args, bool finish)
     return res;
 }
 
+static void hashTypeCompleter(size_t index, std::string_view prefix) 
+{
+    for (auto & type : hashTypes)
+        if (hasPrefix(type, prefix))
+            completions->insert(type);
+}
+
 Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
 {
     return Flag {
@@ -162,7 +220,8 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht)
         .labels = {"hash-algo"},
         .handler = {[ht](std::string s) {
             *ht = parseHashType(s);
-        }}
+        }},
+        .completer = hashTypeCompleter
     };
 }
 
@@ -174,10 +233,42 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional<
         .labels = {"hash-algo"},
         .handler = {[oht](std::string s) {
             *oht = std::optional<HashType> { parseHashType(s) };
-        }}
+        }},
+        .completer = hashTypeCompleter
     };
 }
 
+static void completePath(std::string_view prefix, bool onlyDirs)
+{
+    pathCompletions = true;
+    glob_t globbuf;
+    int flags = GLOB_NOESCAPE | GLOB_TILDE;
+    #ifdef GLOB_ONLYDIR
+    if (onlyDirs)
+        flags |= GLOB_ONLYDIR;
+    #endif
+    if (glob((std::string(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) {
+        for (size_t i = 0; i < globbuf.gl_pathc; ++i) {
+            if (onlyDirs) {
+                auto st = lstat(globbuf.gl_pathv[i]);
+                if (!S_ISDIR(st.st_mode)) continue;
+            }
+            completions->insert(globbuf.gl_pathv[i]);
+        }
+        globfree(&globbuf);
+    }
+}
+
+void completePath(size_t, std::string_view prefix)
+{
+    completePath(prefix, false);
+}
+
+void completeDir(size_t, std::string_view prefix)
+{
+    completePath(prefix, true);
+}
+
 Strings argvToStrings(int argc, char * * argv)
 {
     Strings args;
@@ -225,18 +316,26 @@ void Command::printHelp(const string & programName, std::ostream & out)
 MultiCommand::MultiCommand(const Commands & commands)
     : commands(commands)
 {
-    expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector<std::string> ss) {
-        assert(!command);
-        auto cmd = ss[0];
-        if (auto alias = get(deprecatedAliases, cmd)) {
-            warn("'%s' is a deprecated alias for '%s'", cmd, *alias);
-            cmd = *alias;
-        }
-        auto i = commands.find(cmd);
-        if (i == commands.end())
-            throw UsageError("'%s' is not a recognised command", cmd);
-        command = {cmd, i->second()};
-    }});
+    expectArgs({
+        .label = "command",
+        .optional = true,
+        .handler = {[=](std::string s) {
+            assert(!command);
+            if (auto alias = get(deprecatedAliases, s)) {
+                warn("'%s' is a deprecated alias for '%s'", s, *alias);
+                s = *alias;
+            }
+            if (auto prefix = needsCompletion(s)) {
+                for (auto & [name, command] : commands)
+                    if (hasPrefix(name, *prefix))
+                        completions->insert(name);
+            }
+            auto i = commands.find(s);
+            if (i == commands.end())
+                throw UsageError("'%s' is not a recognised command", s);
+            command = {s, i->second()};
+        }}
+    });
 
     categories[Command::catDefault] = "Available commands";
 }
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 433d26bba..97a517344 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -8,8 +8,6 @@
 
 namespace nix {
 
-MakeError(UsageError, Error);
-
 enum HashType : char;
 
 class Args
@@ -28,61 +26,67 @@ protected:
 
     static const size_t ArityAny = std::numeric_limits<size_t>::max();
 
+    struct Handler
+    {
+        std::function<void(std::vector<std::string>)> fun;
+        size_t arity;
+
+        Handler() {}
+
+        Handler(std::function<void(std::vector<std::string>)> && fun)
+            : fun(std::move(fun))
+            , arity(ArityAny)
+        { }
+
+        Handler(std::function<void()> && handler)
+            : fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); })
+            , arity(0)
+        { }
+
+        Handler(std::function<void(std::string)> && handler)
+            : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
+                handler(std::move(ss[0]));
+              })
+            , arity(1)
+        { }
+
+        Handler(std::function<void(std::string, std::string)> && handler)
+            : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
+                handler(std::move(ss[0]), std::move(ss[1]));
+              })
+            , arity(2)
+        { }
+
+        Handler(std::vector<std::string> * dest)
+            : fun([=](std::vector<std::string> ss) { *dest = ss; })
+            , arity(ArityAny)
+        { }
+
+        template<class T>
+        Handler(T * dest)
+            : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
+            , arity(1)
+        { }
+
+        template<class T>
+        Handler(T * dest, const T & val)
+            : fun([=](std::vector<std::string> ss) { *dest = val; })
+            , arity(0)
+        { }
+    };
+
     /* Flags. */
     struct Flag
     {
         typedef std::shared_ptr<Flag> ptr;
 
-        struct Handler
-        {
-            std::function<void(std::vector<std::string>)> fun;
-            size_t arity;
-
-            Handler() {}
-
-            Handler(std::function<void(std::vector<std::string>)> && fun)
-                : fun(std::move(fun))
-                , arity(ArityAny)
-            { }
-
-            Handler(std::function<void()> && handler)
-                : fun([handler{std::move(handler)}](std::vector<std::string>) { handler(); })
-                , arity(0)
-            { }
-
-            Handler(std::function<void(std::string)> && handler)
-                : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
-                    handler(std::move(ss[0]));
-                  })
-                , arity(1)
-            { }
-
-            Handler(std::function<void(std::string, std::string)> && handler)
-                : fun([handler{std::move(handler)}](std::vector<std::string> ss) {
-                    handler(std::move(ss[0]), std::move(ss[1]));
-                  })
-                , arity(2)
-            { }
-
-            template<class T>
-            Handler(T * dest)
-                : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
-                , arity(1)
-            { }
-
-            template<class T>
-            Handler(T * dest, const T & val)
-                : fun([=](std::vector<std::string> ss) { *dest = val; })
-                , arity(0)
-            { }
-        };
-
         std::string longName;
         char shortName = 0;
         std::string description;
         std::string category;
         Strings labels;
         Handler handler;
+        std::function<void(size_t, std::string_view)> completer;
 
         static Flag mkHashTypeFlag(std::string && longName, HashType * ht);
         static Flag mkHashTypeOptFlag(std::string && longName, std::optional<HashType> * oht);
@@ -99,9 +103,9 @@ protected:
     struct ExpectedArg
     {
         std::string label;
-        size_t arity; // 0 = any
-        bool optional;
-        std::function<void(std::vector<std::string>)> handler;
+        bool optional = false;
+        Handler handler;
+        std::function<void(size_t, std::string_view)> completer;
     };
 
     std::list<ExpectedArg> expectedArgs;
@@ -175,20 +179,28 @@ public:
         });
     }
 
+    void expectArgs(ExpectedArg && arg)
+    {
+        expectedArgs.emplace_back(std::move(arg));
+    }
+
     /* Expect a string argument. */
     void expectArg(const std::string & label, string * dest, bool optional = false)
     {
-        expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector<std::string> ss) {
-            *dest = ss[0];
-        }});
+        expectArgs({
+            .label = label,
+            .optional = true,
+            .handler = {dest}
+        });
     }
 
     /* Expect 0 or more arguments. */
     void expectArgs(const std::string & label, std::vector<std::string> * dest)
     {
-        expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector<std::string> ss) {
-            *dest = std::move(ss);
-        }});
+        expectArgs({
+            .label = label,
+            .handler = {dest}
+        });
     }
 
     friend class MultiCommand;
@@ -259,4 +271,13 @@ typedef std::vector<std::pair<std::string, std::string>> Table2;
 
 void printTable(std::ostream & out, const Table2 & table);
 
+extern std::shared_ptr<std::set<std::string>> completions;
+extern bool pathCompletions;
+
+std::optional<std::string> needsCompletion(std::string_view s);
+
+void completePath(size_t, std::string_view prefix);
+
+void completeDir(size_t, std::string_view prefix);
+
 }
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index f4b3f11bb..0daaf3be2 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -191,6 +191,7 @@ public:
     }
 
 MakeError(Error, BaseError);
+MakeError(UsageError, Error);
 
 class SysError : public Error
 {
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 1a3e7c5d8..2b0390da4 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -8,7 +8,6 @@
 #include "hash.hh"
 #include "archive.hh"
 #include "util.hh"
-#include "istringstream_nocopy.hh"
 
 #include <sys/types.h>
 #include <sys/stat.h>
@@ -16,16 +15,22 @@
 
 namespace nix {
 
+static size_t regularHashSize(HashType type) {
+    switch (type) {
+    case htMD5: return md5HashSize;
+    case htSHA1: return sha1HashSize;
+    case htSHA256: return sha256HashSize;
+    case htSHA512: return sha512HashSize;
+    }
+    abort();
+}
+
+std::set<std::string> hashTypes = { "md5", "sha1", "sha256", "sha512" };
+
 
 void Hash::init()
 {
-    assert(type);
-    switch (*type) {
-    case htMD5: hashSize = md5HashSize; break;
-    case htSHA1: hashSize = sha1HashSize; break;
-    case htSHA256: hashSize = sha256HashSize; break;
-    case htSHA512: hashSize = sha512HashSize; break;
-    }
+    hashSize = regularHashSize(type);
     assert(hashSize <= maxHashSize);
     memset(hash, 0, maxHashSize);
 }
@@ -106,17 +111,11 @@ string printHash16or32(const Hash & hash)
 }
 
 
-HashType assertInitHashType(const Hash & h)
-{
-    assert(h.type);
-    return *h.type;
-}
-
 std::string Hash::to_string(Base base, bool includeType) const
 {
     std::string s;
     if (base == SRI || includeType) {
-        s += printHashType(assertInitHashType(*this));
+        s += printHashType(type);
         s += base == SRI ? '-' : ':';
     }
     switch (base) {
@@ -137,60 +136,66 @@ std::string Hash::to_string(Base base, bool includeType) const
 Hash::Hash(std::string_view s, HashType type) : Hash(s, std::optional { type }) { }
 Hash::Hash(std::string_view s) : Hash(s, std::optional<HashType>{}) { }
 
-Hash::Hash(std::string_view s, std::optional<HashType> type)
-    : type(type)
+Hash::Hash(std::string_view original, std::optional<HashType> optType)
 {
+    auto rest = original;
+
     size_t pos = 0;
     bool isSRI = false;
 
-    auto sep = s.find(':');
-    if (sep == string::npos) {
-        sep = s.find('-');
-        if (sep != string::npos) {
-            isSRI = true;
-        } else if (! type)
-            throw BadHash("hash '%s' does not include a type", s);
+    // Parse the has type before the separater, if there was one.
+    std::optional<HashType> optParsedType;
+    {
+        auto sep = rest.find(':');
+        if (sep == std::string_view::npos) {
+            sep = rest.find('-');
+            if (sep != std::string_view::npos)
+                isSRI = true;
+        }
+        if (sep != std::string_view::npos) {
+            auto hashRaw = rest.substr(0, sep);
+            optParsedType = parseHashType(hashRaw);
+            rest = rest.substr(sep + 1);
+        }
     }
 
-    if (sep != string::npos) {
-        string hts = string(s, 0, sep);
-        this->type = parseHashType(hts);
-        if (!this->type)
-            throw BadHash("unknown hash type '%s'", hts);
-        if (type && type != this->type)
-            throw BadHash("hash '%s' should have type '%s'", s, printHashType(*type));
-        pos = sep + 1;
+    // Either the string or user must provide the type, if they both do they
+    // must agree.
+    if (!optParsedType && !optType) {
+        throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest);
+    } else {
+        this->type = optParsedType ? *optParsedType : *optType;
+        if (optParsedType && optType && *optParsedType != *optType)
+            throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
     }
 
     init();
 
-    size_t size = s.size() - pos;
-
-    if (!isSRI && size == base16Len()) {
+    if (!isSRI && rest.size() == base16Len()) {
 
         auto parseHexDigit = [&](char c) {
             if (c >= '0' && c <= '9') return c - '0';
             if (c >= 'A' && c <= 'F') return c - 'A' + 10;
             if (c >= 'a' && c <= 'f') return c - 'a' + 10;
-            throw BadHash("invalid base-16 hash '%s'", s);
+            throw BadHash("invalid base-16 hash '%s'", original);
         };
 
         for (unsigned int i = 0; i < hashSize; i++) {
             hash[i] =
-                parseHexDigit(s[pos + i * 2]) << 4
-                | parseHexDigit(s[pos + i * 2 + 1]);
+                parseHexDigit(rest[pos + i * 2]) << 4
+                | parseHexDigit(rest[pos + i * 2 + 1]);
         }
     }
 
-    else if (!isSRI && size == base32Len()) {
+    else if (!isSRI && rest.size() == base32Len()) {
 
-        for (unsigned int n = 0; n < size; ++n) {
-            char c = s[pos + size - n - 1];
+        for (unsigned int n = 0; n < rest.size(); ++n) {
+            char c = rest[rest.size() - n - 1];
             unsigned char digit;
             for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
                 if (base32Chars[digit] == c) break;
             if (digit >= 32)
-                throw BadHash("invalid base-32 hash '%s'", s);
+                throw BadHash("invalid base-32 hash '%s'", original);
             unsigned int b = n * 5;
             unsigned int i = b / 8;
             unsigned int j = b % 8;
@@ -200,21 +205,21 @@ Hash::Hash(std::string_view s, std::optional<HashType> type)
                 hash[i + 1] |= digit >> (8 - j);
             } else {
                 if (digit >> (8 - j))
-                    throw BadHash("invalid base-32 hash '%s'", s);
+                    throw BadHash("invalid base-32 hash '%s'", original);
             }
         }
     }
 
-    else if (isSRI || size == base64Len()) {
-        auto d = base64Decode(s.substr(pos));
+    else if (isSRI || rest.size() == base64Len()) {
+        auto d = base64Decode(rest);
         if (d.size() != hashSize)
-            throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s);
+            throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", original);
         assert(hashSize);
         memcpy(hash, d.data(), hashSize);
     }
 
     else
-        throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(*type));
+        throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type));
 }
 
 Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht)
@@ -267,7 +272,7 @@ static void finish(HashType ht, Ctx & ctx, unsigned char * hash)
 }
 
 
-Hash hashString(HashType ht, const string & s)
+Hash hashString(HashType ht, std::string_view s)
 {
     Ctx ctx;
     Hash hash(ht);
@@ -334,7 +339,7 @@ HashResult hashPath(
 
 Hash compressHash(const Hash & hash, unsigned int newSize)
 {
-    Hash h;
+    Hash h(hash.type);
     h.hashSize = newSize;
     for (unsigned int i = 0; i < hash.hashSize; ++i)
         h.hash[i % newSize] ^= hash.hash[i];
@@ -342,7 +347,7 @@ Hash compressHash(const Hash & hash, unsigned int newSize)
 }
 
 
-std::optional<HashType> parseHashTypeOpt(const string & s)
+std::optional<HashType> parseHashTypeOpt(std::string_view s)
 {
     if (s == "md5") return htMD5;
     else if (s == "sha1") return htSHA1;
@@ -351,7 +356,7 @@ std::optional<HashType> parseHashTypeOpt(const string & s)
     else return std::optional<HashType> {};
 }
 
-HashType parseHashType(const string & s)
+HashType parseHashType(std::string_view s)
 {
     auto opt_h = parseHashTypeOpt(s);
     if (opt_h)
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 23259dced..abcd58f24 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -18,6 +18,8 @@ const int sha1HashSize = 20;
 const int sha256HashSize = 32;
 const int sha512HashSize = 64;
 
+extern std::set<std::string> hashTypes;
+
 extern const string base32Chars;
 
 enum Base : int { Base64, Base32, Base16, SRI };
@@ -25,14 +27,11 @@ enum Base : int { Base64, Base32, Base16, SRI };
 
 struct Hash
 {
-    static const unsigned int maxHashSize = 64;
-    unsigned int hashSize = 0;
-    unsigned char hash[maxHashSize] = {};
-
-    std::optional<HashType> type = {};
+    constexpr static size_t maxHashSize = 64;
+    size_t hashSize = 0;
+    uint8_t hash[maxHashSize] = {};
 
-    /* Create an unset hash object. */
-    Hash() { };
+    HashType type;
 
     /* Create a zero-filled hash object. */
     Hash(HashType type) : type(type) { init(); };
@@ -105,14 +104,14 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht);
 string printHash16or32(const Hash & hash);
 
 /* Compute the hash of the given string. */
-Hash hashString(HashType ht, const string & s);
+Hash hashString(HashType ht, std::string_view s);
 
 /* Compute the hash of the given file. */
 Hash hashFile(HashType ht, const Path & path);
 
 /* Compute the hash of the given path.  The hash is defined as
    (essentially) hashString(ht, dumpPath(path)). */
-typedef std::pair<Hash, unsigned long long> HashResult;
+typedef std::pair<Hash, uint64_t> HashResult;
 HashResult hashPath(HashType ht, const Path & path,
     PathFilter & filter = defaultPathFilter);
 
@@ -121,9 +120,10 @@ HashResult hashPath(HashType ht, const Path & path,
 Hash compressHash(const Hash & hash, unsigned int newSize);
 
 /* Parse a string representing a hash type. */
-HashType parseHashType(const string & s);
+HashType parseHashType(std::string_view s);
+
 /* Will return nothing on parse error */
-std::optional<HashType> parseHashTypeOpt(const string & s);
+std::optional<HashType> parseHashTypeOpt(std::string_view s);
 
 /* And the reverse. */
 string printHashType(HashType ht);
@@ -141,7 +141,7 @@ class HashSink : public BufferedSink, public AbstractHashSink
 private:
     HashType ht;
     Ctx * ctx;
-    unsigned long long bytes;
+    uint64_t bytes;
 
 public:
     HashSink(HashType ht);
diff --git a/src/libutil/istringstream_nocopy.hh b/src/libutil/istringstream_nocopy.hh
deleted file mode 100644
index f7beac578..000000000
--- a/src/libutil/istringstream_nocopy.hh
+++ /dev/null
@@ -1,92 +0,0 @@
-/* This file provides a variant of std::istringstream that doesn't
-   copy its string argument. This is useful for large strings. The
-   caller must ensure that the string object is not destroyed while
-   it's referenced by this object. */
-
-#pragma once
-
-#include <string>
-#include <iostream>
-
-template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
-class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits>
-{
-public:
-    typedef std::basic_string<CharT, Traits, Allocator> string_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::off_type off_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::pos_type pos_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::int_type int_type;
-
-    typedef typename std::basic_streambuf<CharT, Traits>::traits_type traits_type;
-
-private:
-    const string_type & s;
-
-    off_type off;
-
-public:
-    basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0}
-    {
-    }
-
-private:
-    pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which)
-    {
-        if (which & std::ios_base::in) {
-            this->off = dir == std::ios_base::beg
-                ? off
-                : (dir == std::ios_base::end
-                    ? s.size() + off
-                    : this->off + off);
-        }
-        return pos_type(this->off);
-    }
-
-    pos_type seekpos(pos_type pos, std::ios_base::openmode which)
-    {
-        return seekoff(pos, std::ios_base::beg, which);
-    }
-
-    std::streamsize showmanyc()
-    {
-        return s.size() - off;
-    }
-
-    int_type underflow()
-    {
-        if (typename string_type::size_type(off) == s.size())
-            return traits_type::eof();
-        return traits_type::to_int_type(s[off]);
-    }
-
-    int_type uflow()
-    {
-        if (typename string_type::size_type(off) == s.size())
-            return traits_type::eof();
-        return traits_type::to_int_type(s[off++]);
-    }
-
-    int_type pbackfail(int_type ch)
-    {
-        if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1]))
-            return traits_type::eof();
-
-        return traits_type::to_int_type(s[--off]);
-    }
-
-};
-
-template <class CharT, class Traits = std::char_traits<CharT>, class Allocator = std::allocator<CharT>>
-class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits>
-{
-    typedef basic_istringbuf_nocopy<CharT, Traits, Allocator> buf_type;
-    buf_type buf;
-public:
-    basic_istringstream_nocopy(const typename buf_type::string_type & s) :
-        std::basic_iostream<CharT, Traits>(&buf), buf(s) {};
-};
-
-typedef basic_istringstream_nocopy<char> istringstream_nocopy;
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index c8b71188f..00c945113 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -322,5 +322,18 @@ void StringSink::operator () (const unsigned char * data, size_t len)
     s->append((const char *) data, len);
 }
 
+size_t ChainSource::read(unsigned char * data, size_t len)
+{
+    if (useSecond) {
+        return source2.read(data, len);
+    } else {
+        try {
+            return source1.read(data, len);
+        } catch (EndOfFile &) {
+            useSecond = true;
+            return this->read(data, len);
+        }
+    }
+}
 
 }
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index a04118512..c29c6b29b 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -166,17 +166,30 @@ struct StringSource : Source
 };
 
 
-/* Adapter class of a Source that saves all data read to `s'. */
+/* A sink that writes all incoming data to two other sinks. */
+struct TeeSink : Sink
+{
+    Sink & sink1, & sink2;
+    TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
+    virtual void operator () (const unsigned char * data, size_t len)
+    {
+        sink1(data, len);
+        sink2(data, len);
+    }
+};
+
+
+/* Adapter class of a Source that saves all data read to a sink. */
 struct TeeSource : Source
 {
     Source & orig;
-    ref<std::string> data;
-    TeeSource(Source & orig)
-        : orig(orig), data(make_ref<std::string>()) { }
+    Sink & sink;
+    TeeSource(Source & orig, Sink & sink)
+        : orig(orig), sink(sink) { }
     size_t read(unsigned char * data, size_t len)
     {
         size_t n = orig.read(data, len);
-        this->data->append((const char *) data, n);
+        sink(data, n);
         return n;
     }
 };
@@ -243,6 +256,19 @@ struct LambdaSource : Source
     }
 };
 
+/* Chain two sources together so after the first is exhausted, the second is
+   used */
+struct ChainSource : Source
+{
+    Source & source1, & source2;
+    bool useSecond = false;
+    ChainSource(Source & s1, Source & s2)
+        : source1(s1), source2(s2)
+    { }
+
+    size_t read(unsigned char * data, size_t len) override;
+};
+
 
 /* Convert a function that feeds data into a Sink into a Source. The
    Source executes the function as a coroutine. */
@@ -286,14 +312,14 @@ T readNum(Source & source)
     source(buf, sizeof(buf));
 
     uint64_t n =
-        ((unsigned long long) buf[0]) |
-        ((unsigned long long) buf[1] << 8) |
-        ((unsigned long long) buf[2] << 16) |
-        ((unsigned long long) buf[3] << 24) |
-        ((unsigned long long) buf[4] << 32) |
-        ((unsigned long long) buf[5] << 40) |
-        ((unsigned long long) buf[6] << 48) |
-        ((unsigned long long) buf[7] << 56);
+        ((uint64_t) buf[0]) |
+        ((uint64_t) buf[1] << 8) |
+        ((uint64_t) buf[2] << 16) |
+        ((uint64_t) buf[3] << 24) |
+        ((uint64_t) buf[4] << 32) |
+        ((uint64_t) buf[5] << 40) |
+        ((uint64_t) buf[6] << 48) |
+        ((uint64_t) buf[7] << 56);
 
     if (n > std::numeric_limits<T>::max())
         throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
@@ -336,4 +362,27 @@ Source & operator >> (Source & in, bool & b)
 }
 
 
+/* An adapter that converts a std::basic_istream into a source. */
+struct StreamToSourceAdapter : Source
+{
+    std::shared_ptr<std::basic_istream<char>> istream;
+
+    StreamToSourceAdapter(std::shared_ptr<std::basic_istream<char>> istream)
+        : istream(istream)
+    { }
+
+    size_t read(unsigned char * data, size_t len) override
+    {
+        if (!istream->read((char *) data, len)) {
+            if (istream->eof()) {
+                if (istream->gcount() == 0)
+                    throw EndOfFile("end of file");
+            } else
+                throw Error("I/O error in StreamToSourceAdapter");
+        }
+        return istream->gcount();
+    }
+};
+
+
 }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 1268b146a..97d278581 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -23,6 +23,7 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/wait.h>
+#include <sys/time.h>
 #include <sys/un.h>
 #include <unistd.h>
 
@@ -79,7 +80,7 @@ void replaceEnv(std::map<std::string, std::string> newEnv)
 }
 
 
-Path absPath(Path path, std::optional<Path> dir)
+Path absPath(Path path, std::optional<Path> dir, bool resolveSymlinks)
 {
     if (path[0] != '/') {
         if (!dir) {
@@ -100,7 +101,7 @@ Path absPath(Path path, std::optional<Path> dir)
         }
         path = *dir + "/" + path;
     }
-    return canonPath(path);
+    return canonPath(path, resolveSymlinks);
 }
 
 
@@ -345,7 +346,6 @@ void writeFile(const Path & path, Source & source, mode_t mode)
     }
 }
 
-
 string readLine(int fd)
 {
     string s;
@@ -374,7 +374,7 @@ void writeLine(int fd, string s)
 }
 
 
-static void _deletePath(int parentfd, const Path & path, unsigned long long & bytesFreed)
+static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
 {
     checkInterrupt();
 
@@ -414,7 +414,7 @@ static void _deletePath(int parentfd, const Path & path, unsigned long long & by
     }
 }
 
-static void _deletePath(const Path & path, unsigned long long & bytesFreed)
+static void _deletePath(const Path & path, uint64_t & bytesFreed)
 {
     Path dir = dirOf(path);
     if (dir == "")
@@ -435,12 +435,12 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed)
 
 void deletePath(const Path & path)
 {
-    unsigned long long dummy;
+    uint64_t dummy;
     deletePath(path, dummy);
 }
 
 
-void deletePath(const Path & path, unsigned long long & bytesFreed)
+void deletePath(const Path & path, uint64_t & bytesFreed)
 {
     //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path);
     bytesFreed = 0;
@@ -581,20 +581,31 @@ Paths createDirs(const Path & path)
 }
 
 
-void createSymlink(const Path & target, const Path & link)
+void createSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime)
 {
     if (symlink(target.c_str(), link.c_str()))
         throw SysError("creating symlink from '%1%' to '%2%'", link, target);
+    if (mtime) {
+        struct timeval times[2];
+        times[0].tv_sec = *mtime;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = *mtime;
+        times[1].tv_usec = 0;
+        if (lutimes(link.c_str(), times))
+            throw SysError("setting time of symlink '%s'", link);
+    }
 }
 
 
-void replaceSymlink(const Path & target, const Path & link)
+void replaceSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime)
 {
     for (unsigned int n = 0; true; n++) {
         Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
 
         try {
-            createSymlink(target, tmp);
+            createSymlink(target, tmp, mtime);
         } catch (SysError & e) {
             if (e.errNo == EEXIST) continue;
             throw;
@@ -1006,12 +1017,14 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
     return res;
 }
 
-
+// Output = "standard out" output stream
 string runProgram(Path program, bool searchPath, const Strings & args,
     const std::optional<std::string> & input)
 {
     RunOptions opts(program, args);
     opts.searchPath = searchPath;
+    // This allows you to refer to a program with a pathname relative to the
+    // PATH variable.
     opts.input = input;
 
     auto res = runProgram(opts);
@@ -1022,6 +1035,7 @@ string runProgram(Path program, bool searchPath, const Strings & args,
     return res.second;
 }
 
+// Output = error code + "standard out" output stream
 std::pair<int, std::string> runProgram(const RunOptions & options_)
 {
     RunOptions options(options_);
@@ -1094,6 +1108,8 @@ void runProgram2(const RunOptions & options)
 
         if (options.searchPath)
             execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
+            // This allows you to refer to a program with a pathname relative
+            // to the PATH variable.
         else
             execv(options.program.c_str(), stringsToCharPtrs(args_).data());
 
@@ -1565,7 +1581,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
 
     struct sockaddr_un addr;
     addr.sun_family = AF_UNIX;
-    if (path.size() >= sizeof(addr.sun_path))
+    if (path.size() + 1 >= sizeof(addr.sun_path))
         throw Error("socket path '%1%' is too long", path);
     strcpy(addr.sun_path, path.c_str());
 
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index 3641daaec..6850b5a7a 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -49,7 +49,9 @@ void clearEnv();
 /* Return an absolutized path, resolving paths relative to the
    specified directory, or the current directory otherwise.  The path
    is also canonicalised. */
-Path absPath(Path path, std::optional<Path> dir = {});
+Path absPath(Path path,
+    std::optional<Path> dir = {},
+    bool resolveSymlinks = false);
 
 /* Canonicalise a path by removing all `.' or `..' components and
    double or trailing slashes.  Optionally resolves all symlink
@@ -123,7 +125,7 @@ void writeLine(int fd, string s);
    second variant returns the number of bytes and blocks freed. */
 void deletePath(const Path & path);
 
-void deletePath(const Path & path, unsigned long long & bytesFreed);
+void deletePath(const Path & path, uint64_t & bytesFreed);
 
 std::string getUserName();
 
@@ -147,10 +149,12 @@ Path getDataDir();
 Paths createDirs(const Path & path);
 
 /* Create a symlink. */
-void createSymlink(const Path & target, const Path & link);
+void createSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime = {});
 
 /* Atomically create or replace a symlink. */
-void replaceSymlink(const Path & target, const Path & link);
+void replaceSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime = {});
 
 
 /* Wrappers arount read()/write() that read/write exactly the