Merge branch 'path-info' into ca-drv-exotic

39 files changed, 1624 insertions, 991 deletions

diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index 30b471af5..0e2b9d12c 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -35,10 +35,6 @@ static ArchiveSettings archiveSettings;
 
 static GlobalConfig::Register rArchiveSettings(&archiveSettings);
 
-const std::string narVersionMagic1 = "nix-archive-1";
-
-static std::string caseHackSuffix = "~nix~case~hack~";
-
 PathFilter defaultPathFilter = [](const Path &) { return true; };
 
 
@@ -234,6 +230,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path)
 
         else if (s == "contents" && type == tpRegular) {
             parseContents(sink, source, path);
+            sink.closeRegularFile();
         }
 
         else if (s == "executable" && type == tpRegular) {
@@ -324,6 +321,12 @@ struct RestoreSink : ParseSink
         if (!fd) throw SysError("creating file '%1%'", p);
     }
 
+    void closeRegularFile() override
+    {
+        /* Call close explicitly to make sure the error is checked */
+        fd.close();
+    }
+
     void isExecutable() override
     {
         struct stat st;
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 79ce08df0..e42dea540 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -60,6 +60,7 @@ struct ParseSink
     virtual void createDirectory(const Path & path) { };
 
     virtual void createRegularFile(const Path & path) { };
+    virtual void closeRegularFile() { };
     virtual void isExecutable() { };
     virtual void preallocateContents(uint64_t size) { };
     virtual void receiveContents(std::string_view data) { };
@@ -102,7 +103,9 @@ void copyNAR(Source & source, Sink & sink);
 void copyPath(const Path & from, const Path & to);
 
 
-extern const std::string narVersionMagic1;
+inline constexpr std::string_view narVersionMagic1 = "nix-archive-1";
+
+inline constexpr std::string_view caseHackSuffix = "~nix~case~hack~";
 
 
 }
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 4b8c55686..753980fd4 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -124,7 +124,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
         bool anyCompleted = false;
         for (size_t n = 0 ; n < flag.handler.arity; ++n) {
             if (pos == end) {
-                if (flag.handler.arity == ArityAny) break;
+                if (flag.handler.arity == ArityAny || anyCompleted) break;
                 throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity);
             }
             if (auto prefix = needsCompletion(*pos)) {
@@ -216,7 +216,7 @@ nlohmann::json Args::toJSON()
         if (flag->shortName)
             j["shortName"] = std::string(1, flag->shortName);
         if (flag->description != "")
-            j["description"] = flag->description;
+            j["description"] = trim(flag->description);
         j["category"] = flag->category;
         if (flag->handler.arity != ArityAny)
             j["arity"] = flag->handler.arity;
@@ -237,7 +237,7 @@ nlohmann::json Args::toJSON()
     }
 
     auto res = nlohmann::json::object();
-    res["description"] = description();
+    res["description"] = trim(description());
     res["flags"] = std::move(flags);
     res["args"] = std::move(args);
     auto s = doc();
@@ -362,6 +362,14 @@ bool MultiCommand::processArgs(const Strings & args, bool finish)
         return Args::processArgs(args, finish);
 }
 
+void MultiCommand::completionHook()
+{
+    if (command)
+        return command->second->completionHook();
+    else
+        return Args::completionHook();
+}
+
 nlohmann::json MultiCommand::toJSON()
 {
     auto cmds = nlohmann::json::object();
@@ -371,7 +379,7 @@ nlohmann::json MultiCommand::toJSON()
         auto j = command->toJSON();
         auto cat = nlohmann::json::object();
         cat["id"] = command->category();
-        cat["description"] = categories[command->category()];
+        cat["description"] = trim(categories[command->category()]);
         j["category"] = std::move(cat);
         cmds[name] = std::move(j);
     }
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index fdd036f9a..84866f12b 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -25,6 +25,8 @@ public:
     /* Return a short one-line description of the command. */
     virtual std::string description() { return ""; }
 
+    virtual bool forceImpureByDefault() { return false; }
+
     /* Return documentation about this command, in Markdown format. */
     virtual std::string doc() { return ""; }
 
@@ -146,6 +148,11 @@ protected:
        argument (if any) have been processed. */
     virtual void initialFlagsProcessed() {}
 
+    /* Called after the command line has been processed if we need to generate
+       completions. Useful for commands that need to know the whole command line
+       in order to know what completions to generate. */
+    virtual void completionHook() { }
+
 public:
 
     void addFlag(Flag && flag);
@@ -221,6 +228,8 @@ public:
 
     bool processArgs(const Strings & args, bool finish) override;
 
+    void completionHook() override;
+
     nlohmann::json toJSON() override;
 };
 
diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc
new file mode 100644
index 000000000..b132b4262
--- /dev/null
+++ b/src/libutil/canon-path.cc
@@ -0,0 +1,103 @@
+#include "canon-path.hh"
+#include "util.hh"
+
+namespace nix {
+
+CanonPath CanonPath::root = CanonPath("/");
+
+CanonPath::CanonPath(std::string_view raw)
+    : path(absPath((Path) raw, "/"))
+{ }
+
+CanonPath::CanonPath(std::string_view raw, const CanonPath & root)
+    : path(absPath((Path) raw, root.abs()))
+{ }
+
+std::optional<CanonPath> CanonPath::parent() const
+{
+    if (isRoot()) return std::nullopt;
+    return CanonPath(unchecked_t(), path.substr(0, std::max((size_t) 1, path.rfind('/'))));
+}
+
+void CanonPath::pop()
+{
+    assert(!isRoot());
+    path.resize(std::max((size_t) 1, path.rfind('/')));
+}
+
+bool CanonPath::isWithin(const CanonPath & parent) const
+{
+    return !(
+        path.size() < parent.path.size()
+        || path.substr(0, parent.path.size()) != parent.path
+        || (parent.path.size() > 1 && path.size() > parent.path.size()
+            && path[parent.path.size()] != '/'));
+}
+
+CanonPath CanonPath::removePrefix(const CanonPath & prefix) const
+{
+    assert(isWithin(prefix));
+    if (prefix.isRoot()) return *this;
+    if (path.size() == prefix.path.size()) return root;
+    return CanonPath(unchecked_t(), path.substr(prefix.path.size()));
+}
+
+void CanonPath::extend(const CanonPath & x)
+{
+    if (x.isRoot()) return;
+    if (isRoot())
+        path += x.rel();
+    else
+        path += x.abs();
+}
+
+CanonPath CanonPath::operator + (const CanonPath & x) const
+{
+    auto res = *this;
+    res.extend(x);
+    return res;
+}
+
+void CanonPath::push(std::string_view c)
+{
+    assert(c.find('/') == c.npos);
+    assert(c != "." && c != "..");
+    if (!isRoot()) path += '/';
+    path += c;
+}
+
+CanonPath CanonPath::operator + (std::string_view c) const
+{
+    auto res = *this;
+    res.push(c);
+    return res;
+}
+
+bool CanonPath::isAllowed(const std::set<CanonPath> & allowed) const
+{
+    /* Check if `this` is an exact match or the parent of an
+       allowed path. */
+    auto lb = allowed.lower_bound(*this);
+    if (lb != allowed.end()) {
+        if (lb->isWithin(*this))
+            return true;
+    }
+
+    /* Check if a parent of `this` is allowed. */
+    auto path = *this;
+    while (!path.isRoot()) {
+        path.pop();
+        if (allowed.count(path))
+            return true;
+    }
+
+    return false;
+}
+
+std::ostream & operator << (std::ostream & stream, const CanonPath & path)
+{
+    stream << path.abs();
+    return stream;
+}
+
+}
diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh
new file mode 100644
index 000000000..9d5984584
--- /dev/null
+++ b/src/libutil/canon-path.hh
@@ -0,0 +1,173 @@
+#pragma once
+
+#include <string>
+#include <optional>
+#include <cassert>
+#include <iostream>
+#include <set>
+
+namespace nix {
+
+/* A canonical representation of a path. It ensures the following:
+
+   - It always starts with a slash.
+
+   - It never ends with a slash, except if the path is "/".
+
+   - A slash is never followed by a slash (i.e. no empty components).
+
+   - There are no components equal to '.' or '..'.
+
+   Note that the path does not need to correspond to an actually
+   existing path, and there is no guarantee that symlinks are
+   resolved.
+*/
+class CanonPath
+{
+    std::string path;
+
+public:
+
+    /* Construct a canon path from a non-canonical path. Any '.', '..'
+       or empty components are removed. */
+    CanonPath(std::string_view raw);
+
+    explicit CanonPath(const char * raw)
+        : CanonPath(std::string_view(raw))
+    { }
+
+    struct unchecked_t { };
+
+    CanonPath(unchecked_t _, std::string path)
+        : path(std::move(path))
+    { }
+
+    static CanonPath root;
+
+    /* If `raw` starts with a slash, return
+       `CanonPath(raw)`. Otherwise return a `CanonPath` representing
+       `root + "/" + raw`. */
+    CanonPath(std::string_view raw, const CanonPath & root);
+
+    bool isRoot() const
+    { return path.size() <= 1; }
+
+    explicit operator std::string_view() const
+    { return path; }
+
+    const std::string & abs() const
+    { return path; }
+
+    /* Like abs(), but return an empty string if this path is
+       '/'. Thus the returned string never ends in a slash. */
+    const std::string & absOrEmpty() const
+    {
+        const static std::string epsilon;
+        return isRoot() ? epsilon : path;
+    }
+
+    const char * c_str() const
+    { return path.c_str(); }
+
+    std::string_view rel() const
+    { return ((std::string_view) path).substr(1); }
+
+    struct Iterator
+    {
+        std::string_view remaining;
+        size_t slash;
+
+        Iterator(std::string_view remaining)
+            : remaining(remaining)
+            , slash(remaining.find('/'))
+        { }
+
+        bool operator != (const Iterator & x) const
+        { return remaining.data() != x.remaining.data(); }
+
+        const std::string_view operator * () const
+        { return remaining.substr(0, slash); }
+
+        void operator ++ ()
+        {
+            if (slash == remaining.npos)
+                remaining = remaining.substr(remaining.size());
+            else {
+                remaining = remaining.substr(slash + 1);
+                slash = remaining.find('/');
+            }
+        }
+    };
+
+    Iterator begin() const { return Iterator(rel()); }
+    Iterator end() const { return Iterator(rel().substr(path.size() - 1)); }
+
+    std::optional<CanonPath> parent() const;
+
+    /* Remove the last component. Panics if this path is the root.  */
+    void pop();
+
+    std::optional<std::string_view> dirOf() const
+    {
+        if (isRoot()) return std::nullopt;
+        return ((std::string_view) path).substr(0, path.rfind('/'));
+    }
+
+    std::optional<std::string_view> baseName() const
+    {
+        if (isRoot()) return std::nullopt;
+        return ((std::string_view) path).substr(path.rfind('/') + 1);
+    }
+
+    bool operator == (const CanonPath & x) const
+    { return path == x.path; }
+
+    bool operator != (const CanonPath & x) const
+    { return path != x.path; }
+
+    /* Compare paths lexicographically except that path separators
+       are sorted before any other character. That is, in the sorted order
+       a directory is always followed directly by its children. For
+       instance, 'foo' < 'foo/bar' < 'foo!'. */
+    bool operator < (const CanonPath & x) const
+    {
+        auto i = path.begin();
+        auto j = x.path.begin();
+        for ( ; i != path.end() && j != x.path.end(); ++i, ++j) {
+            auto c_i = *i;
+            if (c_i == '/') c_i = 0;
+            auto c_j = *j;
+            if (c_j == '/') c_j = 0;
+            if (c_i < c_j) return true;
+            if (c_i > c_j) return false;
+        }
+        return i == path.end() && j != x.path.end();
+    }
+
+    /* Return true if `this` is equal to `parent` or a child of
+       `parent`. */
+    bool isWithin(const CanonPath & parent) const;
+
+    CanonPath removePrefix(const CanonPath & prefix) const;
+
+    /* Append another path to this one. */
+    void extend(const CanonPath & x);
+
+    /* Concatenate two paths. */
+    CanonPath operator + (const CanonPath & x) const;
+
+    /* Add a path component to this one. It must not contain any slashes. */
+    void push(std::string_view c);
+
+    CanonPath operator + (std::string_view c) const;
+
+    /* Check whether access to this path is allowed, which is the case
+       if 1) `this` is within any of the `allowed` paths; or 2) any of
+       the `allowed` paths are within `this`. (The latter condition
+       ensures access to the parents of allowed paths.) */
+    bool isAllowed(const std::set<CanonPath> & allowed) const;
+};
+
+std::ostream & operator << (std::ostream & stream, const CanonPath & path);
+
+}
diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc
new file mode 100644
index 000000000..a008481ca
--- /dev/null
+++ b/src/libutil/cgroup.cc
@@ -0,0 +1,148 @@
+#if __linux__
+
+#include "cgroup.hh"
+#include "util.hh"
+#include "finally.hh"
+
+#include <chrono>
+#include <cmath>
+#include <regex>
+#include <unordered_set>
+#include <thread>
+
+#include <dirent.h>
+#include <mntent.h>
+
+namespace nix {
+
+std::optional<Path> getCgroupFS()
+{
+    static auto res = [&]() -> std::optional<Path> {
+        auto fp = fopen("/proc/mounts", "r");
+        if (!fp) return std::nullopt;
+        Finally delFP = [&]() { fclose(fp); };
+        while (auto ent = getmntent(fp))
+            if (std::string_view(ent->mnt_type) == "cgroup2")
+                return ent->mnt_dir;
+
+        return std::nullopt;
+    }();
+    return res;
+}
+
+// FIXME: obsolete, check for cgroup2
+std::map<std::string, std::string> getCgroups(const Path & cgroupFile)
+{
+    std::map<std::string, std::string> cgroups;
+
+    for (auto & line : tokenizeString<std::vector<std::string>>(readFile(cgroupFile), "\n")) {
+        static std::regex regex("([0-9]+):([^:]*):(.*)");
+        std::smatch match;
+        if (!std::regex_match(line, match, regex))
+            throw Error("invalid line '%s' in '%s'", line, cgroupFile);
+
+        std::string name = hasPrefix(std::string(match[2]), "name=") ? std::string(match[2], 5) : match[2];
+        cgroups.insert_or_assign(name, match[3]);
+    }
+
+    return cgroups;
+}
+
+static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats)
+{
+    if (!pathExists(cgroup)) return {};
+
+    auto procsFile = cgroup + "/cgroup.procs";
+
+    if (!pathExists(procsFile))
+        throw Error("'%s' is not a cgroup", cgroup);
+
+    /* Use the fast way to kill every process in a cgroup, if
+       available. */
+    auto killFile = cgroup + "/cgroup.kill";
+    if (pathExists(killFile))
+        writeFile(killFile, "1");
+
+    /* Otherwise, manually kill every process in the subcgroups and
+       this cgroup. */
+    for (auto & entry : readDirectory(cgroup)) {
+        if (entry.type != DT_DIR) continue;
+        destroyCgroup(cgroup + "/" + entry.name, false);
+    }
+
+    int round = 1;
+
+    std::unordered_set<pid_t> pidsShown;
+
+    while (true) {
+        auto pids = tokenizeString<std::vector<std::string>>(readFile(procsFile));
+
+        if (pids.empty()) break;
+
+        if (round > 20)
+            throw Error("cannot kill cgroup '%s'", cgroup);
+
+        for (auto & pid_s : pids) {
+            pid_t pid;
+            if (auto o = string2Int<pid_t>(pid_s))
+                pid = *o;
+            else
+                throw Error("invalid pid '%s'", pid);
+            if (pidsShown.insert(pid).second) {
+                try {
+                    auto cmdline = readFile(fmt("/proc/%d/cmdline", pid));
+                    using namespace std::string_literals;
+                    warn("killing stray builder process %d (%s)...",
+                        pid, trim(replaceStrings(cmdline, "\0"s, " ")));
+                } catch (SysError &) {
+                }
+            }
+            // FIXME: pid wraparound
+            if (kill(pid, SIGKILL) == -1 && errno != ESRCH)
+                throw SysError("killing member %d of cgroup '%s'", pid, cgroup);
+        }
+
+        auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10)));
+        if (sleep.count() > 100)
+            printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup);
+        std::this_thread::sleep_for(sleep);
+        round++;
+    }
+
+    CgroupStats stats;
+
+    if (returnStats) {
+        auto cpustatPath = cgroup + "/cpu.stat";
+
+        if (pathExists(cpustatPath)) {
+            for (auto & line : tokenizeString<std::vector<std::string>>(readFile(cpustatPath), "\n")) {
+                std::string_view userPrefix = "user_usec ";
+                if (hasPrefix(line, userPrefix)) {
+                    auto n = string2Int<uint64_t>(line.substr(userPrefix.size()));
+                    if (n) stats.cpuUser = std::chrono::microseconds(*n);
+                }
+
+                std::string_view systemPrefix = "system_usec ";
+                if (hasPrefix(line, systemPrefix)) {
+                    auto n = string2Int<uint64_t>(line.substr(systemPrefix.size()));
+                    if (n) stats.cpuSystem = std::chrono::microseconds(*n);
+                }
+            }
+        }
+
+    }
+
+    if (rmdir(cgroup.c_str()) == -1)
+        throw SysError("deleting cgroup '%s'", cgroup);
+
+    return stats;
+}
+
+CgroupStats destroyCgroup(const Path & cgroup)
+{
+    return destroyCgroup(cgroup, true);
+}
+
+}
+
+#endif
diff --git a/src/libutil/cgroup.hh b/src/libutil/cgroup.hh
new file mode 100644
index 000000000..d08c8ad29
--- /dev/null
+++ b/src/libutil/cgroup.hh
@@ -0,0 +1,29 @@
+#pragma once
+
+#if __linux__
+
+#include <chrono>
+#include <optional>
+
+#include "types.hh"
+
+namespace nix {
+
+std::optional<Path> getCgroupFS();
+
+std::map<std::string, std::string> getCgroups(const Path & cgroupFile);
+
+struct CgroupStats
+{
+    std::optional<std::chrono::microseconds> cpuUser, cpuSystem;
+};
+
+/* Destroy the cgroup denoted by 'path'. The postcondition is that
+   'path' does not exist, and thus any processes in the cgroup have
+   been killed. Also return statistics from the cgroup just before
+   destruction. */
+CgroupStats destroyCgroup(const Path & cgroup);
+
+}
+
+#endif
diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh
new file mode 100644
index 000000000..0a4f0b400
--- /dev/null
+++ b/src/libutil/chunked-vector.hh
@@ -0,0 +1,68 @@
+#pragma once
+
+#include <cstdint>
+#include <cstdlib>
+#include <vector>
+#include <limits>
+
+namespace nix {
+
+/* Provides an indexable container like vector<> with memory overhead
+   guarantees like list<> by allocating storage in chunks of ChunkSize
+   elements instead of using a contiguous memory allocation like vector<>
+   does. Not using a single vector that is resized reduces memory overhead
+   on large data sets by on average (growth factor)/2, mostly
+   eliminates copies within the vector during resizing, and provides stable
+   references to its elements. */
+template<typename T, size_t ChunkSize>
+class ChunkedVector {
+private:
+    uint32_t size_ = 0;
+    std::vector<std::vector<T>> chunks;
+
+    /* keep this out of the ::add hot path */
+    [[gnu::noinline]]
+    auto & addChunk()
+    {
+        if (size_ >= std::numeric_limits<uint32_t>::max() - ChunkSize)
+            abort();
+        chunks.emplace_back();
+        chunks.back().reserve(ChunkSize);
+        return chunks.back();
+    }
+
+public:
+    ChunkedVector(uint32_t reserve)
+    {
+        chunks.reserve(reserve);
+        addChunk();
+    }
+
+    uint32_t size() const { return size_; }
+
+    std::pair<T &, uint32_t> add(T value)
+    {
+        const auto idx = size_++;
+        auto & chunk = [&] () -> auto & {
+            if (auto & back = chunks.back(); back.size() < ChunkSize)
+                return back;
+            return addChunk();
+        }();
+        auto & result = chunk.emplace_back(std::move(value));
+        return {result, idx};
+    }
+
+    const T & operator[](uint32_t idx) const
+    {
+        return chunks[idx / ChunkSize][idx % ChunkSize];
+    }
+
+    template<typename Fn>
+    void forEach(Fn fn) const
+    {
+        for (const auto & c : chunks)
+            for (const auto & e : c)
+                fn(e);
+    }
+};
+}
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 9172f67a6..e4f0d4677 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -9,9 +9,9 @@ namespace nix {
 
 const std::string nativeSystem = SYSTEM;
 
-void BaseError::addTrace(std::optional<ErrPos> e, hintformat hint)
+void BaseError::addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame)
 {
-    err.traces.push_front(Trace { .pos = e, .hint = hint });
+    err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame });
 }
 
 // c++ std::exception descendants must have a 'const char* what()' function.
@@ -30,91 +30,46 @@ const std::string & BaseError::calcWhat() const
 
 std::optional<std::string> ErrorInfo::programName = std::nullopt;
 
-std::ostream & operator<<(std::ostream & os, const hintformat & hf)
+std::ostream & operator <<(std::ostream & os, const hintformat & hf)
 {
     return os << hf.str();
 }
 
-std::string showErrPos(const ErrPos & errPos)
+std::ostream & operator <<(std::ostream & str, const AbstractPos & pos)
 {
-    if (errPos.line > 0) {
-        if (errPos.column > 0) {
-            return fmt("%d:%d", errPos.line, errPos.column);
-        } else {
-            return fmt("%d", errPos.line);
-        }
-    }
-    else {
-        return "";
-    }
+    pos.print(str);
+    str << ":" << pos.line;
+    if (pos.column > 0)
+        str << ":" << pos.column;
+    return str;
 }
 
-std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
+std::optional<LinesOfCode> AbstractPos::getCodeLines() const
 {
-    if (errPos.line <= 0)
+    if (line == 0)
         return std::nullopt;
 
-    if (errPos.origin == foFile) {
-        LinesOfCode loc;
-        try {
-            // FIXME: when running as the daemon, make sure we don't
-            // open a file to which the client doesn't have access.
-            AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
-            if (!fd) return {};
-
-            // count the newlines.
-            int count = 0;
-            std::string line;
-            int pl = errPos.line - 1;
-            do
-            {
-                line = readLine(fd.get());
-                ++count;
-                if (count < pl)
-                    ;
-                else if (count == pl)
-                    loc.prevLineOfCode = line;
-                else if (count == pl + 1)
-                    loc.errLineOfCode = line;
-                else if (count == pl + 2) {
-                    loc.nextLineOfCode = line;
-                    break;
-                }
-            } while (true);
-            return loc;
-        }
-        catch (EndOfFile & eof) {
-            if (loc.errLineOfCode.has_value())
-                return loc;
-            else
-                return std::nullopt;
-        }
-        catch (std::exception & e) {
-            return std::nullopt;
-        }
-    } else {
-        std::istringstream iss(errPos.file);
+    if (auto source = getSource()) {
+
+        std::istringstream iss(*source);
         // count the newlines.
         int count = 0;
-        std::string line;
-        int pl = errPos.line - 1;
+        std::string curLine;
+        int pl = line - 1;
 
         LinesOfCode loc;
 
-        do
-        {
-            std::getline(iss, line);
+        do {
+            std::getline(iss, curLine);
             ++count;
             if (count < pl)
-            {
                 ;
-            }
             else if (count == pl) {
-                loc.prevLineOfCode = line;
+                loc.prevLineOfCode = curLine;
             } else if (count == pl + 1) {
-                loc.errLineOfCode = line;
+                loc.errLineOfCode = curLine;
             } else if (count == pl + 2) {
-                loc.nextLineOfCode = line;
+                loc.nextLineOfCode = curLine;
                 break;
             }
 
@@ -124,12 +79,14 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
 
         return loc;
     }
+
+    return std::nullopt;
 }
 
 // print lines of code to the ostream, indicating the error column.
 void printCodeLines(std::ostream & out,
     const std::string & prefix,
-    const ErrPos & errPos,
+    const AbstractPos & errPos,
     const LinesOfCode & loc)
 {
     // previous line of code.
@@ -176,28 +133,6 @@ void printCodeLines(std::ostream & out,
     }
 }
 
-void printAtPos(const ErrPos & pos, std::ostream & out)
-{
-    if (pos) {
-        switch (pos.origin) {
-            case foFile: {
-                out << fmt(ANSI_BLUE "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos));
-                break;
-            }
-            case foString: {
-                out << fmt(ANSI_BLUE "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos));
-                break;
-            }
-            case foStdin: {
-                out << fmt(ANSI_BLUE "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos));
-                break;
-            }
-            default:
-                throw Error("invalid FileOrigin in errPos");
-        }
-    }
-}
-
 static std::string indent(std::string_view indentFirst, std::string_view indentRest, std::string_view s)
 {
     std::string res;
@@ -262,49 +197,160 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
         prefix += ":" ANSI_NORMAL " ";
 
     std::ostringstream oss;
-    oss << einfo.msg << "\n";
 
-    if (einfo.errPos.has_value() && *einfo.errPos) {
-        oss << "\n";
-        printAtPos(*einfo.errPos, oss);
+    auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n";
+
+    /*
+     * Traces
+     * ------
+     *
+     *  The semantics of traces is a bit weird. We have only one option to
+     *  print them and to make them verbose (--show-trace). In the code they
+     *  are always collected, but they are not printed by default. The code
+     *  also collects more traces when the option is on. This means that there
+     *  is no way to print the simplified traces at all.
+     *
+     *  I (layus) designed the code to attach positions to a restricted set of
+     *  messages. This means that we have  a lot of traces with no position at
+     *  all, including most of the base error messages. For example "type
+     *  error: found a string while a set was expected" has no position, but
+     *  will come with several traces detailing it's precise relation to the
+     *  closest know position. This makes erroring without printing traces
+     *  quite useless.
+     *
+     *  This is why I introduced the idea to always print a few traces on
+     *  error. The number 3 is quite arbitrary, and was selected so as not to
+     *  clutter the console on error. For the same reason, a trace with an
+     *  error position takes more space, and counts as two traces towards the
+     *  limit.
+     *
+     *  The rest is truncated, unless --show-trace is passed. This preserves
+     *  the same bad semantics of --show-trace to both show the trace and
+     *  augment it with new data. Not too sure what is the best course of
+     *  action.
+     *
+     *  The issue is that it is fundamentally hard to provide a trace for a
+     *  lazy language. The trace will only cover the current spine of the
+     *  evaluation, missing things that have been evaluated before. For
+     *  example, most type errors are hard to inspect because there is not
+     *  trace for the faulty value. These errors should really print the faulty
+     *  value itself.
+     *
+     *  In function calls, the --show-trace flag triggers extra traces for each
+     *  function invocation. These work as scopes, allowing to follow the
+     *  current spine of the evaluation graph. Without that flag, the error
+     *  trace should restrict itself to a restricted prefix of that trace,
+     *  until the first scope. If we ever get to such a precise error
+     *  reporting, there would be no need to add an arbitrary limit here. We
+     *  could always print the full trace, and it would just be small without
+     *  the flag.
+     *
+     *  One idea I had is for XxxError.addTrace() to perform nothing if one
+     *  scope has already been traced. Alternatively, we could stop here when
+     *  we encounter such a scope instead of after an arbitrary number of
+     *  traces. This however requires to augment traces with the notion of
+     *  "scope".
+     *
+     *  This is particularly visible in code like evalAttrs(...) where we have
+     *  to make a decision between the two following options.
+     *
+     *  ``` long traces
+     *  inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, std::string_view errorCtx)
+     *  {
+     *      try {
+     *          e->eval(*this, env, v);
+     *          if (v.type() != nAttrs)
+     *              throwTypeError("value is %1% while a set was expected", v);
+     *      } catch (Error & e) {
+     *          e.addTrace(pos, errorCtx);
+     *          throw;
+     *      }
+     *  }
+     *  ```
+     *
+     *  ``` short traces
+     *  inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const Pos & pos, std::string_view errorCtx)
+     *  {
+     *      e->eval(*this, env, v);
+     *      try {
+     *          if (v.type() != nAttrs)
+     *              throwTypeError("value is %1% while a set was expected", v);
+     *      } catch (Error & e) {
+     *          e.addTrace(pos, errorCtx);
+     *          throw;
+     *      }
+     *  }
+     *  ```
+     *
+     *  The second example can be rewritten more concisely, but kept in this
+     *  form to highlight the symmetry. The first option adds more information,
+     *  because whatever caused an error down the line, in the generic eval
+     *  function, will get annotated with the code location that uses and
+     *  required it. The second option is less verbose, but does not provide
+     *  any context at all as to where and why a failing value was required.
+     *
+     *  Scopes would fix that, by adding context only when --show-trace is
+     *  passed, and keeping the trace terse otherwise.
+     *
+     */
+
+    // Enough indent to align with with the `... `
+    // prepended to each element of the trace
+    auto ellipsisIndent = "  ";
+
+    bool frameOnly = false;
+    if (!einfo.traces.empty()) {
+        size_t count = 0;
+        for (const auto & trace : einfo.traces) {
+            if (!showTrace && count > 3) {
+                oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n";
+                break;
+            }
+
+            if (trace.hint.str().empty()) continue;
+            if (frameOnly && !trace.frame) continue;
+
+            count++;
+            frameOnly = trace.frame;
+
+            oss << "\n" << "… " << trace.hint.str() << "\n";
+
+            if (trace.pos) {
+                count++;
+
+                oss << "\n" << ellipsisIndent << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":";
+
+                if (auto loc = trace.pos->getCodeLines()) {
+                    oss << "\n";
+                    printCodeLines(oss, "", *trace.pos, *loc);
+                    oss << "\n";
+                } else
+                    oss << noSource;
+            }
+        }
+        oss << "\n" << prefix;
+    }
+
+    oss << einfo.msg << "\n";
 
-        auto loc = getCodeLines(*einfo.errPos);
+    if (einfo.errPos) {
+        oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":";
 
-        // lines of code.
-        if (loc.has_value()) {
+        if (auto loc = einfo.errPos->getCodeLines()) {
             oss << "\n";
             printCodeLines(oss, "", *einfo.errPos, *loc);
             oss << "\n";
-        }
+        } else
+            oss << noSource;
     }
 
     auto suggestions = einfo.suggestions.trim();
-    if (! suggestions.suggestions.empty()){
+    if (!suggestions.suggestions.empty()) {
         oss << "Did you mean " <<
             suggestions.trim() <<
             "?" << std::endl;
     }
 
-    // traces
-    if (showTrace && !einfo.traces.empty()) {
-        for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) {
-            oss << "\n" << "… " << iter->hint.str() << "\n";
-
-            if (iter->pos.has_value() && (*iter->pos)) {
-                auto pos = iter->pos.value();
-                oss << "\n";
-                printAtPos(pos, oss);
-
-                auto loc = getCodeLines(pos);
-                if (loc.has_value()) {
-                    oss << "\n";
-                    printCodeLines(oss, "", pos, *loc);
-                    oss << "\n";
-                }
-            }
-        }
-    }
-
     out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str()));
 
     return out;
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index 7a5316896..7d236028c 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -54,13 +54,6 @@ typedef enum {
     lvlVomit
 } Verbosity;
 
-/* adjust Pos::origin bit width when adding stuff here */
-typedef enum {
-    foFile,
-    foStdin,
-    foString
-} FileOrigin;
-
 // the lines of code surrounding an error.
 struct LinesOfCode {
     std::optional<std::string> prevLineOfCode;
@@ -68,49 +61,38 @@ struct LinesOfCode {
     std::optional<std::string> nextLineOfCode;
 };
 
-// ErrPos indicates the location of an error in a nix file.
-struct ErrPos {
-    int line = 0;
-    int column = 0;
-    std::string file;
-    FileOrigin origin;
+/* An abstract type that represents a location in a source file. */
+struct AbstractPos
+{
+    uint32_t line = 0;
+    uint32_t column = 0;
 
-    operator bool() const
-    {
-        return line != 0;
-    }
+    /* Return the contents of the source file. */
+    virtual std::optional<std::string> getSource() const
+    { return std::nullopt; };
 
-    // convert from the Pos struct, found in libexpr.
-    template <class P>
-    ErrPos & operator=(const P & pos)
-    {
-        origin = pos.origin;
-        line = pos.line;
-        column = pos.column;
-        // is file symbol null?
-        if (pos.file.set())
-            file = pos.file;
-        else
-            file = "";
-        return *this;
-    }
+    virtual void print(std::ostream & out) const = 0;
 
-    template <class P>
-    ErrPos(const P & p)
-    {
-        *this = p;
-    }
+    std::optional<LinesOfCode> getCodeLines() const;
 };
 
+std::ostream & operator << (std::ostream & str, const AbstractPos & pos);
+
+void printCodeLines(std::ostream & out,
+    const std::string & prefix,
+    const AbstractPos & errPos,
+    const LinesOfCode & loc);
+
 struct Trace {
-    std::optional<ErrPos> pos;
+    std::shared_ptr<AbstractPos> pos;
     hintformat hint;
+    bool frame;
 };
 
 struct ErrorInfo {
     Verbosity level;
     hintformat msg;
-    std::optional<ErrPos> errPos;
+    std::shared_ptr<AbstractPos> errPos;
     std::list<Trace> traces;
 
     Suggestions suggestions;
@@ -133,6 +115,8 @@ protected:
 public:
     unsigned int status = 1; // exit status
 
+    BaseError(const BaseError &) = default;
+
     template<typename... Args>
     BaseError(unsigned int status, const Args & ... args)
         : err { .level = lvlError, .msg = hintfmt(args...) }
@@ -171,15 +155,22 @@ public:
     const std::string & msg() const { return calcWhat(); }
     const ErrorInfo & info() const { calcWhat(); return err; }
 
+    void pushTrace(Trace trace)
+    {
+        err.traces.push_front(trace);
+    }
+
     template<typename... Args>
-    void addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args)
+    void addTrace(std::shared_ptr<AbstractPos> && e, std::string_view fs, const Args & ... args)
     {
-        addTrace(e, hintfmt(fs, args...));
+        addTrace(std::move(e), hintfmt(std::string(fs), args...));
     }
 
-    void addTrace(std::optional<ErrPos> e, hintformat hint);
+    void addTrace(std::shared_ptr<AbstractPos> && e, hintformat hint, bool frame = false);
 
     bool hasTrace() const { return !err.traces.empty(); }
+
+    const ErrorInfo & info() { return err; };
 };
 
 #define MakeError(newClass, superClass) \
@@ -199,12 +190,19 @@ public:
     int errNo;
 
     template<typename... Args>
-    SysError(const Args & ... args)
-        : Error(""), errNo(errno)
+    SysError(int errNo_, const Args & ... args)
+        : Error("")
     {
+        errNo = errNo_;
         auto hf = hintfmt(args...);
         err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
     }
+
+    template<typename... Args>
+    SysError(const Args & ... args)
+        : SysError(errno, args ...)
+    {
+    }
 };
 
 }
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index e033a4116..e0902971e 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -13,6 +13,9 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
     { Xp::RecursiveNix, "recursive-nix" },
     { Xp::NoUrlLiterals, "no-url-literals" },
     { Xp::FetchClosure, "fetch-closure" },
+    { Xp::ReplFlake, "repl-flake" },
+    { Xp::AutoAllocateUids, "auto-allocate-uids" },
+    { Xp::Cgroups, "cgroups" },
 };
 
 const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
@@ -35,7 +38,9 @@ const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::str
 
 std::string_view showExperimentalFeature(const ExperimentalFeature feature)
 {
-    return stringifiedXpFeatures.at(feature);
+    const auto ret = get(stringifiedXpFeatures, feature);
+    assert(ret);
+    return *ret;
 }
 
 std::set<ExperimentalFeature> parseFeatures(const std::set<std::string> & rawFeatures)
@@ -58,4 +63,20 @@ std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & featu
     return str << showExperimentalFeature(feature);
 }
 
+void to_json(nlohmann::json & j, const ExperimentalFeature & feature)
+{
+    j = showExperimentalFeature(feature);
+}
+
+void from_json(const nlohmann::json & j, ExperimentalFeature & feature)
+{
+    const std::string input = j;
+    const auto parsed = parseExperimentalFeature(input);
+
+    if (parsed.has_value())
+        feature = *parsed;
+    else
+        throw Error("Unknown experimental feature '%s' in JSON input", input);
+}
+
 }
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index 266e41a22..af775feb0 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -22,6 +22,9 @@ enum struct ExperimentalFeature
     RecursiveNix,
     NoUrlLiterals,
     FetchClosure,
+    ReplFlake,
+    AutoAllocateUids,
+    Cgroups,
 };
 
 /**
@@ -51,4 +54,11 @@ public:
     MissingExperimentalFeature(ExperimentalFeature);
 };
 
+/**
+ * Semi-magic conversion to and from json.
+ * See the nlohmann/json readme for more details.
+ */
+void to_json(nlohmann::json &, const ExperimentalFeature &);
+void from_json(const nlohmann::json &, ExperimentalFeature &);
+
 }
diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc
new file mode 100644
index 000000000..3a732cff8
--- /dev/null
+++ b/src/libutil/filesystem.cc
@@ -0,0 +1,173 @@
+#include <sys/time.h>
+#include <filesystem>
+#include <atomic>
+
+#include "finally.hh"
+#include "util.hh"
+#include "types.hh"
+
+namespace fs = std::filesystem;
+
+namespace nix {
+
+static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
+    std::atomic<unsigned int> & counter)
+{
+    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
+    if (includePid)
+        return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str();
+    else
+        return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str();
+}
+
+Path createTempDir(const Path & tmpRoot, const Path & prefix,
+    bool includePid, bool useGlobalCounter, mode_t mode)
+{
+    static std::atomic<unsigned int> globalCounter = 0;
+    std::atomic<unsigned int> localCounter = 0;
+    auto & counter(useGlobalCounter ? globalCounter : localCounter);
+
+    while (1) {
+        checkInterrupt();
+        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
+        if (mkdir(tmpDir.c_str(), mode) == 0) {
+#if __FreeBSD__
+            /* Explicitly set the group of the directory.  This is to
+               work around around problems caused by BSD's group
+               ownership semantics (directories inherit the group of
+               the parent).  For instance, the group of /tmp on
+               FreeBSD is "wheel", so all directories created in /tmp
+               will be owned by "wheel"; but if the user is not in
+               "wheel", then "tar" will fail to unpack archives that
+               have the setgid bit set on directories. */
+            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
+                throw SysError("setting group of directory '%1%'", tmpDir);
+#endif
+            return tmpDir;
+        }
+        if (errno != EEXIST)
+            throw SysError("creating directory '%1%'", tmpDir);
+    }
+}
+
+
+std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
+{
+    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
+    // Strictly speaking, this is UB, but who cares...
+    // FIXME: use O_TMPFILE.
+    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
+    if (!fd)
+        throw SysError("creating temporary file '%s'", tmpl);
+    closeOnExec(fd.get());
+    return {std::move(fd), tmpl};
+}
+
+void createSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime)
+{
+    if (symlink(target.c_str(), link.c_str()))
+        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
+    if (mtime) {
+        struct timeval times[2];
+        times[0].tv_sec = *mtime;
+        times[0].tv_usec = 0;
+        times[1].tv_sec = *mtime;
+        times[1].tv_usec = 0;
+        if (lutimes(link.c_str(), times))
+            throw SysError("setting time of symlink '%s'", link);
+    }
+}
+
+void replaceSymlink(const Path & target, const Path & link,
+    std::optional<time_t> mtime)
+{
+    for (unsigned int n = 0; true; n++) {
+        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
+
+        try {
+            createSymlink(target, tmp, mtime);
+        } catch (SysError & e) {
+            if (e.errNo == EEXIST) continue;
+            throw;
+        }
+
+        renameFile(tmp, link);
+
+        break;
+    }
+}
+
+void setWriteTime(const fs::path & p, const struct stat & st)
+{
+    struct timeval times[2];
+    times[0] = {
+        .tv_sec = st.st_atime,
+        .tv_usec = 0,
+    };
+    times[1] = {
+        .tv_sec = st.st_mtime,
+        .tv_usec = 0,
+    };
+    if (lutimes(p.c_str(), times) != 0)
+        throw SysError("changing modification time of '%s'", p);
+}
+
+void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete)
+{
+    // TODO: Rewrite the `is_*` to use `symlink_status()`
+    auto statOfFrom = lstat(from.path().c_str());
+    auto fromStatus = from.symlink_status();
+
+    // Mark the directory as writable so that we can delete its children
+    if (andDelete && fs::is_directory(fromStatus)) {
+        fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+    }
+
+
+    if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) {
+        fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing);
+    } else if (fs::is_directory(fromStatus)) {
+        fs::create_directory(to);
+        for (auto & entry : fs::directory_iterator(from.path())) {
+            copy(entry, to / entry.path().filename(), andDelete);
+        }
+    } else {
+        throw Error("file '%s' has an unsupported type", from.path());
+    }
+
+    setWriteTime(to, statOfFrom);
+    if (andDelete) {
+        if (!fs::is_symlink(fromStatus))
+            fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow);
+        fs::remove(from.path());
+    }
+}
+
+void renameFile(const Path & oldName, const Path & newName)
+{
+    fs::rename(oldName, newName);
+}
+
+void moveFile(const Path & oldName, const Path & newName)
+{
+    try {
+        renameFile(oldName, newName);
+    } catch (fs::filesystem_error & e) {
+        auto oldPath = fs::path(oldName);
+        auto newPath = fs::path(newName);
+        // For the move to be as atomic as possible, copy to a temporary
+        // directory
+        fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp");
+        Finally removeTemp = [&]() { fs::remove(temp); };
+        auto tempCopyTarget = temp / "copy-target";
+        if (e.code().value() == EXDEV) {
+            fs::remove(newPath);
+            warn("Can’t rename %s as %s, copying instead", oldName, newName);
+            copy(fs::directory_entry(oldPath), tempCopyTarget, true);
+            renameFile(tempCopyTarget, newPath);
+        }
+    }
+}
+
+}
diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index 0821b3b74..e879fd3b8 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -2,7 +2,6 @@
 
 #include <boost/format.hpp>
 #include <string>
-#include <regex>
 #include "ansicolor.hh"
 
 
@@ -149,21 +148,10 @@ inline hintformat hintfmt(const std::string & fs, const Args & ... args)
     return f;
 }
 
-inline hintformat hintfmt(std::string plain_string)
+inline hintformat hintfmt(const std::string & plain_string)
 {
     // we won't be receiving any args in this case, so just print the original string
     return hintfmt("%s", normaltxt(plain_string));
 }
 
-/* Highlight all the given matches in the given string `s` by wrapping
-   them between `prefix` and `postfix`.
-
-   If some matches overlap, then their union will be wrapped rather
-   than the individual matches. */
-std::string hiliteMatches(
-    std::string_view s,
-    std::vector<std::smatch> matches,
-    std::string_view prefix,
-    std::string_view postfix);
-
 }
diff --git a/src/libutil/git.cc b/src/libutil/git.cc
new file mode 100644
index 000000000..f35c2fdb7
--- /dev/null
+++ b/src/libutil/git.cc
@@ -0,0 +1,25 @@
+#include "git.hh"
+
+#include <regex>
+
+namespace nix {
+namespace git {
+
+std::optional<LsRemoteRefLine> parseLsRemoteLine(std::string_view line)
+{
+    const static std::regex line_regex("^(ref: *)?([^\\s]+)(?:\\t+(.*))?$");
+    std::match_results<std::string_view::const_iterator> match;
+    if (!std::regex_match(line.cbegin(), line.cend(), match, line_regex))
+        return std::nullopt;
+
+    return LsRemoteRefLine {
+        .kind = match[1].length() == 0
+            ? LsRemoteRefLine::Kind::Object
+            : LsRemoteRefLine::Kind::Symbolic,
+        .target = match[2],
+        .reference = match[3].length() == 0 ? std::nullopt : std::optional<std::string>{ match[3] }
+    };
+}
+
+}
+}
diff --git a/src/libutil/git.hh b/src/libutil/git.hh
new file mode 100644
index 000000000..cb13ef0e5
--- /dev/null
+++ b/src/libutil/git.hh
@@ -0,0 +1,40 @@
+#pragma once
+
+#include <string>
+#include <string_view>
+#include <optional>
+
+namespace nix {
+
+namespace git {
+
+// A line from the output of `git ls-remote --symref`.
+//
+// These can be of two kinds:
+//
+// - Symbolic references of the form
+//
+//      ref: {target}	{reference}
+//
+//    where {target} is itself a reference and {reference} is optional
+//
+// - Object references of the form
+//
+//      {target}	{reference}
+//
+//    where {target} is a commit id and {reference} is mandatory
+struct LsRemoteRefLine {
+    enum struct Kind {
+        Symbolic,
+        Object
+    };
+    Kind kind;
+    std::string target;
+    std::optional<std::string> reference;
+};
+
+std::optional<LsRemoteRefLine> parseLsRemoteLine(std::string_view line);
+
+}
+
+}
diff --git a/src/libutil/fmt.cc b/src/libutil/hilite.cc
index 3dd93d73e..e5088230d 100644
--- a/src/libutil/fmt.cc
+++ b/src/libutil/hilite.cc
@@ -1,6 +1,4 @@
-#include "fmt.hh"
-
-#include <regex>
+#include "hilite.hh"
 
 namespace nix {
 
@@ -10,9 +8,9 @@ std::string hiliteMatches(
     std::string_view prefix,
     std::string_view postfix)
 {
-    // Avoid copy on zero matches
+    // Avoid extra work on zero matches
     if (matches.size() == 0)
-        return (std::string) s;
+        return std::string(s);
 
     std::sort(matches.begin(), matches.end(), [](const auto & a, const auto & b) {
         return a.position() < b.position();
diff --git a/src/libutil/hilite.hh b/src/libutil/hilite.hh
new file mode 100644
index 000000000..f8bdbfc55
--- /dev/null
+++ b/src/libutil/hilite.hh
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <regex>
+#include <vector>
+#include <string>
+
+namespace nix {
+
+/* Highlight all the given matches in the given string `s` by wrapping
+   them between `prefix` and `postfix`.
+
+   If some matches overlap, then their union will be wrapped rather
+   than the individual matches. */
+std::string hiliteMatches(
+    std::string_view s,
+    std::vector<std::smatch> matches,
+    std::string_view prefix,
+    std::string_view postfix);
+
+}
diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh
new file mode 100644
index 000000000..b8a031227
--- /dev/null
+++ b/src/libutil/json-utils.hh
@@ -0,0 +1,21 @@
+#pragma once
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+const nlohmann::json * get(const nlohmann::json & map, const std::string & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &*i;
+}
+
+nlohmann::json * get(nlohmann::json & map, const std::string & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &*i;
+}
+
+}
diff --git a/src/libutil/json.cc b/src/libutil/json.cc
deleted file mode 100644
index 3a981376f..000000000
--- a/src/libutil/json.cc
+++ /dev/null
@@ -1,201 +0,0 @@
-#include "json.hh"
-
-#include <iomanip>
-#include <cstring>
-
-namespace nix {
-
-void toJSON(std::ostream & str, const char * start, const char * end)
-{
-    constexpr size_t BUF_SIZE = 4096;
-    char buf[BUF_SIZE + 7]; // BUF_SIZE + largest single sequence of puts
-    size_t bufPos = 0;
-
-    const auto flush = [&] {
-        str.write(buf, bufPos);
-        bufPos = 0;
-    };
-    const auto put = [&] (char c) {
-        buf[bufPos++] = c;
-    };
-
-    put('"');
-    for (auto i = start; i != end; i++) {
-        if (bufPos >= BUF_SIZE) flush();
-        if (*i == '\"' || *i == '\\') { put('\\'); put(*i); }
-        else if (*i == '\n') { put('\\'); put('n'); }
-        else if (*i == '\r') { put('\\'); put('r'); }
-        else if (*i == '\t') { put('\\'); put('t'); }
-        else if (*i >= 0 && *i < 32) {
-            const char hex[17] = "0123456789abcdef";
-            put('\\');
-            put('u');
-            put(hex[(uint16_t(*i) >> 12) & 0xf]);
-            put(hex[(uint16_t(*i) >>  8) & 0xf]);
-            put(hex[(uint16_t(*i) >>  4) & 0xf]);
-            put(hex[(uint16_t(*i) >>  0) & 0xf]);
-        }
-        else put(*i);
-    }
-    put('"');
-    flush();
-}
-
-void toJSON(std::ostream & str, const char * s)
-{
-    if (!s) str << "null"; else toJSON(str, s, s + strlen(s));
-}
-
-template<> void toJSON<int>(std::ostream & str, const int & n) { str << n; }
-template<> void toJSON<unsigned int>(std::ostream & str, const unsigned int & n) { str << n; }
-template<> void toJSON<long>(std::ostream & str, const long & n) { str << n; }
-template<> void toJSON<unsigned long>(std::ostream & str, const unsigned long & n) { str << n; }
-template<> void toJSON<long long>(std::ostream & str, const long long & n) { str << n; }
-template<> void toJSON<unsigned long long>(std::ostream & str, const unsigned long long & n) { str << n; }
-template<> void toJSON<float>(std::ostream & str, const float & n) { str << n; }
-template<> void toJSON<double>(std::ostream & str, const double & n) { str << n; }
-
-template<> void toJSON<std::string>(std::ostream & str, const std::string & s)
-{
-    toJSON(str, s.c_str(), s.c_str() + s.size());
-}
-
-template<> void toJSON<bool>(std::ostream & str, const bool & b)
-{
-    str << (b ? "true" : "false");
-}
-
-template<> void toJSON<std::nullptr_t>(std::ostream & str, const std::nullptr_t & b)
-{
-    str << "null";
-}
-
-JSONWriter::JSONWriter(std::ostream & str, bool indent)
-    : state(new JSONState(str, indent))
-{
-    state->stack++;
-}
-
-JSONWriter::JSONWriter(JSONState * state)
-    : state(state)
-{
-    state->stack++;
-}
-
-JSONWriter::~JSONWriter()
-{
-    if (state) {
-        assertActive();
-        state->stack--;
-        if (state->stack == 0) delete state;
-    }
-}
-
-void JSONWriter::comma()
-{
-    assertActive();
-    if (first) {
-        first = false;
-    } else {
-        state->str << ',';
-    }
-    if (state->indent) indent();
-}
-
-void JSONWriter::indent()
-{
-    state->str << '\n' << std::string(state->depth * 2, ' ');
-}
-
-void JSONList::open()
-{
-    state->depth++;
-    state->str << '[';
-}
-
-JSONList::~JSONList()
-{
-    state->depth--;
-    if (state->indent && !first) indent();
-    state->str << "]";
-}
-
-JSONList JSONList::list()
-{
-    comma();
-    return JSONList(state);
-}
-
-JSONObject JSONList::object()
-{
-    comma();
-    return JSONObject(state);
-}
-
-JSONPlaceholder JSONList::placeholder()
-{
-    comma();
-    return JSONPlaceholder(state);
-}
-
-void JSONObject::open()
-{
-    state->depth++;
-    state->str << '{';
-}
-
-JSONObject::~JSONObject()
-{
-    if (state) {
-        state->depth--;
-        if (state->indent && !first) indent();
-        state->str << "}";
-    }
-}
-
-void JSONObject::attr(const std::string & s)
-{
-    comma();
-    toJSON(state->str, s);
-    state->str << ':';
-    if (state->indent) state->str << ' ';
-}
-
-JSONList JSONObject::list(const std::string & name)
-{
-    attr(name);
-    return JSONList(state);
-}
-
-JSONObject JSONObject::object(const std::string & name)
-{
-    attr(name);
-    return JSONObject(state);
-}
-
-JSONPlaceholder JSONObject::placeholder(const std::string & name)
-{
-    attr(name);
-    return JSONPlaceholder(state);
-}
-
-JSONList JSONPlaceholder::list()
-{
-    assertValid();
-    first = false;
-    return JSONList(state);
-}
-
-JSONObject JSONPlaceholder::object()
-{
-    assertValid();
-    first = false;
-    return JSONObject(state);
-}
-
-JSONPlaceholder::~JSONPlaceholder()
-{
-    assert(!first || std::uncaught_exceptions());
-}
-
-}
diff --git a/src/libutil/json.hh b/src/libutil/json.hh
deleted file mode 100644
index 83213ca66..000000000
--- a/src/libutil/json.hh
+++ /dev/null
@@ -1,186 +0,0 @@
-#pragma once
-
-#include <iostream>
-#include <vector>
-#include <cassert>
-
-namespace nix {
-
-void toJSON(std::ostream & str, const char * start, const char * end);
-void toJSON(std::ostream & str, const char * s);
-
-template<typename T>
-void toJSON(std::ostream & str, const T & n);
-
-class JSONWriter
-{
-protected:
-
-    struct JSONState
-    {
-        std::ostream & str;
-        bool indent;
-        size_t depth = 0;
-        size_t stack = 0;
-        JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { }
-        ~JSONState()
-        {
-            assert(stack == 0);
-        }
-    };
-
-    JSONState * state;
-
-    bool first = true;
-
-    JSONWriter(std::ostream & str, bool indent);
-
-    JSONWriter(JSONState * state);
-
-    ~JSONWriter();
-
-    void assertActive()
-    {
-        assert(state->stack != 0);
-    }
-
-    void comma();
-
-    void indent();
-};
-
-class JSONObject;
-class JSONPlaceholder;
-
-class JSONList : JSONWriter
-{
-private:
-
-    friend class JSONObject;
-    friend class JSONPlaceholder;
-
-    void open();
-
-    JSONList(JSONState * state)
-        : JSONWriter(state)
-    {
-        open();
-    }
-
-public:
-
-    JSONList(std::ostream & str, bool indent = false)
-        : JSONWriter(str, indent)
-    {
-        open();
-    }
-
-    ~JSONList();
-
-    template<typename T>
-    JSONList & elem(const T & v)
-    {
-        comma();
-        toJSON(state->str, v);
-        return *this;
-    }
-
-    JSONList list();
-
-    JSONObject object();
-
-    JSONPlaceholder placeholder();
-};
-
-class JSONObject : JSONWriter
-{
-private:
-
-    friend class JSONList;
-    friend class JSONPlaceholder;
-
-    void open();
-
-    JSONObject(JSONState * state)
-        : JSONWriter(state)
-    {
-        open();
-    }
-
-    void attr(const std::string & s);
-
-public:
-
-    JSONObject(std::ostream & str, bool indent = false)
-        : JSONWriter(str, indent)
-    {
-        open();
-    }
-
-    JSONObject(const JSONObject & obj) = delete;
-
-    JSONObject(JSONObject && obj)
-        : JSONWriter(obj.state)
-    {
-        obj.state = 0;
-    }
-
-    ~JSONObject();
-
-    template<typename T>
-    JSONObject & attr(const std::string & name, const T & v)
-    {
-        attr(name);
-        toJSON(state->str, v);
-        return *this;
-    }
-
-    JSONList list(const std::string & name);
-
-    JSONObject object(const std::string & name);
-
-    JSONPlaceholder placeholder(const std::string & name);
-};
-
-class JSONPlaceholder : JSONWriter
-{
-
-private:
-
-    friend class JSONList;
-    friend class JSONObject;
-
-    JSONPlaceholder(JSONState * state)
-        : JSONWriter(state)
-    {
-    }
-
-    void assertValid()
-    {
-        assertActive();
-        assert(first);
-    }
-
-public:
-
-    JSONPlaceholder(std::ostream & str, bool indent = false)
-        : JSONWriter(str, indent)
-    {
-    }
-
-    ~JSONPlaceholder();
-
-    template<typename T>
-    void write(const T & v)
-    {
-        assertValid();
-        first = false;
-        toJSON(state->str, v);
-    }
-
-    JSONList list();
-
-    JSONObject object();
-};
-
-}
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index cb2b15b41..904ba6ebe 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -105,14 +105,6 @@ public:
 
 Verbosity verbosity = lvlInfo;
 
-void warnOnce(bool & haveWarned, const FormatOrString & fs)
-{
-    if (!haveWarned) {
-        warn(fs.s);
-        haveWarned = true;
-    }
-}
-
 void writeToStderr(std::string_view s)
 {
     try {
@@ -130,15 +122,30 @@ Logger * makeSimpleLogger(bool printBuildLogs)
     return new SimpleLogger(printBuildLogs);
 }
 
-std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32};
+std::atomic<uint64_t> nextId{0};
 
 Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type,
     const std::string & s, const Logger::Fields & fields, ActivityId parent)
-    : logger(logger), id(nextId++)
+    : logger(logger), id(nextId++ + (((uint64_t) getpid()) << 32))
 {
     logger.startActivity(id, lvl, type, s, fields, parent);
 }
 
+void to_json(nlohmann::json & json, std::shared_ptr<AbstractPos> pos)
+{
+    if (pos) {
+        json["line"] = pos->line;
+        json["column"] = pos->column;
+        std::ostringstream str;
+        pos->print(str);
+        json["file"] = str.str();
+    } else {
+        json["line"] = nullptr;
+        json["column"] = nullptr;
+        json["file"] = nullptr;
+    }
+}
+
 struct JSONLogger : Logger {
     Logger & prevLogger;
 
@@ -185,27 +192,14 @@ struct JSONLogger : Logger {
         json["level"] = ei.level;
         json["msg"] = oss.str();
         json["raw_msg"] = ei.msg.str();
-
-        if (ei.errPos.has_value() && (*ei.errPos)) {
-            json["line"] = ei.errPos->line;
-            json["column"] = ei.errPos->column;
-            json["file"] = ei.errPos->file;
-        } else {
-            json["line"] = nullptr;
-            json["column"] = nullptr;
-            json["file"] = nullptr;
-        }
+        to_json(json, ei.errPos);
 
         if (loggerSettings.showTrace.get() && !ei.traces.empty()) {
             nlohmann::json traces = nlohmann::json::array();
             for (auto iter = ei.traces.rbegin(); iter != ei.traces.rend(); ++iter) {
                 nlohmann::json stackFrame;
                 stackFrame["raw_msg"] = iter->hint.str();
-                if (iter->pos.has_value() && (*iter->pos)) {
-                    stackFrame["line"] = iter->pos->line;
-                    stackFrame["column"] = iter->pos->column;
-                    stackFrame["file"] = iter->pos->file;
-                }
+                to_json(stackFrame, iter->pos);
                 traces.push_back(stackFrame);
             }
 
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 6f81b92de..4642c49f7 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -82,7 +82,7 @@ public:
         log(lvlInfo, fs);
     }
 
-    virtual void logEI(const ErrorInfo &ei) = 0;
+    virtual void logEI(const ErrorInfo & ei) = 0;
 
     void logEI(Verbosity lvl, ErrorInfo ei)
     {
@@ -111,6 +111,9 @@ public:
 
     virtual std::optional<char> ask(std::string_view s)
     { return {}; }
+
+    virtual void setPrintBuildLogs(bool printBuildLogs)
+    { }
 };
 
 ActivityId getCurActivity();
@@ -222,7 +225,11 @@ inline void warn(const std::string & fs, const Args & ... args)
     logger->warn(f.str());
 }
 
-void warnOnce(bool & haveWarned, const FormatOrString & fs);
+#define warnOnce(haveWarned, args...) \
+    if (!haveWarned) {                \
+        haveWarned = true;            \
+        warn(args);                   \
+    }
 
 void writeToStderr(std::string_view s);
 
diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh
index 347b81f73..7d38b059c 100644
--- a/src/libutil/ref.hh
+++ b/src/libutil/ref.hh
@@ -7,7 +7,7 @@
 namespace nix {
 
 /* A simple non-nullable reference-counted pointer. Actually a wrapper
-   around std::shared_ptr that prevents non-null constructions. */
+   around std::shared_ptr that prevents null constructions. */
 template<typename T>
 class ref
 {
@@ -83,6 +83,11 @@ public:
         return p != other.p;
     }
 
+    bool operator < (const ref<T> & other) const
+    {
+        return p < other.p;
+    }
+
 private:
 
     template<typename T2, typename... Args>
@@ -99,47 +104,4 @@ make_ref(Args&&... args)
     return ref<T>(p);
 }
 
-
-/* A non-nullable pointer.
-   This is similar to a C++ "& reference", but mutable.
-   This is similar to ref<T> but backed by a regular pointer instead of a smart pointer.
- */
-template<typename T>
-class ptr {
-private:
-    T * p;
-
-public:
-    ptr<T>(const ptr<T> & r)
-        : p(r.p)
-    { }
-
-    explicit ptr<T>(T * p)
-        : p(p)
-    {
-        if (!p)
-            throw std::invalid_argument("null pointer cast to ptr");
-    }
-
-    T* operator ->() const
-    {
-        return &*p;
-    }
-
-    T& operator *() const
-    {
-        return *p;
-    }
-
-    bool operator == (const ptr<T> & other) const
-    {
-        return p == other.p;
-    }
-
-    bool operator != (const ptr<T> & other) const
-    {
-        return p != other.p;
-    }
-};
-
 }
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 8ff904583..c653db9d0 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -48,24 +48,9 @@ FdSink::~FdSink()
 }
 
 
-size_t threshold = 256 * 1024 * 1024;
-
-static void warnLargeDump()
-{
-    warn("dumping very large path (> 256 MiB); this may run out of memory");
-}
-
-
 void FdSink::write(std::string_view data)
 {
     written += data.size();
-    static bool warned = false;
-    if (warn && !warned) {
-        if (written > threshold) {
-            warnLargeDump();
-            warned = true;
-        }
-    }
     try {
         writeFull(fd, data);
     } catch (SysError & e) {
@@ -353,7 +338,7 @@ Sink & operator << (Sink & sink, const StringSet & s)
 
 Sink & operator << (Sink & sink, const Error & ex)
 {
-    auto info = ex.info();
+    auto & info = ex.info();
     sink
         << "Error"
         << info.level
@@ -448,11 +433,6 @@ Error readError(Source & source)
 
 void StringSink::operator () (std::string_view data)
 {
-    static bool warned = false;
-    if (!warned && s.size() > threshold) {
-        warnLargeDump();
-        warned = true;
-    }
     s.append(data);
 }
 
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 13da26c6a..7da5b07fd 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -97,19 +97,17 @@ protected:
 struct FdSink : BufferedSink
 {
     int fd;
-    bool warn = false;
     size_t written = 0;
 
     FdSink() : fd(-1) { }
     FdSink(int fd) : fd(fd) { }
     FdSink(FdSink&&) = default;
 
-    FdSink& operator=(FdSink && s)
+    FdSink & operator=(FdSink && s)
     {
         flush();
         fd = s.fd;
         s.fd = -1;
-        warn = s.warn;
         written = s.written;
         return *this;
     }
@@ -333,17 +331,9 @@ T readNum(Source & source)
     unsigned char buf[8];
     source((char *) buf, sizeof(buf));
 
-    uint64_t n =
-        ((uint64_t) buf[0]) |
-        ((uint64_t) buf[1] << 8) |
-        ((uint64_t) buf[2] << 16) |
-        ((uint64_t) buf[3] << 24) |
-        ((uint64_t) buf[4] << 32) |
-        ((uint64_t) buf[5] << 40) |
-        ((uint64_t) buf[6] << 48) |
-        ((uint64_t) buf[7] << 56);
-
-    if (n > (uint64_t)std::numeric_limits<T>::max())
+    auto n = readLittleEndian<uint64_t>(buf);
+
+    if (n > (uint64_t) std::numeric_limits<T>::max())
         throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name());
 
     return (T) n;
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index a7db58559..238d0a7a6 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -77,9 +77,7 @@ TarArchive::~TarArchive()
 
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {
-    int flags = ARCHIVE_EXTRACT_FFLAGS
-        | ARCHIVE_EXTRACT_PERM
-        | ARCHIVE_EXTRACT_TIME
+    int flags = ARCHIVE_EXTRACT_TIME
         | ARCHIVE_EXTRACT_SECURE_SYMLINKS
         | ARCHIVE_EXTRACT_SECURE_NODOTDOT;
 
@@ -98,6 +96,10 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
         archive_entry_copy_pathname(entry,
             (destDir + "/" + name).c_str());
 
+        // sources can and do contain dirs with no rx bits
+        if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500)
+            archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500);
+
         // Patch hardlink path
         const char *original_hardlink = archive_entry_hardlink(entry);
         if (original_hardlink) {
diff --git a/src/libutil/tests/canon-path.cc b/src/libutil/tests/canon-path.cc
new file mode 100644
index 000000000..c1c5adadf
--- /dev/null
+++ b/src/libutil/tests/canon-path.cc
@@ -0,0 +1,155 @@
+#include "canon-path.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    TEST(CanonPath, basic) {
+        {
+            CanonPath p("/");
+            ASSERT_EQ(p.abs(), "/");
+            ASSERT_EQ(p.rel(), "");
+            ASSERT_EQ(p.baseName(), std::nullopt);
+            ASSERT_EQ(p.dirOf(), std::nullopt);
+            ASSERT_FALSE(p.parent());
+        }
+
+        {
+            CanonPath p("/foo//");
+            ASSERT_EQ(p.abs(), "/foo");
+            ASSERT_EQ(p.rel(), "foo");
+            ASSERT_EQ(*p.baseName(), "foo");
+            ASSERT_EQ(*p.dirOf(), ""); // FIXME: do we want this?
+            ASSERT_EQ(p.parent()->abs(), "/");
+        }
+
+        {
+            CanonPath p("foo/bar");
+            ASSERT_EQ(p.abs(), "/foo/bar");
+            ASSERT_EQ(p.rel(), "foo/bar");
+            ASSERT_EQ(*p.baseName(), "bar");
+            ASSERT_EQ(*p.dirOf(), "/foo");
+            ASSERT_EQ(p.parent()->abs(), "/foo");
+        }
+
+        {
+            CanonPath p("foo//bar/");
+            ASSERT_EQ(p.abs(), "/foo/bar");
+            ASSERT_EQ(p.rel(), "foo/bar");
+            ASSERT_EQ(*p.baseName(), "bar");
+            ASSERT_EQ(*p.dirOf(), "/foo");
+        }
+    }
+
+    TEST(CanonPath, pop) {
+        CanonPath p("foo/bar/x");
+        ASSERT_EQ(p.abs(), "/foo/bar/x");
+        p.pop();
+        ASSERT_EQ(p.abs(), "/foo/bar");
+        p.pop();
+        ASSERT_EQ(p.abs(), "/foo");
+        p.pop();
+        ASSERT_EQ(p.abs(), "/");
+    }
+
+    TEST(CanonPath, removePrefix) {
+        CanonPath p1("foo/bar");
+        CanonPath p2("foo/bar/a/b/c");
+        ASSERT_EQ(p2.removePrefix(p1).abs(), "/a/b/c");
+        ASSERT_EQ(p1.removePrefix(p1).abs(), "/");
+        ASSERT_EQ(p1.removePrefix(CanonPath("/")).abs(), "/foo/bar");
+    }
+
+    TEST(CanonPath, iter) {
+        {
+            CanonPath p("a//foo/bar//");
+            std::vector<std::string_view> ss;
+            for (auto & c : p) ss.push_back(c);
+            ASSERT_EQ(ss, std::vector<std::string_view>({"a", "foo", "bar"}));
+        }
+
+        {
+            CanonPath p("/");
+            std::vector<std::string_view> ss;
+            for (auto & c : p) ss.push_back(c);
+            ASSERT_EQ(ss, std::vector<std::string_view>());
+        }
+    }
+
+    TEST(CanonPath, concat) {
+        {
+            CanonPath p1("a//foo/bar//");
+            CanonPath p2("xyzzy/bla");
+            ASSERT_EQ((p1 + p2).abs(), "/a/foo/bar/xyzzy/bla");
+        }
+
+        {
+            CanonPath p1("/");
+            CanonPath p2("/a/b");
+            ASSERT_EQ((p1 + p2).abs(), "/a/b");
+        }
+
+        {
+            CanonPath p1("/a/b");
+            CanonPath p2("/");
+            ASSERT_EQ((p1 + p2).abs(), "/a/b");
+        }
+
+        {
+            CanonPath p("/foo/bar");
+            ASSERT_EQ((p + "x").abs(), "/foo/bar/x");
+        }
+
+        {
+            CanonPath p("/");
+            ASSERT_EQ((p + "foo" + "bar").abs(), "/foo/bar");
+        }
+    }
+
+    TEST(CanonPath, within) {
+        {
+            ASSERT_TRUE(CanonPath("foo").isWithin(CanonPath("foo")));
+            ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("bar")));
+            ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("fo")));
+            ASSERT_TRUE(CanonPath("foo/bar").isWithin(CanonPath("foo")));
+            ASSERT_FALSE(CanonPath("foo").isWithin(CanonPath("foo/bar")));
+            ASSERT_TRUE(CanonPath("/foo/bar/default.nix").isWithin(CanonPath("/")));
+            ASSERT_TRUE(CanonPath("/").isWithin(CanonPath("/")));
+        }
+    }
+
+    TEST(CanonPath, sort) {
+        ASSERT_FALSE(CanonPath("foo") < CanonPath("foo"));
+        ASSERT_TRUE (CanonPath("foo") < CanonPath("foo/bar"));
+        ASSERT_TRUE (CanonPath("foo/bar") < CanonPath("foo!"));
+        ASSERT_FALSE(CanonPath("foo!") < CanonPath("foo"));
+        ASSERT_TRUE (CanonPath("foo") < CanonPath("foo!"));
+    }
+
+    TEST(CanonPath, allowed) {
+        {
+            std::set<CanonPath> allowed {
+                CanonPath("foo/bar"),
+                CanonPath("foo!"),
+                CanonPath("xyzzy"),
+                CanonPath("a/b/c"),
+            };
+
+            ASSERT_TRUE (CanonPath("foo/bar").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("foo/bar/bla").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("foo").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("bar").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("bar/a").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("a").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("a/b").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("a/b/c").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("a/b/c/d").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("a/b/c/d/e").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("a/b/a").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("a/b/d").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("aaa").isAllowed(allowed));
+            ASSERT_FALSE(CanonPath("zzz").isAllowed(allowed));
+            ASSERT_TRUE (CanonPath("/").isAllowed(allowed));
+        }
+    }
+}
diff --git a/src/libutil/tests/chunked-vector.cc b/src/libutil/tests/chunked-vector.cc
new file mode 100644
index 000000000..868d11f6f
--- /dev/null
+++ b/src/libutil/tests/chunked-vector.cc
@@ -0,0 +1,54 @@
+#include "chunked-vector.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+    TEST(ChunkedVector, InitEmpty) {
+        auto v = ChunkedVector<int, 2>(100);
+        ASSERT_EQ(v.size(), 0);
+    }
+
+    TEST(ChunkedVector, GrowsCorrectly) {
+        auto v = ChunkedVector<int, 2>(100);
+        for (auto i = 1; i < 20; i++) {
+            v.add(i);
+            ASSERT_EQ(v.size(), i);
+        }
+    }
+
+    TEST(ChunkedVector, AddAndGet) {
+        auto v = ChunkedVector<int, 2>(100);
+        for (auto i = 1; i < 20; i++) {
+            auto [i2, idx] = v.add(i);
+            auto & i3 = v[idx];
+            ASSERT_EQ(i, i2);
+            ASSERT_EQ(&i2, &i3);
+        }
+    }
+
+    TEST(ChunkedVector, ForEach) {
+        auto v = ChunkedVector<int, 2>(100);
+        for (auto i = 1; i < 20; i++) {
+            v.add(i);
+        }
+        int count = 0;
+        v.forEach([&count](int elt) {
+            count++;
+        });
+        ASSERT_EQ(count, v.size());
+    }
+
+    TEST(ChunkedVector, OverflowOK) {
+        // Similar to the AddAndGet, but intentionnally use a small
+        // initial ChunkedVector to force it to overflow
+        auto v = ChunkedVector<int, 2>(2);
+        for (auto i = 1; i < 20; i++) {
+            auto [i2, idx] = v.add(i);
+            auto & i3 = v[idx];
+            ASSERT_EQ(i, i2);
+            ASSERT_EQ(&i2, &i3);
+        }
+    }
+
+}
+
diff --git a/src/libutil/tests/git.cc b/src/libutil/tests/git.cc
new file mode 100644
index 000000000..5b5715fc2
--- /dev/null
+++ b/src/libutil/tests/git.cc
@@ -0,0 +1,33 @@
+#include "git.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+    TEST(GitLsRemote, parseSymrefLineWithReference) {
+        auto line = "ref: refs/head/main	HEAD";
+        auto res = git::parseLsRemoteLine(line);
+        ASSERT_TRUE(res.has_value());
+        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic);
+        ASSERT_EQ(res->target, "refs/head/main");
+        ASSERT_EQ(res->reference, "HEAD");
+    }
+
+    TEST(GitLsRemote, parseSymrefLineWithNoReference) {
+        auto line = "ref: refs/head/main";
+        auto res = git::parseLsRemoteLine(line);
+        ASSERT_TRUE(res.has_value());
+        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic);
+        ASSERT_EQ(res->target, "refs/head/main");
+        ASSERT_EQ(res->reference, std::nullopt);
+    }
+
+    TEST(GitLsRemote, parseObjectRefLine) {
+        auto line = "abc123	refs/head/main";
+        auto res = git::parseLsRemoteLine(line);
+        ASSERT_TRUE(res.has_value());
+        ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object);
+        ASSERT_EQ(res->target, "abc123");
+        ASSERT_EQ(res->reference, "refs/head/main");
+    }
+}
+
diff --git a/src/libutil/tests/fmt.cc b/src/libutil/tests/hilite.cc
index 33772162c..1ff5980d5 100644
--- a/src/libutil/tests/fmt.cc
+++ b/src/libutil/tests/hilite.cc
@@ -1,9 +1,7 @@
-#include "fmt.hh"
+#include "hilite.hh"
 
 #include <gtest/gtest.h>
 
-#include <regex>
-
 namespace nix {
 /* ----------- tests for fmt.hh -------------------------------------------------*/
 
diff --git a/src/libutil/tests/json.cc b/src/libutil/tests/json.cc
deleted file mode 100644
index dea73f53a..000000000
--- a/src/libutil/tests/json.cc
+++ /dev/null
@@ -1,193 +0,0 @@
-#include "json.hh"
-#include <gtest/gtest.h>
-#include <sstream>
-
-namespace nix {
-
-    /* ----------------------------------------------------------------------------
-     * toJSON
-     * --------------------------------------------------------------------------*/
-
-    TEST(toJSON, quotesCharPtr) {
-        const char* input = "test";
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "\"test\"");
-    }
-
-    TEST(toJSON, quotesStdString) {
-        std::string input = "test";
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "\"test\"");
-    }
-
-    TEST(toJSON, convertsNullptrtoNull) {
-        auto input = nullptr;
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "null");
-    }
-
-    TEST(toJSON, convertsNullToNull) {
-        const char* input = 0;
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "null");
-    }
-
-
-    TEST(toJSON, convertsFloat) {
-        auto input = 1.024f;
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "1.024");
-    }
-
-    TEST(toJSON, convertsDouble) {
-        const double input = 1.024;
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "1.024");
-    }
-
-    TEST(toJSON, convertsBool) {
-        auto input = false;
-        std::stringstream out;
-        toJSON(out, input);
-
-        ASSERT_EQ(out.str(), "false");
-    }
-
-    TEST(toJSON, quotesTab) {
-        std::stringstream out;
-        toJSON(out, "\t");
-
-        ASSERT_EQ(out.str(), "\"\\t\"");
-    }
-
-    TEST(toJSON, quotesNewline) {
-        std::stringstream out;
-        toJSON(out, "\n");
-
-        ASSERT_EQ(out.str(), "\"\\n\"");
-    }
-
-    TEST(toJSON, quotesCreturn) {
-        std::stringstream out;
-        toJSON(out, "\r");
-
-        ASSERT_EQ(out.str(), "\"\\r\"");
-    }
-
-    TEST(toJSON, quotesCreturnNewLine) {
-        std::stringstream out;
-        toJSON(out, "\r\n");
-
-        ASSERT_EQ(out.str(), "\"\\r\\n\"");
-    }
-
-    TEST(toJSON, quotesDoublequotes) {
-        std::stringstream out;
-        toJSON(out, "\"");
-
-        ASSERT_EQ(out.str(), "\"\\\"\"");
-    }
-
-    TEST(toJSON, substringEscape) {
-        std::stringstream out;
-        const char *s = "foo\t";
-        toJSON(out, s+3, s + strlen(s));
-
-        ASSERT_EQ(out.str(), "\"\\t\"");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * JSONObject
-     * --------------------------------------------------------------------------*/
-
-    TEST(JSONObject, emptyObject) {
-        std::stringstream out;
-        {
-            JSONObject t(out);
-        }
-        ASSERT_EQ(out.str(), "{}");
-    }
-
-    TEST(JSONObject, objectWithList) {
-        std::stringstream out;
-        {
-            JSONObject t(out);
-            auto l = t.list("list");
-            l.elem("element");
-        }
-        ASSERT_EQ(out.str(), R"#({"list":["element"]})#");
-    }
-
-    TEST(JSONObject, objectWithListIndent) {
-        std::stringstream out;
-        {
-            JSONObject t(out, true);
-            auto l = t.list("list");
-            l.elem("element");
-        }
-        ASSERT_EQ(out.str(),
-R"#({
-  "list": [
-    "element"
-  ]
-})#");
-    }
-
-    TEST(JSONObject, objectWithPlaceholderAndList) {
-        std::stringstream out;
-        {
-            JSONObject t(out);
-            auto l = t.placeholder("list");
-            l.list().elem("element");
-        }
-
-        ASSERT_EQ(out.str(), R"#({"list":["element"]})#");
-    }
-
-    TEST(JSONObject, objectWithPlaceholderAndObject) {
-        std::stringstream out;
-        {
-            JSONObject t(out);
-            auto l = t.placeholder("object");
-            l.object().attr("key", "value");
-        }
-
-        ASSERT_EQ(out.str(), R"#({"object":{"key":"value"}})#");
-    }
-
-    /* ----------------------------------------------------------------------------
-     * JSONList
-     * --------------------------------------------------------------------------*/
-
-    TEST(JSONList, empty) {
-        std::stringstream out;
-        {
-            JSONList l(out);
-        }
-        ASSERT_EQ(out.str(), R"#([])#");
-    }
-
-    TEST(JSONList, withElements) {
-        std::stringstream out;
-        {
-            JSONList l(out);
-            l.elem("one");
-            l.object();
-            l.placeholder().write("three");
-        }
-        ASSERT_EQ(out.str(), R"#(["one",{},"three"])#");
-    }
-}
-
diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index 92972ed14..6e325db98 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -548,7 +548,7 @@ namespace nix {
 
     TEST(get, emptyContainer) {
         StringMap s = { };
-        auto expected = std::nullopt;
+        auto expected = nullptr;
 
         ASSERT_EQ(get(s, "one"), expected);
     }
@@ -559,7 +559,23 @@ namespace nix {
         s["two"] = "er";
         auto expected = "yi";
 
-        ASSERT_EQ(get(s, "one"), expected);
+        ASSERT_EQ(*get(s, "one"), expected);
+    }
+
+    TEST(getOr, emptyContainer) {
+        StringMap s = { };
+        auto expected = "yi";
+
+        ASSERT_EQ(getOr(s, "one", "yi"), expected);
+    }
+
+    TEST(getOr, getFromContainer) {
+        StringMap s;
+        s["one"] = "yi";
+        s["two"] = "er";
+        auto expected = "yi";
+
+        ASSERT_EQ(getOr(s, "one", "nope"), expected);
     }
 
     /* ----------------------------------------------------------------------------
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 00ba567c6..6bcbd7e1d 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -5,6 +5,7 @@
 #include <list>
 #include <set>
 #include <string>
+#include <limits>
 #include <map>
 #include <variant>
 #include <vector>
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index f6232d255..5b7abeb49 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -1,6 +1,7 @@
 #include "url.hh"
 #include "url-parts.hh"
 #include "util.hh"
+#include "split.hh"
 
 namespace nix {
 
@@ -136,4 +137,21 @@ bool ParsedURL::operator ==(const ParsedURL & other) const
         && fragment == other.fragment;
 }
 
+/**
+ * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme'
+ * into a tuple '(applicationScheme, transportScheme)'
+ *
+ * > parseUrlScheme("http") == ParsedUrlScheme{ {}, "http"}
+ * > parseUrlScheme("tarball+http") == ParsedUrlScheme{ {"tarball"}, "http"}
+ */
+ParsedUrlScheme parseUrlScheme(std::string_view scheme)
+{
+    auto application = splitPrefixTo(scheme, '+');
+    auto transport = scheme;
+    return ParsedUrlScheme {
+        .application = application,
+        .transport = transport,
+    };
+}
+
 }
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
index 6e77142e3..2a9fb34c1 100644
--- a/src/libutil/url.hh
+++ b/src/libutil/url.hh
@@ -27,4 +27,19 @@ std::map<std::string, std::string> decodeQuery(const std::string & query);
 
 ParsedURL parseURL(const std::string & url);
 
+/*
+ * Although that’s not really standardized anywhere, an number of tools
+ * use a scheme of the form 'x+y' in urls, where y is the “transport layer”
+ * scheme, and x is the “application layer” scheme.
+ *
+ * For example git uses `git+https` to designate remotes using a Git
+ * protocol over http.
+ */
+struct ParsedUrlScheme {
+    std::optional<std::string_view> application;
+    std::string_view transport;
+};
+
+ParsedUrlScheme parseUrlScheme(std::string_view scheme);
+
 }
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 656804007..993dc1cb6 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -2,6 +2,7 @@
 #include "sync.hh"
 #include "finally.hh"
 #include "serialise.hh"
+#include "cgroup.hh"
 
 #include <array>
 #include <cctype>
@@ -29,11 +30,14 @@
 
 #ifdef __APPLE__
 #include <sys/syscall.h>
+#include <mach-o/dyld.h>
 #endif
 
 #ifdef __linux__
 #include <sys/prctl.h>
 #include <sys/resource.h>
+
+#include <cmath>
 #endif
 
 
@@ -349,7 +353,7 @@ void readFile(const Path & path, Sink & sink)
 }
 
 
-void writeFile(const Path & path, std::string_view s, mode_t mode)
+void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
     if (!fd)
@@ -360,10 +364,16 @@ void writeFile(const Path & path, std::string_view s, mode_t mode)
         e.addTrace({}, "writing file '%1%'", path);
         throw;
     }
+    if (sync)
+        fd.fsync();
+    // Explicitly close to make sure exceptions are propagated.
+    fd.close();
+    if (sync)
+        syncParent(path);
 }
 
 
-void writeFile(const Path & path, Source & source, mode_t mode)
+void writeFile(const Path & path, Source & source, mode_t mode, bool sync)
 {
     AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
     if (!fd)
@@ -382,6 +392,20 @@ void writeFile(const Path & path, Source & source, mode_t mode)
         e.addTrace({}, "writing file '%1%'", path);
         throw;
     }
+    if (sync)
+        fd.fsync();
+    // Explicitly close to make sure exceptions are propagated.
+    fd.close();
+    if (sync)
+        syncParent(path);
+}
+
+void syncParent(const Path & path)
+{
+    AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0);
+    if (!fd)
+        throw SysError("opening file '%1%'", path);
+    fd.fsync();
 }
 
 std::string readLine(int fd)
@@ -504,61 +528,6 @@ void deletePath(const Path & path, uint64_t & bytesFreed)
 }
 
 
-static Path tempName(Path tmpRoot, const Path & prefix, bool includePid,
-    int & counter)
-{
-    tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true);
-    if (includePid)
-        return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str();
-    else
-        return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str();
-}
-
-
-Path createTempDir(const Path & tmpRoot, const Path & prefix,
-    bool includePid, bool useGlobalCounter, mode_t mode)
-{
-    static int globalCounter = 0;
-    int localCounter = 0;
-    int & counter(useGlobalCounter ? globalCounter : localCounter);
-
-    while (1) {
-        checkInterrupt();
-        Path tmpDir = tempName(tmpRoot, prefix, includePid, counter);
-        if (mkdir(tmpDir.c_str(), mode) == 0) {
-#if __FreeBSD__
-            /* Explicitly set the group of the directory.  This is to
-               work around around problems caused by BSD's group
-               ownership semantics (directories inherit the group of
-               the parent).  For instance, the group of /tmp on
-               FreeBSD is "wheel", so all directories created in /tmp
-               will be owned by "wheel"; but if the user is not in
-               "wheel", then "tar" will fail to unpack archives that
-               have the setgid bit set on directories. */
-            if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0)
-                throw SysError("setting group of directory '%1%'", tmpDir);
-#endif
-            return tmpDir;
-        }
-        if (errno != EEXIST)
-            throw SysError("creating directory '%1%'", tmpDir);
-    }
-}
-
-
-std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix)
-{
-    Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX");
-    // Strictly speaking, this is UB, but who cares...
-    // FIXME: use O_TMPFILE.
-    AutoCloseFD fd(mkstemp((char *) tmpl.c_str()));
-    if (!fd)
-        throw SysError("creating temporary file '%s'", tmpl);
-    closeOnExec(fd.get());
-    return {std::move(fd), tmpl};
-}
-
-
 std::string getUserName()
 {
     auto pw = getpwuid(geteuid());
@@ -573,7 +542,21 @@ Path getHome()
 {
     static Path homeDir = []()
     {
+        std::optional<std::string> unownedUserHomeDir = {};
         auto homeDir = getEnv("HOME");
+        if (homeDir) {
+            // Only use $HOME if doesn't exist or is owned by the current user.
+            struct stat st;
+            int result = stat(homeDir->c_str(), &st);
+            if (result != 0) {
+                if (errno != ENOENT) {
+                    warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno);
+                    homeDir.reset();
+                }
+            } else if (st.st_uid != geteuid()) {
+                unownedUserHomeDir.swap(homeDir);
+            }
+        }
         if (!homeDir) {
             std::vector<char> buf(16384);
             struct passwd pwbuf;
@@ -582,6 +565,9 @@ Path getHome()
                 || !pw || !pw->pw_dir || !pw->pw_dir[0])
                 throw Error("cannot determine user's home directory");
             homeDir = pw->pw_dir;
+            if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) {
+                warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir);
+            }
         }
         return *homeDir;
     }();
@@ -619,6 +605,27 @@ Path getDataDir()
 }
 
 
+std::optional<Path> getSelfExe()
+{
+    static auto cached = []() -> std::optional<Path>
+    {
+        #if __linux__
+        return readLink("/proc/self/exe");
+        #elif __APPLE__
+        char buf[1024];
+        uint32_t size = sizeof(buf);
+        if (_NSGetExecutablePath(buf, &size) == 0)
+            return buf;
+        else
+            return std::nullopt;
+        #else
+        return std::nullopt;
+        #endif
+    }();
+    return cached;
+}
+
+
 Paths createDirs(const Path & path)
 {
     Paths created;
@@ -642,44 +649,6 @@ Paths createDirs(const Path & path)
 }
 
 
-void createSymlink(const Path & target, const Path & link,
-    std::optional<time_t> mtime)
-{
-    if (symlink(target.c_str(), link.c_str()))
-        throw SysError("creating symlink from '%1%' to '%2%'", link, target);
-    if (mtime) {
-        struct timeval times[2];
-        times[0].tv_sec = *mtime;
-        times[0].tv_usec = 0;
-        times[1].tv_sec = *mtime;
-        times[1].tv_usec = 0;
-        if (lutimes(link.c_str(), times))
-            throw SysError("setting time of symlink '%s'", link);
-    }
-}
-
-
-void replaceSymlink(const Path & target, const Path & link,
-    std::optional<time_t> mtime)
-{
-    for (unsigned int n = 0; true; n++) {
-        Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link)));
-
-        try {
-            createSymlink(target, tmp, mtime);
-        } catch (SysError & e) {
-            if (e.errNo == EEXIST) continue;
-            throw;
-        }
-
-        if (rename(tmp.c_str(), link.c_str()) != 0)
-            throw SysError("renaming '%1%' to '%2%'", tmp, link);
-
-        break;
-    }
-}
-
-
 void readFull(int fd, char * buf, size_t count)
 {
     while (count) {
@@ -752,7 +721,32 @@ void drainFD(int fd, Sink & sink, bool block)
     }
 }
 
+//////////////////////////////////////////////////////////////////////
 
+unsigned int getMaxCPU()
+{
+    #if __linux__
+    try {
+        auto cgroupFS = getCgroupFS();
+        if (!cgroupFS) return 0;
+
+        auto cgroups = getCgroups("/proc/self/cgroup");
+        auto cgroup = cgroups[""];
+        if (cgroup == "") return 0;
+
+        auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max";
+
+        auto cpuMax = readFile(cpuFile);
+        auto cpuMaxParts = tokenizeString<std::vector<std::string>>(cpuMax, " \n");
+        auto quota = cpuMaxParts[0];
+        auto period = cpuMaxParts[1];
+        if (quota != "max")
+                return std::ceil(std::stoi(quota) / std::stof(period));
+    } catch (Error &) { ignoreException(lvlDebug); }
+    #endif
+
+    return 0;
+}
 
 //////////////////////////////////////////////////////////////////////
 
@@ -844,6 +838,20 @@ void AutoCloseFD::close()
     }
 }
 
+void AutoCloseFD::fsync()
+{
+  if (fd != -1) {
+      int result;
+#if __APPLE__
+      result = ::fcntl(fd, F_FULLFSYNC);
+#else
+      result = ::fsync(fd);
+#endif
+      if (result == -1)
+          throw SysError("fsync file descriptor %1%", fd);
+  }
+}
+
 
 AutoCloseFD::operator bool() const
 {
@@ -1082,7 +1090,7 @@ std::string runProgram(Path program, bool searchPath, const Strings & args,
     auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input});
 
     if (!statusOk(res.first))
-        throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first)));
+        throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first));
 
     return res.second;
 }
@@ -1210,7 +1218,7 @@ void runProgram2(const RunOptions & options)
     if (source) promise.get_future().get();
 
     if (status)
-        throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status)));
+        throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status));
 }
 
 
@@ -1396,7 +1404,7 @@ std::string shellEscape(const std::string_view s)
 }
 
 
-void ignoreException()
+void ignoreException(Verbosity lvl)
 {
     /* Make sure no exceptions leave this function.
        printError() also throws when remote is closed. */
@@ -1404,7 +1412,7 @@ void ignoreException()
         try {
             throw;
         } catch (std::exception & e) {
-            printError("error (ignored): %1%", e.what());
+            printMsg(lvl, "error (ignored): %1%", e.what());
         }
     } catch (...) { }
 }
@@ -1586,8 +1594,22 @@ std::string stripIndentation(std::string_view s)
 }
 
 
-//////////////////////////////////////////////////////////////////////
+std::pair<std::string_view, std::string_view> getLine(std::string_view s)
+{
+    auto newline = s.find('\n');
 
+    if (newline == s.npos) {
+        return {s, ""};
+    } else {
+        auto line = s.substr(0, newline);
+        if (!line.empty() && line[line.size() - 1] == '\r')
+            line = line.substr(0, line.size() - 1);
+        return {line, s.substr(newline + 1)};
+    }
+}
+
+
+//////////////////////////////////////////////////////////////////////
 
 static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}};
 
@@ -1819,7 +1841,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
     if (chmod(path.c_str(), mode) == -1)
         throw SysError("changing permissions on '%1%'", path);
 
-    if (listen(fdSocket.get(), 5) == -1)
+    if (listen(fdSocket.get(), 100) == -1)
         throw SysError("cannot listen on socket '%1%'", path);
 
     return fdSocket;
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index a1d0e0e6b..9b149de80 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -115,9 +115,12 @@ std::string readFile(const Path & path);
 void readFile(const Path & path, Sink & sink);
 
 /* Write a string to a file. */
-void writeFile(const Path & path, std::string_view s, mode_t mode = 0666);
+void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false);
 
-void writeFile(const Path & path, Source & source, mode_t mode = 0666);
+void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false);
+
+/* Flush a file's parent directory to disk */
+void syncParent(const Path & path);
 
 /* Read a line from a file descriptor. */
 std::string readLine(int fd);
@@ -149,10 +152,14 @@ std::vector<Path> getConfigDirs();
 /* Return $XDG_DATA_HOME or $HOME/.local/share. */
 Path getDataDir();
 
+/* Return the path of the current executable. */
+std::optional<Path> getSelfExe();
+
 /* Create a directory and all its parents, if necessary.  Returns the
    list of created directories, in order of creation. */
 Paths createDirs(const Path & path);
-inline Paths createDirs(PathView path) {
+inline Paths createDirs(PathView path)
+{
     return createDirs(Path(path));
 }
 
@@ -164,6 +171,17 @@ void createSymlink(const Path & target, const Path & link,
 void replaceSymlink(const Path & target, const Path & link,
     std::optional<time_t> mtime = {});
 
+void renameFile(const Path & src, const Path & dst);
+
+/**
+ * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst`
+ * are on a different filesystem.
+ *
+ * Beware that this might not be atomic because of the copy that happens behind
+ * the scenes
+ */
+void moveFile(const Path & src, const Path & dst);
+
 
 /* Wrappers arount read()/write() that read/write exactly the
    requested number of bytes. */
@@ -178,6 +196,9 @@ std::string drainFD(int fd, bool block = true, const size_t reserveSize=0);
 
 void drainFD(int fd, Sink & sink, bool block = true);
 
+/* If cgroups are active, attempt to calculate the number of CPUs available.
+   If cgroups are unavailable or if cpu.max is set to "max", return 0. */
+unsigned int getMaxCPU();
 
 /* Automatic cleanup of resources. */
 
@@ -213,6 +234,7 @@ public:
     explicit operator bool() const;
     int release();
     void close();
+    void fsync();
 };
 
 
@@ -488,6 +510,18 @@ std::optional<N> string2Float(const std::string_view s)
 }
 
 
+/* Convert a little-endian integer to host order. */
+template<typename T>
+T readLittleEndian(unsigned char * p)
+{
+    T x = 0;
+    for (size_t i = 0; i < sizeof(x); ++i, ++p) {
+        x |= ((T) *p) << (i * 8);
+    }
+    return x;
+}
+
+
 /* Return true iff `s' starts with `prefix'. */
 bool hasPrefix(std::string_view s, std::string_view prefix);
 
@@ -506,7 +540,7 @@ std::string shellEscape(const std::string_view s);
 
 /* Exception handling in destructors: print an error message, then
    ignore the exception. */
-void ignoreException();
+void ignoreException(Verbosity lvl = lvlError);
 
 
 
@@ -541,15 +575,39 @@ std::string base64Decode(std::string_view s);
 std::string stripIndentation(std::string_view s);
 
 
+/* Get the prefix of 's' up to and excluding the next line break (LF
+   optionally preceded by CR), and the remainder following the line
+   break. */
+std::pair<std::string_view, std::string_view> getLine(std::string_view s);
+
+
 /* Get a value for the specified key from an associate container. */
 template <class T>
-std::optional<typename T::mapped_type> get(const T & map, const typename T::key_type & key)
+const typename T::mapped_type * get(const T & map, const typename T::key_type & key)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return nullptr;
+    return &i->second;
+}
+
+template <class T>
+typename T::mapped_type * get(T & map, const typename T::key_type & key)
 {
     auto i = map.find(key);
-    if (i == map.end()) return {};
-    return std::optional<typename T::mapped_type>(i->second);
+    if (i == map.end()) return nullptr;
+    return &i->second;
 }
 
+/* Get a value for the specified key from an associate container, or a default value if the key isn't present. */
+template <class T>
+const typename T::mapped_type & getOr(T & map,
+    const typename T::key_type & key,
+    const typename T::mapped_type & defaultValue)
+{
+    auto i = map.find(key);
+    if (i == map.end()) return defaultValue;
+    return i->second;
+}
 
 /* Remove and return the first item from a container. */
 template <class T>
@@ -682,4 +740,28 @@ template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
 std::string showBytes(uint64_t bytes);
 
 
+/* Provide an addition operator between strings and string_views
+   inexplicably omitted from the standard library. */
+inline std::string operator + (const std::string & s1, std::string_view s2)
+{
+    auto s = s1;
+    s.append(s2);
+    return s;
+}
+
+inline std::string operator + (std::string && s, std::string_view s2)
+{
+    s.append(s2);
+    return std::move(s);
+}
+
+inline std::string operator + (std::string_view s1, const char * s2)
+{
+    std::string s;
+    s.reserve(s1.size() + strlen(s2));
+    s.append(s1);
+    s.append(s2);
+    return s;
+}
+
 }