Merge remote-tracking branch 'upstream/master' into path-info

24 files changed, 613 insertions, 444 deletions

diff --git a/src/libutil/ansicolor.hh b/src/libutil/ansicolor.hh
index ae741f867..38305e71c 100644
--- a/src/libutil/ansicolor.hh
+++ b/src/libutil/ansicolor.hh
@@ -9,7 +9,7 @@ namespace nix {
 #define ANSI_ITALIC "\e[3m"
 #define ANSI_RED "\e[31;1m"
 #define ANSI_GREEN "\e[32;1m"
-#define ANSI_YELLOW "\e[33;1m"
+#define ANSI_WARNING "\e[35;1m"
 #define ANSI_BLUE "\e[34;1m"
 #define ANSI_MAGENTA "\e[35;1m"
 #define ANSI_CYAN "\e[36;1m"
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index afed0670f..9df279faf 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -331,6 +331,7 @@ MultiCommand::MultiCommand(const Commands & commands_)
             if (i == commands.end())
                 throw UsageError("'%s' is not a recognised command", s);
             command = {s, i->second()};
+            command->second->parent = this;
         }}
     });
 
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index c08ba8abd..7521b3065 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -12,6 +12,8 @@ namespace nix {
 
 enum HashType : char;
 
+class MultiCommand;
+
 class Args
 {
 public:
@@ -89,6 +91,14 @@ protected:
               })
             , arity(1)
         { }
+
+        template<class I>
+        Handler(std::optional<I> * dest)
+            : fun([=](std::vector<std::string> ss) {
+                *dest = string2IntWithUnitPrefix<I>(ss[0]);
+            })
+            , arity(1)
+        { }
     };
 
     /* Options. */
@@ -169,11 +179,13 @@ public:
     virtual nlohmann::json toJSON();
 
     friend class MultiCommand;
+
+    MultiCommand * parent = nullptr;
 };
 
 /* A command is an argument parser that can be executed by calling its
    run() method. */
-struct Command : virtual Args
+struct Command : virtual public Args
 {
     friend class MultiCommand;
 
@@ -193,7 +205,7 @@ typedef std::map<std::string, std::function<ref<Command>()>> Commands;
 
 /* An argument parser that supports multiple subcommands,
    i.e. ‘<command> <subcommand>’. */
-class MultiCommand : virtual Args
+class MultiCommand : virtual public Args
 {
 public:
     Commands commands;
diff --git a/src/libutil/closure.hh b/src/libutil/closure.hh
new file mode 100644
index 000000000..779b9b2d5
--- /dev/null
+++ b/src/libutil/closure.hh
@@ -0,0 +1,69 @@
+#include <set>
+#include <future>
+#include "sync.hh"
+
+using std::set;
+
+namespace nix {
+
+template<typename T>
+using GetEdgesAsync = std::function<void(const T &, std::function<void(std::promise<set<T>> &)>)>;
+
+template<typename T>
+void computeClosure(
+    const set<T> startElts,
+    set<T> & res,
+    GetEdgesAsync<T> getEdgesAsync
+)
+{
+    struct State
+    {
+        size_t pending;
+        set<T> & res;
+        std::exception_ptr exc;
+    };
+
+    Sync<State> state_(State{0, res, 0});
+
+    std::function<void(const T &)> enqueue;
+
+    std::condition_variable done;
+
+    enqueue = [&](const T & current) -> void {
+        {
+            auto state(state_.lock());
+            if (state->exc) return;
+            if (!state->res.insert(current).second) return;
+            state->pending++;
+        }
+
+        getEdgesAsync(current, [&](std::promise<set<T>> & prom) {
+            try {
+                auto children = prom.get_future().get();
+                for (auto & child : children)
+                    enqueue(child);
+                {
+                    auto state(state_.lock());
+                    assert(state->pending);
+                    if (!--state->pending) done.notify_one();
+                }
+            } catch (...) {
+                auto state(state_.lock());
+                if (!state->exc) state->exc = std::current_exception();
+                assert(state->pending);
+                if (!--state->pending) done.notify_one();
+            };
+        });
+    };
+
+    for (auto & startElt : startElts)
+        enqueue(startElt);
+
+    {
+        auto state(state_.lock());
+        while (state->pending) state.wait(done);
+        if (state->exc) std::rethrow_exception(state->exc);
+    }
+}
+
+}
diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh
index 0315dc506..eecd5b819 100644
--- a/src/libutil/comparator.hh
+++ b/src/libutil/comparator.hh
@@ -25,6 +25,8 @@
     }
 #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
 #define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
+#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args)
 #define GENERATE_CMP(args...) \
     GENERATE_EQUAL(args) \
-    GENERATE_LEQ(args)
+    GENERATE_LEQ(args) \
+    GENERATE_NEQ(args)
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 986ba2976..d26f68fde 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -1,18 +1,17 @@
 #include "compression.hh"
+#include "tarfile.hh"
 #include "util.hh"
 #include "finally.hh"
 #include "logging.hh"
 
-#include <lzma.h>
-#include <bzlib.h>
+#include <archive.h>
+#include <archive_entry.h>
 #include <cstdio>
 #include <cstring>
 
 #include <brotli/decode.h>
 #include <brotli/encode.h>
 
-#include <zlib.h>
-
 #include <iostream>
 
 namespace nix {
@@ -27,7 +26,7 @@ struct ChunkedCompressionSink : CompressionSink
         const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
         while (!data.empty()) {
             size_t n = std::min(CHUNK_SIZE, data.size());
-            writeInternal(data);
+            writeInternal(data.substr(0, n));
             data.remove_prefix(n);
         }
     }
@@ -35,177 +34,101 @@ struct ChunkedCompressionSink : CompressionSink
     virtual void writeInternal(std::string_view data) = 0;
 };
 
-struct NoneSink : CompressionSink
-{
-    Sink & nextSink;
-    NoneSink(Sink & nextSink) : nextSink(nextSink) { }
-    void finish() override { flush(); }
-    void write(std::string_view data) override { nextSink(data); }
-};
-
-struct GzipDecompressionSink : CompressionSink
+struct ArchiveDecompressionSource : Source
 {
-    Sink & nextSink;
-    z_stream strm;
-    bool finished = false;
-    uint8_t outbuf[BUFSIZ];
-
-    GzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
-    {
-        strm.zalloc = Z_NULL;
-        strm.zfree = Z_NULL;
-        strm.opaque = Z_NULL;
-        strm.avail_in = 0;
-        strm.next_in = Z_NULL;
-        strm.next_out = outbuf;
-        strm.avail_out = sizeof(outbuf);
-
-        // Enable gzip and zlib decoding (+32) with 15 windowBits
-        int ret = inflateInit2(&strm,15+32);
-        if (ret != Z_OK)
-            throw CompressionError("unable to initialise gzip encoder");
-    }
-
-    ~GzipDecompressionSink()
-    {
-        inflateEnd(&strm);
-    }
-
-    void finish() override
-    {
-        CompressionSink::flush();
-        write({});
-    }
-
-    void write(std::string_view data) override
-    {
-        assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
-
-        strm.next_in = (Bytef *) data.data();
-        strm.avail_in = data.size();
-
-        while (!finished && (!data.data() || strm.avail_in)) {
-            checkInterrupt();
-
-            int ret = inflate(&strm,Z_SYNC_FLUSH);
-            if (ret != Z_OK && ret != Z_STREAM_END)
-                throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
-                    zError(ret), data.size(), strm.avail_in);
-
-            finished = ret == Z_STREAM_END;
-
-            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
-                strm.next_out = (Bytef *) outbuf;
-                strm.avail_out = sizeof(outbuf);
+    std::unique_ptr<TarArchive> archive = 0;
+    Source & src;
+    ArchiveDecompressionSource(Source & src) : src(src) {}
+    ~ArchiveDecompressionSource() override {}
+    size_t read(char * data, size_t len) override {
+        struct archive_entry * ae;
+        if (!archive) {
+            archive = std::make_unique<TarArchive>(src, true);
+            this->archive->check(archive_read_next_header(this->archive->archive, &ae),
+                "failed to read header (%s)");
+            if (archive_filter_count(this->archive->archive) < 2) {
+                throw CompressionError("input compression not recognized");
             }
         }
+        ssize_t result = archive_read_data(this->archive->archive, data, len);
+        if (result > 0) return result;
+        if (result == 0) {
+            throw EndOfFile("reached end of compressed file");
+        }
+        this->archive->check(result, "failed to read compressed data (%s)");
+        return result;
     }
 };
 
-struct XzDecompressionSink : CompressionSink
+struct ArchiveCompressionSink : CompressionSink
 {
     Sink & nextSink;
-    uint8_t outbuf[BUFSIZ];
-    lzma_stream strm = LZMA_STREAM_INIT;
-    bool finished = false;
-
-    XzDecompressionSink(Sink & nextSink) : nextSink(nextSink)
-    {
-        lzma_ret ret = lzma_stream_decoder(
-            &strm, UINT64_MAX, LZMA_CONCATENATED);
-        if (ret != LZMA_OK)
-            throw CompressionError("unable to initialise lzma decoder");
-
-        strm.next_out = outbuf;
-        strm.avail_out = sizeof(outbuf);
+    struct archive * archive;
+
+    ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel) : nextSink(nextSink) {
+        archive = archive_write_new();
+        if (!archive) throw Error("failed to initialize libarchive");
+        check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)");
+        check(archive_write_set_format_raw(archive));
+        if (format == "xz" && parallel) {
+            check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0"));
+        }
+        // disable internal buffering
+        check(archive_write_set_bytes_per_block(archive, 0));
+        // disable output padding
+        check(archive_write_set_bytes_in_last_block(archive, 1));
+        open();
     }
 
-    ~XzDecompressionSink()
+    ~ArchiveCompressionSink() override
     {
-        lzma_end(&strm);
+        if (archive) archive_write_free(archive);
     }
 
     void finish() override
     {
-        CompressionSink::flush();
-        write({});
+        flush();
+        check(archive_write_close(archive));
     }
 
-    void write(std::string_view data) override
+    void check(int err, const std::string & reason = "failed to compress (%s)")
     {
-        strm.next_in = (const unsigned char *) data.data();
-        strm.avail_in = data.size();
-
-        while (!finished && (!data.data() || strm.avail_in)) {
-            checkInterrupt();
-
-            lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
-            if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-                throw CompressionError("error %d while decompressing xz file", ret);
-
-            finished = ret == LZMA_STREAM_END;
-
-            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
-                strm.next_out = outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-        }
+        if (err == ARCHIVE_EOF)
+            throw EndOfFile("reached end of archive");
+        else if (err != ARCHIVE_OK)
+            throw Error(reason, archive_error_string(this->archive));
     }
-};
 
-struct BzipDecompressionSink : ChunkedCompressionSink
-{
-    Sink & nextSink;
-    bz_stream strm;
-    bool finished = false;
-
-    BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink)
+    void write(std::string_view data) override
     {
-        memset(&strm, 0, sizeof(strm));
-        int ret = BZ2_bzDecompressInit(&strm, 0, 0);
-        if (ret != BZ_OK)
-            throw CompressionError("unable to initialise bzip2 decoder");
-
-        strm.next_out = (char *) outbuf;
-        strm.avail_out = sizeof(outbuf);
+        ssize_t result = archive_write_data(archive, data.data(), data.length());
+        if (result <= 0) check(result);
     }
 
-    ~BzipDecompressionSink()
+private:
+    void open()
     {
-        BZ2_bzDecompressEnd(&strm);
+        check(archive_write_open(archive, this, nullptr, ArchiveCompressionSink::callback_write, nullptr));
+        auto ae = archive_entry_new();
+        archive_entry_set_filetype(ae, AE_IFREG);
+        check(archive_write_header(archive, ae));
+        archive_entry_free(ae);
     }
 
-    void finish() override
+    static ssize_t callback_write(struct archive * archive, void * _self, const void * buffer, size_t length)
     {
-        flush();
-        write({});
+        auto self = (ArchiveCompressionSink *) _self;
+        self->nextSink({(const char *) buffer, length});
+        return length;
     }
+};
 
-    void writeInternal(std::string_view data) override
-    {
-        assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
-
-        strm.next_in = (char *) data.data();
-        strm.avail_in = data.size();
-
-        while (strm.avail_in) {
-            checkInterrupt();
-
-            int ret = BZ2_bzDecompress(&strm);
-            if (ret != BZ_OK && ret != BZ_STREAM_END)
-                throw CompressionError("error while decompressing bzip2 file");
-
-            finished = ret == BZ_STREAM_END;
-
-            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
-                strm.next_out = (char *) outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-        }
-    }
+struct NoneSink : CompressionSink
+{
+    Sink & nextSink;
+    NoneSink(Sink & nextSink) : nextSink(nextSink) { }
+    void finish() override { flush(); }
+    void write(std::string_view data) override { nextSink(data); }
 };
 
 struct BrotliDecompressionSink : ChunkedCompressionSink
@@ -268,159 +191,24 @@ ref<std::string> decompress(const std::string & method, const std::string & in)
     return ssink.s;
 }
 
-ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
+std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink)
 {
     if (method == "none" || method == "")
-        return make_ref<NoneSink>(nextSink);
-    else if (method == "xz")
-        return make_ref<XzDecompressionSink>(nextSink);
-    else if (method == "bzip2")
-        return make_ref<BzipDecompressionSink>(nextSink);
-    else if (method == "gzip")
-        return make_ref<GzipDecompressionSink>(nextSink);
+        return std::make_unique<NoneSink>(nextSink);
     else if (method == "br")
-        return make_ref<BrotliDecompressionSink>(nextSink);
+        return std::make_unique<BrotliDecompressionSink>(nextSink);
     else
-        throw UnknownCompressionMethod("unknown compression method '%s'", method);
+        return sourceToSink([&](Source & source) {
+            auto decompressionSource = std::make_unique<ArchiveDecompressionSource>(source);
+            decompressionSource->drainInto(nextSink);
+        });
 }
 
-struct XzCompressionSink : CompressionSink
-{
-    Sink & nextSink;
-    uint8_t outbuf[BUFSIZ];
-    lzma_stream strm = LZMA_STREAM_INIT;
-    bool finished = false;
-
-    XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink)
-    {
-        lzma_ret ret;
-        bool done = false;
-
-        if (parallel) {
-#ifdef HAVE_LZMA_MT
-            lzma_mt mt_options = {};
-            mt_options.flags = 0;
-            mt_options.timeout = 300; // Using the same setting as the xz cmd line
-            mt_options.preset = LZMA_PRESET_DEFAULT;
-            mt_options.filters = NULL;
-            mt_options.check = LZMA_CHECK_CRC64;
-            mt_options.threads = lzma_cputhreads();
-            mt_options.block_size = 0;
-            if (mt_options.threads == 0)
-                mt_options.threads = 1;
-            // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
-            // number of threads.
-            ret = lzma_stream_encoder_mt(&strm, &mt_options);
-            done = true;
-#else
-            printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression");
-#endif
-        }
-
-        if (!done)
-            ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
-
-        if (ret != LZMA_OK)
-            throw CompressionError("unable to initialise lzma encoder");
-
-        // FIXME: apply the x86 BCJ filter?
-
-        strm.next_out = outbuf;
-        strm.avail_out = sizeof(outbuf);
-    }
-
-    ~XzCompressionSink()
-    {
-        lzma_end(&strm);
-    }
-
-    void finish() override
-    {
-        CompressionSink::flush();
-        write({});
-    }
-
-    void write(std::string_view data) override
-    {
-        strm.next_in = (const unsigned char *) data.data();
-        strm.avail_in = data.size();
-
-        while (!finished && (!data.data() || strm.avail_in)) {
-            checkInterrupt();
-
-            lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
-            if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-                throw CompressionError("error %d while compressing xz file", ret);
-
-            finished = ret == LZMA_STREAM_END;
-
-            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
-                strm.next_out = outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-        }
-    }
-};
-
-struct BzipCompressionSink : ChunkedCompressionSink
-{
-    Sink & nextSink;
-    bz_stream strm;
-    bool finished = false;
-
-    BzipCompressionSink(Sink & nextSink) : nextSink(nextSink)
-    {
-        memset(&strm, 0, sizeof(strm));
-        int ret = BZ2_bzCompressInit(&strm, 9, 0, 30);
-        if (ret != BZ_OK)
-            throw CompressionError("unable to initialise bzip2 encoder");
-
-        strm.next_out = (char *) outbuf;
-        strm.avail_out = sizeof(outbuf);
-    }
-
-    ~BzipCompressionSink()
-    {
-        BZ2_bzCompressEnd(&strm);
-    }
-
-    void finish() override
-    {
-        flush();
-        writeInternal({});
-    }
-
-    void writeInternal(std::string_view data) override
-    {
-        assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
-
-        strm.next_in = (char *) data.data();
-        strm.avail_in = data.size();
-
-        while (!finished && (!data.data() || strm.avail_in)) {
-            checkInterrupt();
-
-            int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH);
-            if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
-                throw CompressionError("error %d while compressing bzip2 file", ret);
-
-            finished = ret == BZ_STREAM_END;
-
-            if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
-                nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
-                strm.next_out = (char *) outbuf;
-                strm.avail_out = sizeof(outbuf);
-            }
-        }
-    }
-};
-
 struct BrotliCompressionSink : ChunkedCompressionSink
 {
     Sink & nextSink;
     uint8_t outbuf[BUFSIZ];
-    BrotliEncoderState *state;
+    BrotliEncoderState * state;
     bool finished = false;
 
     BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink)
@@ -471,12 +259,14 @@ struct BrotliCompressionSink : ChunkedCompressionSink
 
 ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
 {
+    std::vector<std::string> la_supports = {
+        "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"
+    };
+    if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) {
+        return make_ref<ArchiveCompressionSink>(nextSink, method, parallel);
+    }
     if (method == "none")
         return make_ref<NoneSink>(nextSink);
-    else if (method == "xz")
-        return make_ref<XzCompressionSink>(nextSink, parallel);
-    else if (method == "bzip2")
-        return make_ref<BzipCompressionSink>(nextSink);
     else if (method == "br")
         return make_ref<BrotliCompressionSink>(nextSink);
     else
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index dd666a4e1..338a0d9f2 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -8,14 +8,16 @@
 
 namespace nix {
 
-struct CompressionSink : BufferedSink
+struct CompressionSink : BufferedSink, FinishSink
 {
-    virtual void finish() = 0;
+    using BufferedSink::operator ();
+    using BufferedSink::write;
+    using FinishSink::finish;
 };
 
 ref<std::string> decompress(const std::string & method, const std::string & in);
 
-ref<CompressionSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
+std::unique_ptr<FinishSink> makeDecompressionSink(const std::string & method, Sink & nextSink);
 
 ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
 
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index bda07cd55..c247c7dae 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -152,6 +152,16 @@ nlohmann::json Config::toJSON()
     return res;
 }
 
+std::string Config::toKeyValue()
+{
+    auto res = std::string();
+    for (auto & s : _settings)
+        if (!s.second.isAlias) {
+            res += fmt("%s = %s\n", s.first, s.second.setting->to_string());
+        }
+    return res;
+}
+
 void Config::convertToArgs(Args & args, const std::string & category)
 {
     for (auto & s : _settings)
@@ -167,11 +177,6 @@ AbstractSetting::AbstractSetting(
 {
 }
 
-void AbstractSetting::setDefault(const std::string & str)
-{
-    if (!overridden) set(str);
-}
-
 nlohmann::json AbstractSetting::toJSON()
 {
     return nlohmann::json(toJSONObject());
@@ -385,6 +390,16 @@ nlohmann::json GlobalConfig::toJSON()
     return res;
 }
 
+std::string GlobalConfig::toKeyValue()
+{
+    std::string res;
+    std::map<std::string, Config::SettingInfo> settings;
+    globalConfig.getSettings(settings);
+    for (auto & s : settings)
+        res += fmt("%s = %s\n", s.first, s.second.value);
+    return res;
+}
+
 void GlobalConfig::convertToArgs(Args & args, const std::string & category)
 {
     for (auto & config : *configRegistrations)
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index bf81b4892..736810bf3 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -100,6 +100,12 @@ public:
     virtual nlohmann::json toJSON() = 0;
 
     /**
+     * Outputs all settings in a key-value pair format suitable to be used as
+     * `nix.conf`
+     */
+    virtual std::string toKeyValue() = 0;
+
+    /**
      * Converts settings to `Args` to be used on the command line interface
      * - args: args to write to
      * - category: category of the settings
@@ -169,6 +175,8 @@ public:
 
     nlohmann::json toJSON() override;
 
+    std::string toKeyValue() override;
+
     void convertToArgs(Args & args, const std::string & category) override;
 };
 
@@ -186,8 +194,6 @@ public:
 
     bool overridden = false;
 
-    void setDefault(const std::string & str);
-
 protected:
 
     AbstractSetting(
@@ -245,6 +251,7 @@ public:
     bool operator !=(const T & v2) const { return value != v2; }
     void operator =(const T & v) { assign(v); }
     virtual void assign(const T & v) { value = v; }
+    void setDefault(const T & v) { if (!overridden) value = v; }
 
     void set(const std::string & str, bool append = false) override;
 
@@ -330,6 +337,8 @@ struct GlobalConfig : public AbstractConfig
 
     nlohmann::json toJSON() override;
 
+    std::string toKeyValue() override;
+
     void convertToArgs(Args & args, const std::string & category) override;
 
     struct Register
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 0eea3455d..203d79087 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -185,15 +185,15 @@ void printAtPos(const ErrPos & pos, std::ostream & out)
     if (pos) {
         switch (pos.origin) {
             case foFile: {
-                out << fmt(ANSI_BLUE "at " ANSI_YELLOW "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos));
+                out << fmt(ANSI_BLUE "at " ANSI_WARNING "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos));
                 break;
             }
             case foString: {
-                out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«string»:%s" ANSI_NORMAL ":", showErrPos(pos));
+                out << fmt(ANSI_BLUE "at " ANSI_WARNING "«string»:%s" ANSI_NORMAL ":", showErrPos(pos));
                 break;
             }
             case foStdin: {
-                out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos));
+                out << fmt(ANSI_BLUE "at " ANSI_WARNING "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos));
                 break;
             }
             default:
@@ -232,7 +232,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
             break;
         }
         case Verbosity::lvlWarn: {
-            prefix = ANSI_YELLOW "warning";
+            prefix = ANSI_WARNING "warning";
             break;
         }
         case Verbosity::lvlInfo: {
@@ -252,7 +252,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s
             break;
         }
         case Verbosity::lvlDebug: {
-            prefix = ANSI_YELLOW "debug";
+            prefix = ANSI_WARNING "debug";
             break;
         }
         default:
diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh
index 85c0e9429..fd335b811 100644
--- a/src/libutil/fmt.hh
+++ b/src/libutil/fmt.hh
@@ -82,7 +82,7 @@ struct yellowtxt
 template <class T>
 std::ostream & operator<<(std::ostream & out, const yellowtxt<T> & y)
 {
-    return out << ANSI_YELLOW << y.value << ANSI_NORMAL;
+    return out << ANSI_WARNING << y.value << ANSI_NORMAL;
 }
 
 template <class T>
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index 5341c58e6..f880c0fc5 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -6,7 +6,7 @@ libutil_DIR := $(d)
 
 libutil_SOURCES := $(wildcard $(d)/*.cc)
 
-libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
+libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
 
 ifeq ($(HAVE_LIBCPUID), 1)
 	libutil_LDFLAGS += -lcpuid
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index d2e801175..f8a121ed1 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -27,7 +27,7 @@ Logger * logger = makeSimpleLogger(true);
 
 void Logger::warn(const std::string & msg)
 {
-    log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg);
+    log(lvlWarn, ANSI_WARNING "warning:" ANSI_NORMAL " " + msg);
 }
 
 void Logger::writeToStdout(std::string_view s)
@@ -46,7 +46,7 @@ public:
         : printBuildLogs(printBuildLogs)
     {
         systemd = getEnv("IN_SYSTEMD") == "1";
-        tty = isatty(STDERR_FILENO);
+        tty = shouldANSI();
     }
 
     bool isVerbose() override {
@@ -163,7 +163,7 @@ struct JSONLogger : Logger {
 
     void write(const nlohmann::json & json)
     {
-        prevLogger.log(lvlError, "@nix " + json.dump());
+        prevLogger.log(lvlError, "@nix " + json.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace));
     }
 
     void log(Verbosity lvl, const FormatOrString & fs) override
diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh
index 0be2a7e74..d6bf53bb8 100644
--- a/src/libutil/ref.hh
+++ b/src/libutil/ref.hh
@@ -73,6 +73,16 @@ public:
         return ref<T2>((std::shared_ptr<T2>) p);
     }
 
+    bool operator == (const ref<T> & other) const
+    {
+        return p == other.p;
+    }
+
+    bool operator != (const ref<T> & other) const
+    {
+        return p != other.p;
+    }
+
 private:
 
     template<typename T2, typename... Args>
@@ -89,4 +99,47 @@ make_ref(Args&&... args)
     return ref<T>(p);
 }
 
+
+/* A non-nullable pointer.
+   This is similar to a C++ "& reference", but mutable.
+   This is similar to ref<T> but backed by a regular pointer instead of a smart pointer.
+ */
+template<typename T>
+class ptr {
+private:
+    T * p;
+
+public:
+    ptr<T>(const ptr<T> & r)
+        : p(r.p)
+    { }
+
+    explicit ptr<T>(T * p)
+        : p(p)
+    {
+        if (!p)
+            throw std::invalid_argument("null pointer cast to ptr");
+    }
+
+    T* operator ->() const
+    {
+        return &*p;
+    }
+
+    T& operator *() const
+    {
+        return *p;
+    }
+
+    bool operator == (const ptr<T> & other) const
+    {
+        return p == other.p;
+    }
+
+    bool operator != (const ptr<T> & other) const
+    {
+        return p != other.p;
+    }
+};
+
 }
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index d1a16b6ba..16f3476c2 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -201,6 +201,62 @@ static DefaultStackAllocator defaultAllocatorSingleton;
 StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
 
 
+std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun)
+{
+    struct SourceToSink : FinishSink
+    {
+        typedef boost::coroutines2::coroutine<bool> coro_t;
+
+        std::function<void(Source &)> fun;
+        std::optional<coro_t::push_type> coro;
+
+        SourceToSink(std::function<void(Source &)> fun) : fun(fun)
+        {
+        }
+
+        std::string_view cur;
+
+        void operator () (std::string_view in) override
+        {
+            if (in.empty()) return;
+            cur = in;
+
+            if (!coro)
+                coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) {
+                    LambdaSource source([&](char *out, size_t out_len) {
+                        if (cur.empty()) {
+                            yield();
+                            if (yield.get()) {
+                                return (size_t)0;
+                            }
+                        }
+
+                        size_t n = std::min(cur.size(), out_len);
+                        memcpy(out, cur.data(), n);
+                        cur.remove_prefix(n);
+                        return n;
+                    });
+                    fun(source);
+                });
+
+            if (!*coro) { abort(); }
+
+            if (!cur.empty()) (*coro)(false);
+        }
+
+        void finish() override
+        {
+            if (!coro) return;
+            if (!*coro) abort();
+            (*coro)(true);
+            if (*coro) abort();
+        }
+    };
+
+    return std::make_unique<SourceToSink>(fun);
+}
+
+
 std::unique_ptr<Source> sinkToSource(
     std::function<void(Sink &)> fun,
     std::function<void()> eof)
@@ -212,7 +268,6 @@ std::unique_ptr<Source> sinkToSource(
         std::function<void(Sink &)> fun;
         std::function<void()> eof;
         std::optional<coro_t::pull_type> coro;
-        bool started = false;
 
         SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof)
             : fun(fun), eof(eof)
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index 5bbbc7ce3..0fe6e8332 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -25,6 +25,13 @@ struct NullSink : Sink
     { }
 };
 
+
+struct FinishSink : virtual Sink
+{
+    virtual void finish() = 0;
+};
+
+
 /* A buffered abstract sink. Warning: a BufferedSink should not be
    used from multiple threads concurrently. */
 struct BufferedSink : virtual Sink
@@ -281,6 +288,7 @@ struct ChainSource : Source
     size_t read(char * data, size_t len) override;
 };
 
+std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun);
 
 /* Convert a function that feeds data into a Sink into a Source. The
    Source executes the function as a coroutine. */
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 2da169ba7..50e691a3d 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -2,83 +2,76 @@
 #include <archive_entry.h>
 
 #include "serialise.hh"
+#include "tarfile.hh"
 
 namespace nix {
 
-struct TarArchive {
-    struct archive * archive;
-    Source * source;
-    std::vector<unsigned char> buffer;
-
-    void check(int err, const char * reason = "failed to extract archive: %s")
-    {
-        if (err == ARCHIVE_EOF)
-            throw EndOfFile("reached end of archive");
-        else if (err != ARCHIVE_OK)
-            throw Error(reason, archive_error_string(this->archive));
-    }
-
-    TarArchive(Source & source) : buffer(4096)
-    {
-        this->archive = archive_read_new();
-        this->source = &source;
-
-        archive_read_support_filter_all(archive);
-        archive_read_support_format_all(archive);
-        check(archive_read_open(archive,
-                (void *)this,
-                TarArchive::callback_open,
-                TarArchive::callback_read,
-                TarArchive::callback_close),
-            "failed to open archive: %s");
-    }
-
-    TarArchive(const Path & path)
-    {
-        this->archive = archive_read_new();
+static int callback_open(struct archive *, void * self)
+{
+    return ARCHIVE_OK;
+}
 
-        archive_read_support_filter_all(archive);
-        archive_read_support_format_all(archive);
-        check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
+static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
+{
+    auto self = (TarArchive *) _self;
+    *buffer = self->buffer.data();
+
+    try {
+        return self->source->read((char *) self->buffer.data(), 4096);
+    } catch (EndOfFile &) {
+        return 0;
+    } catch (std::exception & err) {
+        archive_set_error(archive, EIO, "Source threw exception: %s", err.what());
+        return -1;
     }
+}
 
-    TarArchive(const TarArchive &) = delete;
+static int callback_close(struct archive *, void * self)
+{
+    return ARCHIVE_OK;
+}
 
-    void close()
-    {
-        check(archive_read_close(archive), "failed to close archive: %s");
-    }
+void TarArchive::check(int err, const std::string & reason)
+{
+    if (err == ARCHIVE_EOF)
+        throw EndOfFile("reached end of archive");
+    else if (err != ARCHIVE_OK)
+        throw Error(reason, archive_error_string(this->archive));
+}
 
-    ~TarArchive()
-    {
-        if (this->archive) archive_read_free(this->archive);
-    }
+TarArchive::TarArchive(Source & source, bool raw)
+    : source(&source), buffer(4096)
+{
+    init();
+    if (!raw)
+        archive_read_support_format_all(archive);
+    else
+        archive_read_support_format_raw(archive);
+    check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
+}
 
-private:
+TarArchive::TarArchive(const Path & path)
+{
+    init();
+    archive_read_support_format_all(archive);
+    check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
+}
 
-    static int callback_open(struct archive *, void * self) {
-        return ARCHIVE_OK;
-    }
+void TarArchive::init()
+{
+    archive = archive_read_new();
+    archive_read_support_filter_all(archive);
+}
 
-    static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer)
-    {
-        auto self = (TarArchive *)_self;
-        *buffer = self->buffer.data();
-
-        try {
-            return self->source->read((char *) self->buffer.data(), 4096);
-        } catch (EndOfFile &) {
-            return 0;
-        } catch (std::exception & err) {
-            archive_set_error(archive, EIO, "source threw exception: %s", err.what());
-            return -1;
-        }
-    }
+void TarArchive::close()
+{
+    check(archive_read_close(this->archive), "Failed to close archive (%s)");
+}
 
-    static int callback_close(struct archive *, void * self) {
-        return ARCHIVE_OK;
-    }
-};
+TarArchive::~TarArchive()
+{
+    if (this->archive) archive_read_free(this->archive);
+}
 
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {
@@ -92,13 +85,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
         struct archive_entry * entry;
         int r = archive_read_next_header(archive.archive, &entry);
         if (r == ARCHIVE_EOF) break;
-        else if (r == ARCHIVE_WARN)
+        auto name = archive_entry_pathname(entry);
+        if (!name)
+            throw Error("cannot get archive member name: %s", archive_error_string(archive.archive));
+        if (r == ARCHIVE_WARN)
             warn(archive_error_string(archive.archive));
         else
             archive.check(r);
 
         archive_entry_set_pathname(entry,
-            (destDir + "/" + archive_entry_pathname(entry)).c_str());
+            (destDir + "/" + name).c_str());
 
         archive.check(archive_read_extract(archive.archive, entry, flags));
     }
diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh
index 89a024f1d..f107a7e2e 100644
--- a/src/libutil/tarfile.hh
+++ b/src/libutil/tarfile.hh
@@ -1,7 +1,29 @@
 #include "serialise.hh"
+#include <archive.h>
 
 namespace nix {
 
+struct TarArchive {
+    struct archive * archive;
+    Source * source;
+    std::vector<unsigned char> buffer;
+
+    void check(int err, const std::string & reason = "failed to extract archive (%s)");
+
+    TarArchive(Source & source, bool raw = false);
+
+    TarArchive(const Path & path);
+
+    // disable copy constructor
+    TarArchive(const TarArchive &) = delete;
+
+    void init();
+
+    void close();
+
+    ~TarArchive();
+};
+
 void unpackTarfile(Source & source, const Path & destDir);
 
 void unpackTarfile(const Path & tarFile, const Path & destDir);
diff --git a/src/libutil/tests/closure.cc b/src/libutil/tests/closure.cc
new file mode 100644
index 000000000..7597e7807
--- /dev/null
+++ b/src/libutil/tests/closure.cc
@@ -0,0 +1,70 @@
+#include "closure.hh"
+#include <gtest/gtest.h>
+
+namespace nix {
+
+using namespace std;
+
+map<string, set<string>> testGraph = {
+    { "A", { "B", "C", "G" } },
+    { "B", { "A" } }, // Loops back to A
+    { "C", { "F" } }, // Indirect reference
+    { "D", { "A" } }, // Not reachable, but has backreferences
+    { "E", {} }, // Just not reachable
+    { "F", {} },
+    { "G", { "G" } }, // Self reference
+};
+
+TEST(closure, correctClosure) {
+    set<string> aClosure;
+    set<string> expectedClosure = {"A", "B", "C", "F", "G"};
+    computeClosure<string>(
+        {"A"},
+        aClosure,
+        [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
+            promise<set<string>> promisedNodes;
+            promisedNodes.set_value(testGraph[currentNode]);
+            processEdges(promisedNodes);
+        }
+    );
+
+    ASSERT_EQ(aClosure, expectedClosure);
+}
+
+TEST(closure, properlyHandlesDirectExceptions) {
+    struct TestExn {};
+    set<string> aClosure;
+    EXPECT_THROW(
+        computeClosure<string>(
+            {"A"},
+            aClosure,
+            [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
+                throw TestExn();
+            }
+        ),
+        TestExn
+    );
+}
+
+TEST(closure, properlyHandlesExceptionsInPromise) {
+    struct TestExn {};
+    set<string> aClosure;
+    EXPECT_THROW(
+        computeClosure<string>(
+            {"A"},
+            aClosure,
+            [&](const string currentNode, function<void(promise<set<string>> &)> processEdges) {
+                promise<set<string>> promise;
+                try {
+                    throw TestExn();
+                } catch (...) {
+                    promise.set_exception(std::current_exception());
+                }
+                processEdges(promise);
+            }
+        ),
+        TestExn
+    );
+}
+
+}
diff --git a/src/libutil/tests/compression.cc b/src/libutil/tests/compression.cc
index 5b7a2c5b9..2efa3266b 100644
--- a/src/libutil/tests/compression.cc
+++ b/src/libutil/tests/compression.cc
@@ -17,6 +17,24 @@ namespace nix {
         ASSERT_EQ(*o, "this-is-a-test");
     }
 
+    TEST(decompress, decompressNoneCompressed) {
+        auto method = "none";
+        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        ref<std::string> o = decompress(method, str);
+
+        ASSERT_EQ(*o, str);
+    }
+
+    TEST(decompress, decompressEmptyCompressed) {
+        // Empty-method decompression used e.g. by S3 store
+        // (Content-Encoding == "").
+        auto method = "";
+        auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
+        ref<std::string> o = decompress(method, str);
+
+        ASSERT_EQ(*o, str);
+    }
+
     TEST(decompress, decompressXzCompressed) {
         auto method = "xz";
         auto str = "slfja;sljfklsa;jfklsjfkl;sdjfkl;sadjfkl;sdjf;lsdfjsadlf";
diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
index 6c649820b..2ffdc2e9b 100644
--- a/src/libutil/tests/logging.cc
+++ b/src/libutil/tests/logging.cc
@@ -336,7 +336,7 @@ namespace nix {
 
         ASSERT_STREQ(
             hintfmt("only one arg %1% %2%", "fulfilled").str().c_str(),
-            "only one arg " ANSI_YELLOW "fulfilled" ANSI_NORMAL " ");
+            "only one arg " ANSI_WARNING "fulfilled" ANSI_NORMAL " ");
 
     }
 
@@ -344,7 +344,7 @@ namespace nix {
 
         ASSERT_STREQ(
             hintfmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(),
-            "what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
+            "what about this " ANSI_WARNING "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL);
 
     }
 
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index c1bab866c..f6232d255 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -32,7 +32,7 @@ ParsedURL parseURL(const std::string & url)
         auto isFile = scheme.find("file") != std::string::npos;
 
         if (authority && *authority != "" && isFile)
-            throw Error("file:// URL '%s' has unexpected authority '%s'",
+            throw BadURL("file:// URL '%s' has unexpected authority '%s'",
                 url, *authority);
 
         if (isFile && path.empty())
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index dea9c74b7..bc841f425 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -4,16 +4,18 @@
 #include "finally.hh"
 #include "serialise.hh"
 
+#include <array>
 #include <cctype>
 #include <cerrno>
+#include <climits>
 #include <cstdio>
 #include <cstdlib>
 #include <cstring>
-#include <climits>
+#include <future>
 #include <iostream>
+#include <mutex>
 #include <sstream>
 #include <thread>
-#include <future>
 
 #include <fcntl.h>
 #include <grp.h>
@@ -32,6 +34,7 @@
 
 #ifdef __linux__
 #include <sys/prctl.h>
+#include <sys/resource.h>
 #endif
 
 
@@ -143,16 +146,21 @@ Path canonPath(const Path & path, bool resolveSymlinks)
             s += '/';
             while (i != end && *i != '/') s += *i++;
 
-            /* If s points to a symlink, resolve it and restart (since
-               the symlink target might contain new symlinks). */
+            /* If s points to a symlink, resolve it and continue from there */
             if (resolveSymlinks && isLink(s)) {
                 if (++followCount >= maxFollow)
                     throw Error("infinite symlink recursion in path '%1%'", path);
-                temp = absPath(readLink(s), dirOf(s))
-                    + string(i, end);
-                i = temp.begin(); /* restart */
+                temp = readLink(s) + string(i, end);
+                i = temp.begin();
                 end = temp.end();
-                s = "";
+                if (!temp.empty() && temp[0] == '/') {
+                    s.clear();  /* restart for symlinks pointing to absolute path */
+                } else {
+                    s = dirOf(s);
+                    if (s == "/") {  // we don’t want trailing slashes here, which dirOf only produces if s = /
+                        s.clear();
+                    }
+                }
             }
         }
     }
@@ -407,7 +415,7 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
         }
 
         int fd = openat(parentfd, path.c_str(), O_RDONLY);
-        if (!fd)
+        if (fd == -1)
             throw SysError("opening directory '%1%'", path);
         AutoCloseDir dir(fdopendir(fd));
         if (!dir)
@@ -429,12 +437,9 @@ static void _deletePath(const Path & path, uint64_t & bytesFreed)
     if (dir == "")
         dir = "/";
 
-    AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY));
+    AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)};
     if (!dirfd) {
-        // This really shouldn't fail silently, but it's left this way
-        // for backwards compatibility.
         if (errno == ENOENT) return;
-
         throw SysError("opening directory '%1%'", path);
     }
 
@@ -752,13 +757,13 @@ AutoCloseFD::AutoCloseFD() : fd{-1} {}
 AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
 
 
-AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd}
+AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
 {
     that.fd = -1;
 }
 
 
-AutoCloseFD& AutoCloseFD::operator =(AutoCloseFD&& that)
+AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
 {
     close();
     fd = that.fd;
@@ -789,6 +794,7 @@ void AutoCloseFD::close()
         if (::close(fd) == -1)
             /* This should never happen. */
             throw SysError("closing file descriptor %1%", fd);
+        fd = -1;
     }
 }
 
@@ -822,6 +828,12 @@ void Pipe::create()
 }
 
 
+void Pipe::close()
+{
+    readSide.close();
+    writeSide.close();
+}
+
 
 //////////////////////////////////////////////////////////////////////
 
@@ -1022,17 +1034,10 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss)
     return res;
 }
 
-// Output = "standard out" output stream
 string runProgram(Path program, bool searchPath, const Strings & args,
     const std::optional<std::string> & input)
 {
-    RunOptions opts(program, args);
-    opts.searchPath = searchPath;
-    // This allows you to refer to a program with a pathname relative to the
-    // PATH variable.
-    opts.input = input;
-
-    auto res = runProgram(opts);
+    auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input});
 
     if (!statusOk(res.first))
         throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first)));
@@ -1041,9 +1046,8 @@ string runProgram(Path program, bool searchPath, const Strings & args,
 }
 
 // Output = error code + "standard out" output stream
-std::pair<int, std::string> runProgram(const RunOptions & options_)
+std::pair<int, std::string> runProgram(RunOptions && options)
 {
-    RunOptions options(options_);
     StringSink sink;
     options.standardOut = &sink;
 
@@ -1109,7 +1113,7 @@ void runProgram2(const RunOptions & options)
         Strings args_(options.args);
         args_.push_front(options.program);
 
-        restoreSignals();
+        restoreProcessContext();
 
         if (options.searchPath)
             execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
@@ -1121,7 +1125,7 @@ void runProgram2(const RunOptions & options)
         throw SysError("executing '%1%'", options.program);
     }, processOptions);
 
-    out.writeSide = -1;
+    out.writeSide.close();
 
     std::thread writerThread;
 
@@ -1134,7 +1138,7 @@ void runProgram2(const RunOptions & options)
 
 
     if (source) {
-        in.readSide = -1;
+        in.readSide.close();
         writerThread = std::thread([&]() {
             try {
                 std::vector<char> buf(8 * 1024);
@@ -1151,7 +1155,7 @@ void runProgram2(const RunOptions & options)
             } catch (...) {
                 promise.set_exception(std::current_exception());
             }
-            in.writeSide = -1;
+            in.writeSide.close();
         });
     }
 
@@ -1359,6 +1363,12 @@ void ignoreException()
     }
 }
 
+bool shouldANSI()
+{
+    return isatty(STDERR_FILENO)
+        && getEnv("TERM").value_or("dumb") != "dumb"
+        && !getEnv("NO_COLOR").has_value();
+}
 
 std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width)
 {
@@ -1431,7 +1441,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in
 
 
 static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-
+static std::array<char, 256> base64DecodeChars;
 
 string base64Encode(std::string_view s)
 {
@@ -1456,15 +1466,12 @@ string base64Encode(std::string_view s)
 
 string base64Decode(std::string_view s)
 {
-    bool init = false;
-    char decode[256];
-    if (!init) {
-        // FIXME: not thread-safe.
-        memset(decode, -1, sizeof(decode));
+    static std::once_flag flag;
+    std::call_once(flag, [](){
+        base64DecodeChars = { (char)-1 };
         for (int i = 0; i < 64; i++)
-            decode[(int) base64Chars[i]] = i;
-        init = true;
-    }
+            base64DecodeChars[(int) base64Chars[i]] = i;
+    });
 
     string res;
     unsigned int d = 0, bits = 0;
@@ -1473,7 +1480,7 @@ string base64Decode(std::string_view s)
         if (c == '=') break;
         if (c == '\n') continue;
 
-        char digit = decode[(unsigned char) c];
+        char digit = base64DecodeChars[(unsigned char) c];
         if (digit == -1)
             throw Error("invalid character in Base64 string: '%c'", c);
 
@@ -1605,12 +1612,45 @@ void startSignalHandlerThread()
     std::thread(signalHandlerThread, set).detach();
 }
 
-void restoreSignals()
+static void restoreSignals()
 {
     if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
         throw SysError("restoring signals");
 }
 
+#if __linux__
+rlim_t savedStackSize = 0;
+#endif
+
+void setStackSize(size_t stackSize)
+{
+    #if __linux__
+    struct rlimit limit;
+    if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
+        savedStackSize = limit.rlim_cur;
+        limit.rlim_cur = stackSize;
+        setrlimit(RLIMIT_STACK, &limit);
+    }
+    #endif
+}
+
+void restoreProcessContext()
+{
+    restoreSignals();
+
+    restoreAffinity();
+
+    #if __linux__
+    if (savedStackSize) {
+        struct rlimit limit;
+        if (getrlimit(RLIMIT_STACK, &limit) == 0) {
+            limit.rlim_cur = savedStackSize;
+            setrlimit(RLIMIT_STACK, &limit);
+        }
+    }
+    #endif
+}
+
 /* RAII helper to automatically deregister a callback. */
 struct InterruptCallbackImpl : InterruptCallback
 {
@@ -1673,10 +1713,13 @@ string showBytes(uint64_t bytes)
 }
 
 
+// FIXME: move to libstore/build
 void commonChildInit(Pipe & logPipe)
 {
+    logger = makeSimpleLogger();
+
     const static string pathNullDevice = "/dev/null";
-    restoreSignals();
+    restoreProcessContext();
 
     /* Put the child in a separate session (and thus a separate
        process group) so that it has no controlling terminal (meaning
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index ad49c65b3..bee77b53f 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -188,7 +188,6 @@ public:
 class AutoCloseFD
 {
     int fd;
-    void close();
 public:
     AutoCloseFD();
     AutoCloseFD(int fd);
@@ -200,6 +199,7 @@ public:
     int get() const;
     explicit operator bool() const;
     int release();
+    void close();
 };
 
 
@@ -216,6 +216,7 @@ class Pipe
 public:
     AutoCloseFD readSide, writeSide;
     void create();
+    void close();
 };
 
 
@@ -275,30 +276,33 @@ string runProgram(Path program, bool searchPath = false,
 
 struct RunOptions
 {
+    Path program;
+    bool searchPath = true;
+    Strings args;
     std::optional<uid_t> uid;
     std::optional<uid_t> gid;
     std::optional<Path> chdir;
     std::optional<std::map<std::string, std::string>> environment;
-    Path program;
-    bool searchPath = true;
-    Strings args;
     std::optional<std::string> input;
     Source * standardIn = nullptr;
     Sink * standardOut = nullptr;
     bool mergeStderrToStdout = false;
-    bool _killStderr = false;
-
-    RunOptions(const Path & program, const Strings & args)
-        : program(program), args(args) { };
-
-    RunOptions & killStderr(bool v) { _killStderr = true; return *this; }
 };
 
-std::pair<int, std::string> runProgram(const RunOptions & options);
+std::pair<int, std::string> runProgram(RunOptions && options);
 
 void runProgram2(const RunOptions & options);
 
 
+/* Change the stack size. */
+void setStackSize(size_t stackSize);
+
+
+/* Restore the original inherited Unix process context (such as signal
+   masks, stack size, CPU affinity). */
+void restoreProcessContext();
+
+
 class ExecError : public Error
 {
 public:
@@ -472,6 +476,9 @@ constexpr char treeLast[] = "└───";
 constexpr char treeLine[] = "│   ";
 constexpr char treeNull[] = "    ";
 
+/* Determine whether ANSI escape sequences are appropriate for the
+   present output. */
+bool shouldANSI();
 
 /* Truncate a string to 'width' printable characters. If 'filterAll'
    is true, all ANSI escape sequences are filtered out. Otherwise,
@@ -512,9 +519,6 @@ class Callback;
    on the current thread (and thus any threads created by it). */
 void startSignalHandlerThread();
 
-/* Restore default signal handling. */
-void restoreSignals();
-
 struct InterruptCallback
 {
     virtual ~InterruptCallback() { };