Merge remote-tracking branch 'origin/master' into enum-FileIngestionMethod

13 files changed, 309 insertions, 397 deletions

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index be52e637e..272c4f1ae 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -7,7 +7,7 @@
 #include "affinity.hh"
 #include "builtins.hh"
 #include "builtins/buildenv.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "finally.hh"
 #include "compression.hh"
 #include "json.hh"
@@ -33,7 +33,6 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/utsname.h>
-#include <sys/select.h>
 #include <sys/resource.h>
 #include <sys/socket.h>
 #include <sys/un.h>
@@ -43,6 +42,7 @@
 #include <errno.h>
 #include <cstring>
 #include <termios.h>
+#include <poll.h>
 
 #include <pwd.h>
 #include <grp.h>
@@ -361,7 +361,7 @@ public:
     {
         actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
         actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
-        act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize);
+        act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
         act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
     }
 };
@@ -507,9 +507,10 @@ private:
     Path fnUserLock;
     AutoCloseFD fdUserLock;
 
+    bool isEnabled = false;
     string user;
-    uid_t uid;
-    gid_t gid;
+    uid_t uid = 0;
+    gid_t gid = 0;
     std::vector<gid_t> supplementaryGIDs;
 
 public:
@@ -522,7 +523,9 @@ public:
     uid_t getGID() { assert(gid); return gid; }
     std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
 
-    bool enabled() { return uid != 0; }
+    bool findFreeUser();
+
+    bool enabled() { return isEnabled; }
 
 };
 
@@ -530,6 +533,11 @@ public:
 UserLock::UserLock()
 {
     assert(settings.buildUsersGroup != "");
+    createDirs(settings.nixStateDir + "/userpool");
+}
+
+bool UserLock::findFreeUser() {
+    if (enabled()) return true;
 
     /* Get the members of the build-users-group. */
     struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
@@ -559,7 +567,6 @@ UserLock::UserLock()
             throw Error(format("the user '%1%' in the group '%2%' does not exist")
                 % i % settings.buildUsersGroup);
 
-        createDirs(settings.nixStateDir + "/userpool");
 
         fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
 
@@ -590,16 +597,13 @@ UserLock::UserLock()
             supplementaryGIDs.resize(ngroups);
 #endif
 
-            return;
+            isEnabled = true;
+            return true;
         }
     }
-
-    throw Error(format("all build users are currently in use; "
-        "consider creating additional users and adding them to the '%1%' group")
-        % settings.buildUsersGroup);
+    return false;
 }
 
-
 void UserLock::kill()
 {
     killUser(uid);
@@ -928,6 +932,7 @@ private:
     void closureRepaired();
     void inputsRealised();
     void tryToBuild();
+    void tryLocalBuild();
     void buildDone();
 
     /* Is the build hook willing to perform the build? */
@@ -999,6 +1004,8 @@ private:
         Goal::amDone(result);
     }
 
+    void started();
+
     void done(BuildResult::Status status, const string & msg = "");
 
     StorePathSet exportReferences(const StorePathSet & storePaths);
@@ -1386,6 +1393,19 @@ void DerivationGoal::inputsRealised()
     result = BuildResult();
 }
 
+void DerivationGoal::started() {
+    auto msg = fmt(
+        buildMode == bmRepair ? "repairing outputs of '%s'" :
+        buildMode == bmCheck ? "checking outputs of '%s'" :
+        nrRounds > 1 ? "building '%s' (round %d/%d)" :
+        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
+    fmt("building '%s'", worker.store.printStorePath(drvPath));
+    if (hook) msg += fmt(" on '%s'", machineName);
+    act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
+        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
+    mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
+    worker.updateProgress();
+}
 
 void DerivationGoal::tryToBuild()
 {
@@ -1437,20 +1457,6 @@ void DerivationGoal::tryToBuild()
        supported for local builds. */
     bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
 
-    auto started = [&]() {
-        auto msg = fmt(
-            buildMode == bmRepair ? "repairing outputs of '%s'" :
-            buildMode == bmCheck ? "checking outputs of '%s'" :
-            nrRounds > 1 ? "building '%s' (round %d/%d)" :
-            "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
-        fmt("building '%s'", worker.store.printStorePath(drvPath));
-        if (hook) msg += fmt(" on '%s'", machineName);
-        act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-            Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
-        mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
-        worker.updateProgress();
-    };
-
     /* Is the build hook willing to accept this job? */
     if (!buildLocally) {
         switch (tryBuildHook()) {
@@ -1483,6 +1489,34 @@ void DerivationGoal::tryToBuild()
         return;
     }
 
+    state = &DerivationGoal::tryLocalBuild;
+    worker.wakeUp(shared_from_this());
+}
+
+void DerivationGoal::tryLocalBuild() {
+
+    /* If `build-users-group' is not empty, then we have to build as
+       one of the members of that group. */
+    if (settings.buildUsersGroup != "" && getuid() == 0) {
+#if defined(__linux__) || defined(__APPLE__)
+        if (!buildUser) buildUser = std::make_unique<UserLock>();
+
+        if (buildUser->findFreeUser()) {
+            /* Make sure that no other processes are executing under this
+               uid. */
+            buildUser->kill();
+        } else {
+            debug("waiting for build users");
+            worker.waitForAWhile(shared_from_this());
+            return;
+        }
+#else
+        /* Don't know how to block the creation of setuid/setgid
+           binaries on this platform. */
+        throw Error("build users are not supported on this platform for security reasons");
+#endif
+    }
+
     try {
 
         /* Okay, we have to build. */
@@ -1680,6 +1714,7 @@ void DerivationGoal::buildDone()
         }
 
         if (buildMode == bmCheck) {
+            deleteTmpDir(true);
             done(BuildResult::Built);
             return;
         }
@@ -1942,22 +1977,6 @@ void DerivationGoal::startBuilder()
         #endif
     }
 
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        buildUser = std::make_unique<UserLock>();
-
-        /* Make sure that no other processes are executing under this
-           uid. */
-        buildUser->kill();
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
-    }
-
     /* Create a temporary directory where the build will take
        place. */
     tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
@@ -2161,7 +2180,7 @@ void DerivationGoal::startBuilder()
     if (needsHashRewrite()) {
 
         if (pathExists(homeDir))
-            throw Error(format("directory '%1%' exists; please remove it") % homeDir);
+            throw Error(format("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing") % homeDir);
 
         /* We're not doing a chroot build, but we have some valid
            output paths.  Since we can't just overwrite or delete
@@ -2249,10 +2268,13 @@ void DerivationGoal::startBuilder()
 
         if (chown(slaveName.c_str(), buildUser->getUID(), 0))
             throw SysError("changing owner of pseudoterminal slave");
-    } else {
+    }
+#if __APPLE__
+    else {
         if (grantpt(builderOut.readSide.get()))
             throw SysError("granting access to pseudoterminal slave");
     }
+#endif
 
     #if 0
     // Mount the pt in the sandbox so that the "tty" command works.
@@ -2465,7 +2487,7 @@ void DerivationGoal::initTmpDir() {
                 auto hash = hashString(htSHA256, i.first);
                 string fn = ".attr-" + hash.to_string(Base32, false);
                 Path p = tmpDir + "/" + fn;
-                writeFile(p, i.second);
+                writeFile(p, rewriteStrings(i.second, inputRewrites));
                 chownToBuilder(p);
                 env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
             }
@@ -3151,7 +3173,7 @@ void DerivationGoal::runChild()
                 // Only use nss functions to resolve hosts and
                 // services. Don’t use it for anything else that may
                 // be configured for this system. This limits the
-                // potential impurities introduced in fixed outputs.
+                // potential impurities introduced in fixed-outputs.
                 writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
 
                 ss.push_back("/etc/services");
@@ -3536,6 +3558,29 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
 }
 
 
+static void moveCheckToStore(const Path & src, const Path & dst)
+{
+    /* For the rename of directory to succeed, we must be running as root or
+       the directory must be made temporarily writable (to update the
+       directory's parent link ".."). */
+    struct stat st;
+    if (lstat(src.c_str(), &st) == -1) {
+        throw SysError(format("getting attributes of path '%1%'") % src);
+    }
+
+    bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
+
+    if (changePerm)
+        chmod_(src, st.st_mode | S_IWUSR);
+
+    if (rename(src.c_str(), dst.c_str()))
+        throw SysError(format("renaming '%1%' to '%2%'") % src % dst);
+
+    if (changePerm)
+        chmod_(dst, st.st_mode);
+}
+
+
 void DerivationGoal::registerOutputs()
 {
     /* When using a build hook, the build hook can register the output
@@ -3654,7 +3699,8 @@ void DerivationGoal::registerOutputs()
                 /* The output path should be a regular file without execute permission. */
                 if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
                     throw BuildError(
-                        format("output path '%1%' should be a non-executable regular file") % path);
+                        format("output path '%1%' should be a non-executable regular file "
+                               "since recursive hashing is not enabled (outputHashMode=flat)") % path);
             }
 
             /* Check the hash. In hash mode, move the path produced by
@@ -3716,8 +3762,7 @@ void DerivationGoal::registerOutputs()
                 if (settings.runDiffHook || settings.keepFailed) {
                     Path dst = worker.store.toRealPath(path + checkSuffix);
                     deletePath(dst);
-                    if (rename(actualPath.c_str(), dst.c_str()))
-                        throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
+                    moveCheckToStore(actualPath, dst);
 
                     handleDiffHook(
                         buildUser ? buildUser->getUID() : getuid(),
@@ -3725,10 +3770,10 @@ void DerivationGoal::registerOutputs()
                         path, dst, worker.store.printStorePath(drvPath), tmpDir);
 
                     throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
-                        worker.store.printStorePath(drvPath), path, dst);
+                        worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst);
                 } else
                     throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
-                        worker.store.printStorePath(drvPath), path);
+                        worker.store.printStorePath(drvPath), worker.store.toRealPath(path));
             }
 
             /* Since we verified the build, it's now ultimately trusted. */
@@ -4770,8 +4815,7 @@ void Worker::waitForInput()
        terminated. */
 
     bool useTimeout = false;
-    struct timeval timeout;
-    timeout.tv_usec = 0;
+    long timeout = 0;
     auto before = steady_time_point::clock::now();
 
     /* If we're monitoring for silence on stdout/stderr, or if there
@@ -4789,7 +4833,7 @@ void Worker::waitForInput()
             nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
     }
     if (nearest != steady_time_point::max()) {
-        timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
+        timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
         useTimeout = true;
     }
 
@@ -4798,32 +4842,30 @@ void Worker::waitForInput()
     if (!waitingForAWhile.empty()) {
         useTimeout = true;
         if (lastWokenUp == steady_time_point::min())
-            printError("waiting for locks or build slots...");
+            printError("waiting for locks, build slots or build users...");
         if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
-        timeout.tv_sec = std::max(1L,
+        timeout = std::max(1L,
             (long) std::chrono::duration_cast<std::chrono::seconds>(
                 lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
     } else lastWokenUp = steady_time_point::min();
 
     if (useTimeout)
-        vomit("sleeping %d seconds", timeout.tv_sec);
+        vomit("sleeping %d seconds", timeout);
 
     /* Use select() to wait for the input side of any logger pipe to
        become `available'.  Note that `available' (i.e., non-blocking)
        includes EOF. */
-    fd_set fds;
-    FD_ZERO(&fds);
-    int fdMax = 0;
+    std::vector<struct pollfd> pollStatus;
+    std::map <int, int> fdToPollStatus;
     for (auto & i : children) {
         for (auto & j : i.fds) {
-            if (j >= FD_SETSIZE)
-                throw Error("reached FD_SETSIZE limit");
-            FD_SET(j, &fds);
-            if (j >= fdMax) fdMax = j + 1;
+            pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
+            fdToPollStatus[j] = pollStatus.size() - 1;
         }
     }
 
-    if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) {
+    if (poll(pollStatus.data(), pollStatus.size(),
+            useTimeout ? timeout * 1000 : -1) == -1) {
         if (errno == EINTR) return;
         throw SysError("waiting for input");
     }
@@ -4844,7 +4886,7 @@ void Worker::waitForInput()
         set<int> fds2(j->fds);
         std::vector<unsigned char> buffer(4096);
         for (auto & k : fds2) {
-            if (FD_ISSET(k, &fds)) {
+            if (pollStatus.at(fdToPollStatus.at(k)).revents) {
                 ssize_t rd = read(k, buffer.data(), buffer.size());
                 // FIXME: is there a cleaner way to handle pt close
                 // than EIO? Is this even standard?
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index f6ae5d2e6..486babf14 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -1,5 +1,5 @@
 #include "builtins.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "store-api.hh"
 #include "archive.hh"
 #include "compression.hh"
@@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
     auto mainUrl = getAttr("url");
     bool unpack = get(drv.env, "unpack").value_or("") == "1";
 
-    /* Note: have to use a fresh downloader here because we're in
+    /* Note: have to use a fresh fileTransfer here because we're in
        a forked process. */
-    auto downloader = makeDownloader();
+    auto fileTransfer = makeFileTransfer();
 
     auto fetch = [&](const std::string & url) {
 
@@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
 
             /* No need to do TLS verification, because we check the hash of
                the result anyway. */
-            DownloadRequest request(url);
+            FileTransferRequest request(url);
             request.verifyTLS = false;
             request.decompress = false;
 
             auto decompressor = makeDecompressionSink(
                 unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
-            downloader->download(std::move(request), *decompressor);
+            fileTransfer->download(std::move(request), *decompressor);
             decompressor->finish();
         });
 
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 5934c1912..c68e7b16b 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -378,7 +378,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
         if (h == drvHashes.end()) {
             assert(store.isValidPath(i.first));
             h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store,
-                readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first;
+                readDerivation(store, store.toRealPath(i.first)), false)).first;
         }
         inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
     }
diff --git a/src/libstore/download.cc b/src/libstore/filetransfer.cc
index 8a9b65899..e9684b3d4 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/filetransfer.cc
@@ -1,14 +1,10 @@
-#include "download.hh"
+#include "filetransfer.hh"
 #include "util.hh"
 #include "globals.hh"
-#include "hash.hh"
 #include "store-api.hh"
-#include "archive.hh"
 #include "s3.hh"
 #include "compression.hh"
-#include "pathlocks.hh"
 #include "finally.hh"
-#include "tarfile.hh"
 
 #ifdef ENABLE_S3
 #include <aws/core/client/ClientConfiguration.h>
@@ -31,13 +27,9 @@ using namespace std::string_literals;
 
 namespace nix {
 
-DownloadSettings downloadSettings;
+FileTransferSettings fileTransferSettings;
 
-static GlobalConfig::Register r1(&downloadSettings);
-
-CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
-    : uri(uri), ttl(settings.tarballTtl)
-{ }
+static GlobalConfig::Register r1(&fileTransferSettings);
 
 std::string resolveUri(const std::string & uri)
 {
@@ -47,21 +39,21 @@ std::string resolveUri(const std::string & uri)
         return uri;
 }
 
-struct CurlDownloader : public Downloader
+struct curlFileTransfer : public FileTransfer
 {
     CURLM * curlm = 0;
 
     std::random_device rd;
     std::mt19937 mt19937;
 
-    struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
+    struct TransferItem : public std::enable_shared_from_this<TransferItem>
     {
-        CurlDownloader & downloader;
-        DownloadRequest request;
-        DownloadResult result;
+        curlFileTransfer & fileTransfer;
+        FileTransferRequest request;
+        FileTransferResult result;
         Activity act;
         bool done = false; // whether either the success or failure function has been called
-        Callback<DownloadResult> callback;
+        Callback<FileTransferResult> callback;
         CURL * req = 0;
         bool active = false; // whether the handle has been added to the multi object
         std::string status;
@@ -80,19 +72,26 @@ struct CurlDownloader : public Downloader
 
         curl_off_t writtenToSink = 0;
 
-        DownloadItem(CurlDownloader & downloader,
-            const DownloadRequest & request,
-            Callback<DownloadResult> && callback)
-            : downloader(downloader)
+        TransferItem(curlFileTransfer & fileTransfer,
+            const FileTransferRequest & request,
+            Callback<FileTransferResult> && callback)
+            : fileTransfer(fileTransfer)
             , request(request)
-            , act(*logger, lvlTalkative, actDownload,
+            , act(*logger, lvlTalkative, actFileTransfer,
                 fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
                 {request.uri}, request.parentAct)
             , callback(std::move(callback))
             , finalSink([this](const unsigned char * data, size_t len) {
                 if (this->request.dataCallback) {
-                    writtenToSink += len;
-                    this->request.dataCallback((char *) data, len);
+                    long httpStatus = 0;
+                    curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+
+                    /* Only write data to the sink if this is a
+                       successful response. */
+                    if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
+                        writtenToSink += len;
+                        this->request.dataCallback((char *) data, len);
+                    }
                 } else
                     this->result.data->append((char *) data, len);
               })
@@ -103,17 +102,17 @@ struct CurlDownloader : public Downloader
                 requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
         }
 
-        ~DownloadItem()
+        ~TransferItem()
         {
             if (req) {
                 if (active)
-                    curl_multi_remove_handle(downloader.curlm, req);
+                    curl_multi_remove_handle(fileTransfer.curlm, req);
                 curl_easy_cleanup(req);
             }
             if (requestHeaders) curl_slist_free_all(requestHeaders);
             try {
                 if (!done)
-                    fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri));
+                    fail(FileTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri));
             } catch (...) {
                 ignoreException();
             }
@@ -157,7 +156,7 @@ struct CurlDownloader : public Downloader
 
         static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
         {
-            return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb);
+            return ((TransferItem *) userp)->writeCallback(contents, size, nmemb);
         }
 
         size_t headerCallback(void * contents, size_t size, size_t nmemb)
@@ -199,7 +198,7 @@ struct CurlDownloader : public Downloader
 
         static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
         {
-            return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb);
+            return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
         }
 
         int progressCallback(double dltotal, double dlnow)
@@ -214,7 +213,7 @@ struct CurlDownloader : public Downloader
 
         static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
         {
-            return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow);
+            return ((TransferItem *) userp)->progressCallback(dltotal, dlnow);
         }
 
         static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
@@ -238,7 +237,7 @@ struct CurlDownloader : public Downloader
 
         static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
         {
-            return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
+            return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
         }
 
         void init()
@@ -249,7 +248,7 @@ struct CurlDownloader : public Downloader
 
             if (verbosity >= lvlVomit) {
                 curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
-                curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback);
+                curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback);
             }
 
             curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
@@ -258,19 +257,19 @@ struct CurlDownloader : public Downloader
             curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
             curl_easy_setopt(req, CURLOPT_USERAGENT,
                 ("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
-                    (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str());
+                    (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str());
             #if LIBCURL_VERSION_NUM >= 0x072b00
             curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
             #endif
             #if LIBCURL_VERSION_NUM >= 0x072f00
-            if (downloadSettings.enableHttp2)
+            if (fileTransferSettings.enableHttp2)
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
             else
                 curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
             #endif
-            curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
+            curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
-            curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper);
+            curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
             curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
 
             curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
@@ -298,10 +297,10 @@ struct CurlDownloader : public Downloader
                 curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
             }
 
-            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
+            curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
 
             curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
-            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
+            curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
 
             /* If no file exist in the specified path, curl continues to work
                anyway as if netrc support was disabled. */
@@ -390,6 +389,7 @@ struct CurlDownloader : public Downloader
                         case CURLE_SSL_CACERT_BADFILE:
                         case CURLE_TOO_MANY_REDIRECTS:
                         case CURLE_WRITE_ERROR:
+                        case CURLE_UNSUPPORTED_PROTOCOL:
                             err = Misc;
                             break;
                         default: // Shut up warnings
@@ -401,14 +401,14 @@ struct CurlDownloader : public Downloader
 
                 auto exc =
                     code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
-                    ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
+                    ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
                     : httpStatus != 0
-                    ? DownloadError(err,
+                    ? FileTransferError(err,
                         fmt("unable to %s '%s': HTTP error %d",
                             request.verb(), request.uri, httpStatus)
                         + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
                         )
-                    : DownloadError(err,
+                    : FileTransferError(err,
                         fmt("unable to %s '%s': %s (%d)",
                             request.verb(), request.uri, curl_easy_strerror(code), code));
 
@@ -422,13 +422,13 @@ struct CurlDownloader : public Downloader
                         || writtenToSink == 0
                         || (acceptRanges && encoding.empty())))
                 {
-                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
+                    int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
                     if (writtenToSink)
                         warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
                     else
                         warn("%s; retrying in %d ms", exc.what(), ms);
                     embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
-                    downloader.enqueueItem(shared_from_this());
+                    fileTransfer.enqueueItem(shared_from_this());
                 }
                 else
                     fail(exc);
@@ -439,12 +439,12 @@ struct CurlDownloader : public Downloader
     struct State
     {
         struct EmbargoComparator {
-            bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) {
+            bool operator() (const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2) {
                 return i1->embargo > i2->embargo;
             }
         };
         bool quit = false;
-        std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming;
+        std::priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator> incoming;
     };
 
     Sync<State> state_;
@@ -456,7 +456,7 @@ struct CurlDownloader : public Downloader
 
     std::thread workerThread;
 
-    CurlDownloader()
+    curlFileTransfer()
         : mt19937(rd())
     {
         static std::once_flag globalInit;
@@ -469,7 +469,7 @@ struct CurlDownloader : public Downloader
         #endif
         #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
         curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
-            downloadSettings.httpConnections.get());
+            fileTransferSettings.httpConnections.get());
         #endif
 
         wakeupPipe.create();
@@ -478,7 +478,7 @@ struct CurlDownloader : public Downloader
         workerThread = std::thread([&]() { workerThreadEntry(); });
     }
 
-    ~CurlDownloader()
+    ~curlFileTransfer()
     {
         stopWorkerThread();
 
@@ -504,7 +504,7 @@ struct CurlDownloader : public Downloader
             stopWorkerThread();
         });
 
-        std::map<CURL *, std::shared_ptr<DownloadItem>> items;
+        std::map<CURL *, std::shared_ptr<TransferItem>> items;
 
         bool quit = false;
 
@@ -561,7 +561,7 @@ struct CurlDownloader : public Downloader
                     throw SysError("reading curl wakeup socket");
             }
 
-            std::vector<std::shared_ptr<DownloadItem>> incoming;
+            std::vector<std::shared_ptr<TransferItem>> incoming;
             auto now = std::chrono::steady_clock::now();
 
             {
@@ -609,7 +609,7 @@ struct CurlDownloader : public Downloader
         }
     }
 
-    void enqueueItem(std::shared_ptr<DownloadItem> item)
+    void enqueueItem(std::shared_ptr<TransferItem> item)
     {
         if (item->request.data
             && !hasPrefix(item->request.uri, "http://")
@@ -641,8 +641,8 @@ struct CurlDownloader : public Downloader
     }
 #endif
 
-    void enqueueDownload(const DownloadRequest & request,
-        Callback<DownloadResult> callback) override
+    void enqueueFileTransfer(const FileTransferRequest & request,
+        Callback<FileTransferResult> callback) override
     {
         /* Ugly hack to support s3:// URIs. */
         if (hasPrefix(request.uri, "s3://")) {
@@ -660,9 +660,9 @@ struct CurlDownloader : public Downloader
 
                 // FIXME: implement ETag
                 auto s3Res = s3Helper.getObject(bucketName, key);
-                DownloadResult res;
+                FileTransferResult res;
                 if (!s3Res.data)
-                    throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
+                    throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
                 res.data = s3Res.data;
                 callback(std::move(res));
 #else
@@ -672,26 +672,26 @@ struct CurlDownloader : public Downloader
             return;
         }
 
-        enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback)));
+        enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
     }
 };
 
-ref<Downloader> getDownloader()
+ref<FileTransfer> getFileTransfer()
 {
-    static ref<Downloader> downloader = makeDownloader();
-    return downloader;
+    static ref<FileTransfer> fileTransfer = makeFileTransfer();
+    return fileTransfer;
 }
 
-ref<Downloader> makeDownloader()
+ref<FileTransfer> makeFileTransfer()
 {
-    return make_ref<CurlDownloader>();
+    return make_ref<curlFileTransfer>();
 }
 
-std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request)
+std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
 {
-    auto promise = std::make_shared<std::promise<DownloadResult>>();
-    enqueueDownload(request,
-        {[promise](std::future<DownloadResult> fut) {
+    auto promise = std::make_shared<std::promise<FileTransferResult>>();
+    enqueueFileTransfer(request,
+        {[promise](std::future<FileTransferResult> fut) {
             try {
                 promise->set_value(fut.get());
             } catch (...) {
@@ -701,15 +701,21 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
     return promise->get_future();
 }
 
-DownloadResult Downloader::download(const DownloadRequest & request)
+FileTransferResult FileTransfer::download(const FileTransferRequest & request)
+{
+    return enqueueFileTransfer(request).get();
+}
+
+FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
 {
-    return enqueueDownload(request).get();
+    /* Note: this method is the same as download, but helps in readability */
+    return enqueueFileTransfer(request).get();
 }
 
-void Downloader::download(DownloadRequest && request, Sink & sink)
+void FileTransfer::download(FileTransferRequest && request, Sink & sink)
 {
     /* Note: we can't call 'sink' via request.dataCallback, because
-       that would cause the sink to execute on the downloader
+       that would cause the sink to execute on the fileTransfer
        thread. If 'sink' is a coroutine, this will fail. Also, if the
        sink is expensive (e.g. one that does decompression and writing
        to the Nix store), it would stall the download thread too much.
@@ -755,8 +761,8 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
         state->avail.notify_one();
     };
 
-    enqueueDownload(request,
-        {[_state](std::future<DownloadResult> fut) {
+    enqueueFileTransfer(request,
+        {[_state](std::future<FileTransferResult> fut) {
             auto state(_state->lock());
             state->quit = true;
             try {
@@ -801,141 +807,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
     }
 }
 
-CachedDownloadResult Downloader::downloadCached(
-    ref<Store> store, const CachedDownloadRequest & request)
-{
-    auto url = resolveUri(request.uri);
-
-    auto name = request.name;
-    if (name == "") {
-        auto p = url.rfind('/');
-        if (p != string::npos) name = string(url, p + 1);
-    }
-
-    std::optional<StorePath> expectedStorePath;
-    if (request.expectedHash) {
-        auto method = request.unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-        expectedStorePath = store->makeFixedOutputPath(method, request.expectedHash, name);
-        if (store->isValidPath(*expectedStorePath)) {
-            CachedDownloadResult result;
-            result.storePath = store->printStorePath(*expectedStorePath);
-            result.path = store->toRealPath(result.storePath);
-            return result;
-        }
-    }
-
-    Path cacheDir = getCacheDir() + "/nix/tarballs";
-    createDirs(cacheDir);
-
-    string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
-
-    Path dataFile = cacheDir + "/" + urlHash + ".info";
-    Path fileLink = cacheDir + "/" + urlHash + "-file";
-
-    PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
-
-    std::optional<StorePath> storePath;
-
-    string expectedETag;
-
-    bool skip = false;
-
-    CachedDownloadResult result;
-
-    if (pathExists(fileLink) && pathExists(dataFile)) {
-        storePath = store->parseStorePath(readLink(fileLink));
-        // FIXME
-        store->addTempRoot(*storePath);
-        if (store->isValidPath(*storePath)) {
-            auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
-            if (ss.size() >= 3 && ss[0] == url) {
-                time_t lastChecked;
-                if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
-                    skip = true;
-                    result.effectiveUri = request.uri;
-                    result.etag = ss[1];
-                } else if (!ss[1].empty()) {
-                    debug(format("verifying previous ETag '%1%'") % ss[1]);
-                    expectedETag = ss[1];
-                }
-            }
-        } else
-            storePath.reset();
-    }
-
-    if (!skip) {
-
-        try {
-            DownloadRequest request2(url);
-            request2.expectedETag = expectedETag;
-            auto res = download(request2);
-            result.effectiveUri = res.effectiveUri;
-            result.etag = res.etag;
-
-            if (!res.cached) {
-                StringSink sink;
-                dumpString(*res.data, sink);
-                Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
-                ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
-                info.narHash = hashString(htSHA256, *sink.s);
-                info.narSize = sink.s->size();
-                info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
-                store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
-                storePath = info.path.clone();
-            }
-
-            assert(storePath);
-            replaceSymlink(store->printStorePath(*storePath), fileLink);
-
-            writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
-        } catch (DownloadError & e) {
-            if (!storePath) throw;
-            warn("warning: %s; using cached result", e.msg());
-            result.etag = expectedETag;
-        }
-    }
-
-    if (request.unpack) {
-        Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
-        PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
-        std::optional<StorePath> unpackedStorePath;
-        if (pathExists(unpackedLink)) {
-            unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
-            // FIXME
-            store->addTempRoot(*unpackedStorePath);
-            if (!store->isValidPath(*unpackedStorePath))
-                unpackedStorePath.reset();
-        }
-        if (!unpackedStorePath) {
-            printInfo("unpacking '%s'...", url);
-            Path tmpDir = createTempDir();
-            AutoDelete autoDelete(tmpDir, true);
-            unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir);
-            auto members = readDirectory(tmpDir);
-            if (members.size() != 1)
-                throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
-            auto topDir = tmpDir + "/" + members.begin()->name;
-            unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
-        }
-        replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink);
-        storePath = std::move(*unpackedStorePath);
-    }
-
-    if (expectedStorePath && *storePath != *expectedStorePath) {
-        unsigned int statusCode = 102;
-        Hash gotHash = request.unpack
-            ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first
-            : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath)));
-        throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n  wanted: %s\n  got:    %s",
-            url, request.expectedHash.to_string(), gotHash.to_string());
-    }
-
-    result.storePath = store->printStorePath(*storePath);
-    result.path = store->toRealPath(result.storePath);
-    return result;
-}
-
-
 bool isUri(const string & s)
 {
     if (s.compare(0, 8, "channel:") == 0) return true;
diff --git a/src/libstore/download.hh b/src/libstore/filetransfer.hh
index 5a131c704..2347f363d 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/filetransfer.hh
@@ -9,7 +9,7 @@
 
 namespace nix {
 
-struct DownloadSettings : Config
+struct FileTransferSettings : Config
 {
     Setting<bool> enableHttp2{this, true, "http2",
         "Whether to enable HTTP/2 support."};
@@ -31,15 +31,15 @@ struct DownloadSettings : Config
         "How often Nix will attempt to download a file before giving up."};
 };
 
-extern DownloadSettings downloadSettings;
+extern FileTransferSettings fileTransferSettings;
 
-struct DownloadRequest
+struct FileTransferRequest
 {
     std::string uri;
     std::string expectedETag;
     bool verifyTLS = true;
     bool head = false;
-    size_t tries = downloadSettings.tries;
+    size_t tries = fileTransferSettings.tries;
     unsigned int baseRetryTimeMs = 250;
     ActivityId parentAct;
     bool decompress = true;
@@ -47,7 +47,7 @@ struct DownloadRequest
     std::string mimeType;
     std::function<void(char *, size_t)> dataCallback;
 
-    DownloadRequest(const std::string & uri)
+    FileTransferRequest(const std::string & uri)
         : uri(uri), parentAct(getCurActivity()) { }
 
     std::string verb()
@@ -56,7 +56,7 @@ struct DownloadRequest
     }
 };
 
-struct DownloadResult
+struct FileTransferResult
 {
     bool cached = false;
     std::string etag;
@@ -65,74 +65,52 @@ struct DownloadResult
     uint64_t bodySize = 0;
 };
 
-struct CachedDownloadRequest
-{
-    std::string uri;
-    bool unpack = false;
-    std::string name;
-    Hash expectedHash;
-    unsigned int ttl;
-
-    CachedDownloadRequest(const std::string & uri);
-    CachedDownloadRequest() = delete;
-};
-
-struct CachedDownloadResult
-{
-    // Note: 'storePath' may be different from 'path' when using a
-    // chroot store.
-    Path storePath;
-    Path path;
-    std::optional<std::string> etag;
-    std::string effectiveUri;
-};
-
 class Store;
 
-struct Downloader
+struct FileTransfer
 {
-    virtual ~Downloader() { }
+    virtual ~FileTransfer() { }
 
-    /* Enqueue a download request, returning a future to the result of
-       the download. The future may throw a DownloadError
+    /* Enqueue a data transfer request, returning a future to the result of
+       the download. The future may throw a FileTransferError
        exception. */
-    virtual void enqueueDownload(const DownloadRequest & request,
-        Callback<DownloadResult> callback) = 0;
+    virtual void enqueueFileTransfer(const FileTransferRequest & request,
+        Callback<FileTransferResult> callback) = 0;
 
-    std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);
+    std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request);
 
     /* Synchronously download a file. */
-    DownloadResult download(const DownloadRequest & request);
+    FileTransferResult download(const FileTransferRequest & request);
+
+    /* Synchronously upload a file. */
+    FileTransferResult upload(const FileTransferRequest & request);
 
     /* Download a file, writing its data to a sink. The sink will be
        invoked on the thread of the caller. */
-    void download(DownloadRequest && request, Sink & sink);
-
-    /* Check if the specified file is already in ~/.cache/nix/tarballs
-       and is more recent than ‘tarball-ttl’ seconds. Otherwise,
-       use the recorded ETag to verify if the server has a more
-       recent version, and if so, download it to the Nix store. */
-    CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
+    void download(FileTransferRequest && request, Sink & sink);
 
     enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
 };
 
-/* Return a shared Downloader object. Using this object is preferred
+/* Return a shared FileTransfer object. Using this object is preferred
    because it enables connection reuse and HTTP/2 multiplexing. */
-ref<Downloader> getDownloader();
+ref<FileTransfer> getFileTransfer();
 
-/* Return a new Downloader object. */
-ref<Downloader> makeDownloader();
+/* Return a new FileTransfer object. */
+ref<FileTransfer> makeFileTransfer();
 
-class DownloadError : public Error
+class FileTransferError : public Error
 {
 public:
-    Downloader::Error error;
-    DownloadError(Downloader::Error error, const FormatOrString & fs)
+    FileTransfer::Error error;
+    FileTransferError(FileTransfer::Error error, const FormatOrString & fs)
         : Error(fs), error(error)
     { }
 };
 
 bool isUri(const string & s);
 
+/* Resolve deprecated 'channel:<foo>' URLs. */
+std::string resolveUri(const std::string & uri);
+
 }
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 0c3d89611..95a4bc934 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -202,6 +202,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
     /* Read the `temproots' directory for per-process temporary root
        files. */
     for (auto & i : readDirectory(tempRootsDir)) {
+        if (i.name[0] == '.') {
+            // Ignore hidden files. Some package managers (notably portage) create
+            // those to keep the directory alive.
+            continue;
+        }
         Path path = tempRootsDir + "/" + i.name;
 
         pid_t pid = std::stoi(i.name);
@@ -414,7 +419,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
 
                 try {
                     auto mapFile = fmt("/proc/%s/maps", ent->d_name);
-                    auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
+                    auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile), "\n");
                     for (const auto & line : mapLines) {
                         auto match = std::smatch{};
                         if (std::regex_match(line, match, mapRegex))
@@ -422,7 +427,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
                     }
 
                     auto envFile = fmt("/proc/%s/environ", ent->d_name);
-                    auto envString = readFile(envFile, true);
+                    auto envString = readFile(envFile);
                     auto env_end = std::sregex_iterator{};
                     for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
                         unchecked[i->str()].emplace(envFile);
@@ -884,7 +889,7 @@ void LocalStore::autoGC(bool sync)
         if (statvfs(realStoreDir.c_str(), &st))
             throw SysError("getting filesystem info about '%s'", realStoreDir);
 
-        return (uint64_t) st.f_bavail * st.f_bsize;
+        return (uint64_t) st.f_bavail * st.f_frsize;
     };
 
     std::shared_future<void> future;
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 7e97f3c22..bee94cbd8 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -31,6 +31,7 @@ Settings::Settings()
     , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
     , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
     , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
+    , nixUserConfFiles(getUserConfigFiles())
     , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
     , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
     , nixManDir(canonPath(NIX_MAN_DIR))
@@ -77,11 +78,27 @@ void loadConfFile()
        ~/.nix/nix.conf or the command line. */
     globalConfig.resetOverriden();
 
+    auto files = settings.nixUserConfFiles;
+    for (auto file = files.rbegin(); file != files.rend(); file++) {
+        globalConfig.applyConfigFile(*file);
+    }
+}
+
+std::vector<Path> getUserConfigFiles()
+{
+    // Use the paths specified in NIX_USER_CONF_FILES if it has been defined
+    auto nixConfFiles = getEnv("NIX_USER_CONF_FILES");
+    if (nixConfFiles.has_value()) {
+        return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":");
+    }
+
+    // Use the paths specified by the XDG spec
+    std::vector<Path> files;
     auto dirs = getConfigDirs();
-    // Iterate over them in reverse so that the ones appearing first in the path take priority
-    for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
-        globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
+    for (auto & dir : dirs) {
+        files.insert(files.end(), dir + "/nix/nix.conf");
     }
+    return files;
 }
 
 unsigned int Settings::getDefaultCores()
@@ -113,7 +130,7 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
 void Settings::requireExperimentalFeature(const std::string & name)
 {
     if (!isExperimentalFeatureEnabled(name))
-        throw Error("experimental Nix feature '%s' is disabled", name);
+        throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
 }
 
 bool Settings::isWSL1()
@@ -150,21 +167,24 @@ template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
 
 template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
 {
-    args.mkFlag()
-        .longName(name)
-        .description("Enable sandboxing.")
-        .handler([=](std::vector<std::string> ss) { override(smEnabled); })
-        .category(category);
-    args.mkFlag()
-        .longName("no-" + name)
-        .description("Disable sandboxing.")
-        .handler([=](std::vector<std::string> ss) { override(smDisabled); })
-        .category(category);
-    args.mkFlag()
-        .longName("relaxed-" + name)
-        .description("Enable sandboxing, but allow builds to disable it.")
-        .handler([=](std::vector<std::string> ss) { override(smRelaxed); })
-        .category(category);
+    args.addFlag({
+        .longName = name,
+        .description = "Enable sandboxing.",
+        .category = category,
+        .handler = {[=]() { override(smEnabled); }}
+    });
+    args.addFlag({
+        .longName = "no-" + name,
+        .description = "Disable sandboxing.",
+        .category = category,
+        .handler = {[=]() { override(smDisabled); }}
+    });
+    args.addFlag({
+        .longName = "relaxed-" + name,
+        .description = "Enable sandboxing, but allow builds to disable it.",
+        .category = category,
+        .handler = {[=]() { override(smRelaxed); }}
+    });
 }
 
 void MaxBuildJobsSetting::set(const std::string & str)
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 3aa3653f3..da95fd3ae 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -53,9 +53,12 @@ public:
     /* The directory where state is stored. */
     Path nixStateDir;
 
-    /* The directory where configuration files are stored. */
+    /* The directory where system configuration files are stored. */
     Path nixConfDir;
 
+    /* A list of user configuration files to load. */
+    std::vector<Path> nixUserConfFiles;
+
     /* The directory where internal helper programs are stored. */
     Path nixLibexecDir;
 
@@ -351,12 +354,21 @@ public:
     Setting<Paths> pluginFiles{this, {}, "plugin-files",
         "Plugins to dynamically load at nix initialization time."};
 
+    Setting<std::string> githubAccessToken{this, "", "github-access-token",
+        "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
+
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};
 
     bool isExperimentalFeatureEnabled(const std::string & name);
 
     void requireExperimentalFeature(const std::string & name);
+
+    Setting<bool> allowDirty{this, true, "allow-dirty",
+        "Whether to allow dirty Git/Mercurial trees."};
+
+    Setting<bool> warnDirty{this, true, "warn-dirty",
+        "Whether to warn about dirty Git/Mercurial trees."};
 };
 
 
@@ -369,6 +381,9 @@ void initPlugins();
 
 void loadConfFile();
 
+// Used by the Settings constructor
+std::vector<Path> getUserConfigFiles();
+
 extern const string nixVersion;
 
 }
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 011794c62..451a64785 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -1,5 +1,5 @@
 #include "binary-cache-store.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "globals.hh"
 #include "nar-info-disk-cache.hh"
 
@@ -85,14 +85,14 @@ protected:
         checkEnabled();
 
         try {
-            DownloadRequest request(cacheUri + "/" + path);
+            FileTransferRequest request(cacheUri + "/" + path);
             request.head = true;
-            getDownloader()->download(request);
+            getFileTransfer()->download(request);
             return true;
-        } catch (DownloadError & e) {
+        } catch (FileTransferError & e) {
             /* S3 buckets return 403 if a file doesn't exist and the
                bucket is unlistable, so treat 403 as 404. */
-            if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+            if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
                 return false;
             maybeDisable();
             throw;
@@ -103,19 +103,19 @@ protected:
         const std::string & data,
         const std::string & mimeType) override
     {
-        auto req = DownloadRequest(cacheUri + "/" + path);
+        auto req = FileTransferRequest(cacheUri + "/" + path);
         req.data = std::make_shared<string>(data); // FIXME: inefficient
         req.mimeType = mimeType;
         try {
-            getDownloader()->download(req);
-        } catch (DownloadError & e) {
+            getFileTransfer()->upload(req);
+        } catch (FileTransferError & e) {
             throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
         }
     }
 
-    DownloadRequest makeRequest(const std::string & path)
+    FileTransferRequest makeRequest(const std::string & path)
     {
-        DownloadRequest request(cacheUri + "/" + path);
+        FileTransferRequest request(cacheUri + "/" + path);
         return request;
     }
 
@@ -124,9 +124,9 @@ protected:
         checkEnabled();
         auto request(makeRequest(path));
         try {
-            getDownloader()->download(std::move(request), sink);
-        } catch (DownloadError & e) {
-            if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+            getFileTransfer()->download(std::move(request), sink);
+        } catch (FileTransferError & e) {
+            if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
                 throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
             maybeDisable();
             throw;
@@ -142,12 +142,12 @@ protected:
 
         auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
 
-        getDownloader()->enqueueDownload(request,
-            {[callbackPtr, this](std::future<DownloadResult> result) {
+        getFileTransfer()->enqueueFileTransfer(request,
+            {[callbackPtr, this](std::future<FileTransferResult> result) {
                 try {
                     (*callbackPtr)(result.get().data);
-                } catch (DownloadError & e) {
-                    if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+                } catch (FileTransferError & e) {
+                    if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
                         return (*callbackPtr)(std::shared_ptr<std::string>());
                     maybeDisable();
                     callbackPtr->rethrow();
@@ -174,4 +174,3 @@ static RegisterStoreImplementation regStore([](
 });
 
 }
-
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index f2e4b63e0..b24e7b7d6 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -6,7 +6,7 @@
 #include "nar-info-disk-cache.hh"
 #include "globals.hh"
 #include "compression.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "istringstream_nocopy.hh"
 
 #include <aws/core/Aws.h>
@@ -132,7 +132,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
     return res;
 }
 
-S3Helper::DownloadResult S3Helper::getObject(
+S3Helper::FileTransferResult S3Helper::getObject(
     const std::string & bucketName, const std::string & key)
 {
     debug("fetching 's3://%s/%s'...", bucketName, key);
@@ -146,7 +146,7 @@ S3Helper::DownloadResult S3Helper::getObject(
         return Aws::New<std::stringstream>("STRINGSTREAM");
     });
 
-    DownloadResult res;
+    FileTransferResult res;
 
     auto now1 = std::chrono::steady_clock::now();
 
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index ef5f23d0f..2042bffcf 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -18,13 +18,13 @@ struct S3Helper
 
     ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
 
-    struct DownloadResult
+    struct FileTransferResult
     {
         std::shared_ptr<std::string> data;
         unsigned int durationMs;
     };
 
-    DownloadResult getObject(
+    FileTransferResult getObject(
         const std::string & bucketName, const std::string & key);
 };
 
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index ca1409588..86cb20a26 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -6,6 +6,7 @@
 #include "thread-pool.hh"
 #include "json.hh"
 #include "derivations.hh"
+#include "url.hh"
 
 #include <future>
 
@@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const
         path = absPath(target, dirOf(path));
     }
     if (!isInStore(path))
-        throw Error(format("path '%1%' is not in the Nix store") % path);
+        throw NotInStore("path '%1%' is not in the Nix store", path);
     return path;
 }
 
@@ -873,27 +874,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
     Store::Params params;
     auto q = uri.find('?');
     if (q != std::string::npos) {
-        for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
-            auto e = s.find('=');
-            if (e != std::string::npos) {
-                auto value = s.substr(e + 1);
-                std::string decoded;
-                for (size_t i = 0; i < value.size(); ) {
-                    if (value[i] == '%') {
-                        if (i + 2 >= value.size())
-                            throw Error("invalid URI parameter '%s'", value);
-                        try {
-                            decoded += std::stoul(std::string(value, i + 1, 2), 0, 16);
-                            i += 3;
-                        } catch (...) {
-                            throw Error("invalid URI parameter '%s'", value);
-                        }
-                    } else
-                        decoded += value[i++];
-                }
-                params[s.substr(0, e)] = decoded;
-            }
-        }
+        params = decodeQuery(uri.substr(q + 1));
         uri = uri_.substr(0, q);
     }
     return {uri, params};
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 32c24c500..3def209a6 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -28,6 +28,7 @@ MakeError(InvalidPath, Error);
 MakeError(Unsupported, Error);
 MakeError(SubstituteGone, Error);
 MakeError(SubstituterDisabled, Error);
+MakeError(NotInStore, Error);
 
 
 struct BasicDerivation;