Merge remote-tracking branch 'origin/master' into coerce-string

8 files changed, 340 insertions, 209 deletions

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 41d2e2a1c..173058d1b 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -7,7 +7,6 @@
 #include "finally.hh"
 #include "util.hh"
 #include "archive.hh"
-#include "json.hh"
 #include "compression.hh"
 #include "worker-protocol.hh"
 #include "topo-sort.hh"
@@ -40,7 +39,6 @@
 #include <sys/ioctl.h>
 #include <net/if.h>
 #include <netinet/ip.h>
-#include <sys/personality.h>
 #include <sys/mman.h>
 #include <sched.h>
 #include <sys/param.h>
@@ -135,7 +133,7 @@ void DerivationGoal::killChild()
 void DerivationGoal::timedOut(Error && ex)
 {
     killChild();
-    done(BuildResult::TimedOut, {}, ex);
+    done(BuildResult::TimedOut, {}, std::move(ex));
 }
 
 
@@ -502,6 +500,14 @@ void DerivationGoal::inputsRealised()
                now-known results of dependencies. If so, we become a
                stub goal aliasing that resolved derivation goal. */
             std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
+            if (!attempt) {
+              /* TODO (impure derivations-induced tech debt) (see below):
+                 The above attempt should have found it, but because we manage
+                 inputDrvOutputs statefully, sometimes it gets out of sync with
+                 the real source of truth (store). So we query the store
+                 directly if there's a problem. */
+              attempt = fullDrv.tryResolve(worker.store);
+            }
             assert(attempt);
             Derivation drvResolved { *std::move(attempt) };
 
@@ -528,13 +534,31 @@ void DerivationGoal::inputsRealised()
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            for (auto & j : wantedDepOutputs)
-                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
+            for (auto & j : wantedDepOutputs) {
+                /* TODO (impure derivations-induced tech debt):
+                   Tracking input derivation outputs statefully through the
+                   goals is error prone and has led to bugs.
+                   For a robust nix, we need to move towards the `else` branch,
+                   which does not rely on goal state to match up with the
+                   reality of the store, which is our real source of truth.
+                   However, the impure derivations feature still relies on this
+                   fragile way of doing things, because its builds do not have
+                   a representation in the store, which is a usability problem
+                   in itself */
+                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
                     worker.store.computeFSClosure(*outPath, inputPaths);
-                else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+                }
+                else {
+                    auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
+                    auto outMapPath = outMap.find(j);
+                    if (outMapPath == outMap.end()) {
+                        throw Error(
+                            "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
+                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+                    }
+                    worker.store.computeFSClosure(outMapPath->second, inputPaths);
+                }
+            }
         }
     }
 
@@ -546,10 +570,6 @@ void DerivationGoal::inputsRealised()
     /* What type of derivation are we building? */
     derivationType = drv->type();
 
-    /* Don't repeat fixed-output derivations since they're already
-       verified by their output hash.*/
-    nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1;
-
     /* Okay, try to build.  Note that here we don't wait for a build
        slot to become available, since we don't need one if there is a
        build hook. */
@@ -564,12 +584,11 @@ void DerivationGoal::started()
     auto msg = fmt(
         buildMode == bmRepair ? "repairing outputs of '%s'" :
         buildMode == bmCheck ? "checking outputs of '%s'" :
-        nrRounds > 1 ? "building '%s' (round %d/%d)" :
-        "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
+        "building '%s'", worker.store.printStorePath(drvPath));
     fmt("building '%s'", worker.store.printStorePath(drvPath));
     if (hook) msg += fmt(" on '%s'", machineName);
     act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
-        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
+        Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1});
     mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
     worker.updateProgress();
 }
@@ -869,6 +888,14 @@ void DerivationGoal::buildDone()
 
     cleanupPostChildKill();
 
+    if (buildResult.cpuUser && buildResult.cpuSystem) {
+        debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs",
+            worker.store.printStorePath(drvPath),
+            status,
+            ((double) buildResult.cpuUser->count()) / 1000000,
+            ((double) buildResult.cpuSystem->count()) / 1000000);
+    }
+
     bool diskFull = false;
 
     try {
@@ -915,14 +942,6 @@ void DerivationGoal::buildDone()
 
         cleanupPostOutputsRegisteredModeNonCheck();
 
-        /* Repeat the build if necessary. */
-        if (curRound++ < nrRounds) {
-            outputLocks.unlock();
-            state = &DerivationGoal::tryToBuild;
-            worker.wakeUp(shared_from_this());
-            return;
-        }
-
         /* It is now safe to delete the lock files, since all future
            lockers will see that the output paths are valid; they will
            not create new lock files with the same names as the old
@@ -951,7 +970,7 @@ void DerivationGoal::buildDone()
                 BuildResult::PermanentFailure;
         }
 
-        done(st, {}, e);
+        done(st, {}, std::move(e));
         return;
     }
 }
@@ -983,22 +1002,34 @@ void DerivationGoal::resolvedFinished()
                 throw Error(
                     "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)",
                     worker.store.printStorePath(drvPath), wantedOutput);
-            auto realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput });
-            if (!realisation)
-                throw Error(
-                    "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
-                    worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
+
+            auto realisation = [&]{
+              auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput });
+              if (take1) return *take1;
+
+              /* The above `get` should work. But sateful tracking of
+                 outputs in resolvedResult, this can get out of sync with the
+                 store, which is our actual source of truth. For now we just
+                 check the store directly if it fails. */
+              auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput });
+              if (take2) return *take2;
+
+              throw Error(
+                  "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)",
+                  worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput);
+            }();
+
             if (drv->type().isPure()) {
-                auto newRealisation = *realisation;
+                auto newRealisation = realisation;
                 newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput };
                 newRealisation.signatures.clear();
                 if (!drv->type().isFixed())
-                    newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
+                    newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
                 signRealisation(newRealisation);
                 worker.store.registerDrvOutput(newRealisation);
             }
-            outputPaths.insert(realisation->outPath);
-            builtOutputs.emplace(realisation->id, *realisation);
+            outputPaths.insert(realisation.outPath);
+            builtOutputs.emplace(realisation.id, realisation);
         }
 
         runPostBuildHook(
@@ -1402,7 +1433,7 @@ void DerivationGoal::done(
         fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
     }
 
-    amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
+    amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex));
 }
 
 
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index 2d8bfd592..d33e04cbc 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -115,11 +115,6 @@ struct DerivationGoal : public Goal
 
     BuildMode buildMode;
 
-    /* The current round, if we're building multiple times. */
-    size_t curRound = 1;
-
-    size_t nrRounds;
-
     std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
 
     std::unique_ptr<Activity> act;
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index bea7363db..e1b80165e 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -30,7 +30,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
             if (ex)
                 logError(i->ex->info());
             else
-                ex = i->ex;
+                ex = std::move(i->ex);
         }
         if (i->exitCode != Goal::ecSuccess) {
             if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
@@ -40,7 +40,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
 
     if (failed.size() == 1 && ex) {
         ex->status = worker.exitStatus();
-        throw *ex;
+        throw std::move(*ex);
     } else if (!failed.empty()) {
         if (ex) logError(ex->info());
         throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed));
@@ -109,7 +109,7 @@ void Store::ensurePath(const StorePath & path)
     if (goal->exitCode != Goal::ecSuccess) {
         if (goal->ex) {
             goal->ex->status = worker.exitStatus();
-            throw *goal->ex;
+            throw std::move(*goal->ex);
         } else
             throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path));
     }
diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc
index 1f19ddccc..cb58a1f02 100644
--- a/src/libstore/build/hook-instance.cc
+++ b/src/libstore/build/hook-instance.cc
@@ -16,11 +16,11 @@ HookInstance::HookInstance()
     buildHookArgs.pop_front();
 
     Strings args;
+    args.push_back(std::string(baseNameOf(buildHook)));
 
     for (auto & arg : buildHookArgs)
         args.push_back(arg);
 
-    args.push_back(std::string(baseNameOf(settings.buildHook.get())));
     args.push_back(std::to_string(verbosity));
 
     /* Create a pipe to get the output of the child. */
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 5cea3b590..9d869d513 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -8,13 +8,14 @@
 #include "finally.hh"
 #include "util.hh"
 #include "archive.hh"
-#include "json.hh"
 #include "compression.hh"
 #include "daemon.hh"
 #include "worker-protocol.hh"
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "json-utils.hh"
+#include "cgroup.hh"
+#include "personality.hh"
 
 #include <regex>
 #include <queue>
@@ -24,7 +25,6 @@
 #include <termios.h>
 #include <unistd.h>
 #include <sys/mman.h>
-#include <sys/utsname.h>
 #include <sys/resource.h>
 #include <sys/socket.h>
 
@@ -37,7 +37,6 @@
 #include <sys/ioctl.h>
 #include <net/if.h>
 #include <netinet/ip.h>
-#include <sys/personality.h>
 #include <sys/mman.h>
 #include <sched.h>
 #include <sys/param.h>
@@ -56,6 +55,7 @@
 
 #include <pwd.h>
 #include <grp.h>
+#include <iostream>
 
 namespace nix {
 
@@ -129,26 +129,44 @@ void LocalDerivationGoal::killChild()
     if (pid != -1) {
         worker.childTerminated(this);
 
-        if (buildUser) {
-            /* If we're using a build user, then there is a tricky
-               race condition: if we kill the build user before the
-               child has done its setuid() to the build user uid, then
-               it won't be killed, and we'll potentially lock up in
-               pid.wait().  So also send a conventional kill to the
-               child. */
-            ::kill(-pid, SIGKILL); /* ignore the result */
-            buildUser->kill();
-            pid.wait();
-        } else
-            pid.kill();
+        /* If we're using a build user, then there is a tricky race
+           condition: if we kill the build user before the child has
+           done its setuid() to the build user uid, then it won't be
+           killed, and we'll potentially lock up in pid.wait().  So
+           also send a conventional kill to the child. */
+        ::kill(-pid, SIGKILL); /* ignore the result */
+
+        killSandbox(true);
 
-        assert(pid == -1);
+        pid.wait();
     }
 
     DerivationGoal::killChild();
 }
 
 
+void LocalDerivationGoal::killSandbox(bool getStats)
+{
+    if (cgroup) {
+        #if __linux__
+        auto stats = destroyCgroup(*cgroup);
+        if (getStats) {
+            buildResult.cpuUser = stats.cpuUser;
+            buildResult.cpuSystem = stats.cpuSystem;
+        }
+        #else
+        abort();
+        #endif
+    }
+
+    else if (buildUser) {
+        auto uid = buildUser->getUID();
+        assert(uid != 0);
+        killUser(uid);
+    }
+}
+
+
 void LocalDerivationGoal::tryLocalBuild() {
     unsigned int curBuilds = worker.getNrLocalBuilds();
     if (curBuilds >= settings.maxBuildJobs) {
@@ -158,28 +176,46 @@ void LocalDerivationGoal::tryLocalBuild() {
         return;
     }
 
-    /* If `build-users-group' is not empty, then we have to build as
-       one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
-        if (!buildUser) buildUser = std::make_unique<UserLock>();
+    /* Are we doing a chroot build? */
+    {
+        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
+        if (settings.sandboxMode == smEnabled) {
+            if (noChroot)
+                throw Error("derivation '%s' has '__noChroot' set, "
+                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
+#if __APPLE__
+            if (additionalSandboxProfile != "")
+                throw Error("derivation '%s' specifies a sandbox profile, "
+                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
+#endif
+            useChroot = true;
+        }
+        else if (settings.sandboxMode == smDisabled)
+            useChroot = false;
+        else if (settings.sandboxMode == smRelaxed)
+            useChroot = derivationType.isSandboxed() && !noChroot;
+    }
+
+    auto & localStore = getLocalStore();
+    if (localStore.storeDir != localStore.realStoreDir.get()) {
+        #if __linux__
+            useChroot = true;
+        #else
+            throw Error("building using a diverted store is not supported on this platform");
+        #endif
+    }
 
-        if (buildUser->findFreeUser()) {
-            /* Make sure that no other processes are executing under this
-               uid. */
-            buildUser->kill();
-        } else {
+    if (useBuildUsers()) {
+        if (!buildUser)
+            buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot);
+
+        if (!buildUser) {
             if (!actLock)
                 actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
                     fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
             worker.waitForAWhile(shared_from_this());
             return;
         }
-#else
-        /* Don't know how to block the creation of setuid/setgid
-           binaries on this platform. */
-        throw Error("build users are not supported on this platform for security reasons");
-#endif
     }
 
     actLock.reset();
@@ -193,7 +229,7 @@ void LocalDerivationGoal::tryLocalBuild() {
         outputLocks.unlock();
         buildUser.reset();
         worker.permanentFailure = true;
-        done(BuildResult::InputRejected, {}, e);
+        done(BuildResult::InputRejected, {}, std::move(e));
         return;
     }
 
@@ -270,7 +306,7 @@ void LocalDerivationGoal::cleanupPostChildKill()
        malicious user from leaving behind a process that keeps files
        open and modifies them after they have been chown'ed to
        root. */
-    if (buildUser) buildUser->kill();
+    killSandbox(true);
 
     /* Terminate the recursive Nix daemon. */
     stopDaemon();
@@ -363,6 +399,64 @@ static void linkOrCopy(const Path & from, const Path & to)
 
 void LocalDerivationGoal::startBuilder()
 {
+    if ((buildUser && buildUser->getUIDCount() != 1)
+        #if __linux__
+        || settings.useCgroups
+        #endif
+        )
+    {
+        #if __linux__
+        settings.requireExperimentalFeature(Xp::Cgroups);
+
+        auto cgroupFS = getCgroupFS();
+        if (!cgroupFS)
+            throw Error("cannot determine the cgroups file system");
+
+        auto ourCgroups = getCgroups("/proc/self/cgroup");
+        auto ourCgroup = ourCgroups[""];
+        if (ourCgroup == "")
+            throw Error("cannot determine cgroup name from /proc/self/cgroup");
+
+        auto ourCgroupPath = canonPath(*cgroupFS + "/" + ourCgroup);
+
+        if (!pathExists(ourCgroupPath))
+            throw Error("expected cgroup directory '%s'", ourCgroupPath);
+
+        static std::atomic<unsigned int> counter{0};
+
+        cgroup = buildUser
+            ? fmt("%s/nix-build-uid-%d", ourCgroupPath, buildUser->getUID())
+            : fmt("%s/nix-build-pid-%d-%d", ourCgroupPath, getpid(), counter++);
+
+        debug("using cgroup '%s'", *cgroup);
+
+        /* When using a build user, record the cgroup we used for that
+           user so that if we got interrupted previously, we can kill
+           any left-over cgroup first. */
+        if (buildUser) {
+            auto cgroupsDir = settings.nixStateDir + "/cgroups";
+            createDirs(cgroupsDir);
+
+            auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID());
+
+            if (pathExists(cgroupFile)) {
+                auto prevCgroup = readFile(cgroupFile);
+                destroyCgroup(prevCgroup);
+            }
+
+            writeFile(cgroupFile, *cgroup);
+        }
+
+        #else
+        throw Error("cgroups are not supported on this platform");
+        #endif
+    }
+
+    /* Make sure that no other processes are executing under the
+       sandbox uids. This must be done before any chownToBuilder()
+       calls. */
+    killSandbox(false);
+
     /* Right platform? */
     if (!parsedDrv->canBuildLocally(worker.store))
         throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
@@ -376,35 +470,6 @@ void LocalDerivationGoal::startBuilder()
     additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
 #endif
 
-    /* Are we doing a chroot build? */
-    {
-        auto noChroot = parsedDrv->getBoolAttr("__noChroot");
-        if (settings.sandboxMode == smEnabled) {
-            if (noChroot)
-                throw Error("derivation '%s' has '__noChroot' set, "
-                    "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
-            if (additionalSandboxProfile != "")
-                throw Error("derivation '%s' specifies a sandbox profile, "
-                    "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
-            useChroot = true;
-        }
-        else if (settings.sandboxMode == smDisabled)
-            useChroot = false;
-        else if (settings.sandboxMode == smRelaxed)
-            useChroot = derivationType.isSandboxed() && !noChroot;
-    }
-
-    auto & localStore = getLocalStore();
-    if (localStore.storeDir != localStore.realStoreDir.get()) {
-        #if __linux__
-            useChroot = true;
-        #else
-            throw Error("building using a diverted store is not supported on this platform");
-        #endif
-    }
-
     /* Create a temporary directory where the build will take
        place. */
     tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
@@ -580,10 +645,11 @@ void LocalDerivationGoal::startBuilder()
 
         printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
 
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
+        // FIXME: make this 0700
+        if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1)
             throw SysError("cannot create '%1%'", chrootRootDir);
 
-        if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
+        if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1)
             throw SysError("cannot change ownership of '%1%'", chrootRootDir);
 
         /* Create a writable /tmp in the chroot.  Many builders need
@@ -597,6 +663,10 @@ void LocalDerivationGoal::startBuilder()
            nobody account.  The latter is kind of a hack to support
            Samba-in-QEMU. */
         createDirs(chrootRootDir + "/etc");
+        chownToBuilder(chrootRootDir + "/etc");
+
+        if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536))
+            throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name);
 
         /* Declare the build user's group so that programs get a consistent
            view of the system (e.g., "id -gn"). */
@@ -647,12 +717,28 @@ void LocalDerivationGoal::startBuilder()
                 dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
         }
 
-#elif __APPLE__
-        /* We don't really have any parent prep work to do (yet?)
-           All work happens in the child, instead. */
+        if (cgroup) {
+            if (mkdir(cgroup->c_str(), 0755) != 0)
+                throw SysError("creating cgroup '%s'", *cgroup);
+            chownToBuilder(*cgroup);
+            chownToBuilder(*cgroup + "/cgroup.procs");
+            chownToBuilder(*cgroup + "/cgroup.threads");
+            //chownToBuilder(*cgroup + "/cgroup.subtree_control");
+        }
+
 #else
-        throw Error("sandboxing builds is not supported on this platform");
+        if (parsedDrv->useUidRange())
+            throw Error("feature 'uid-range' is not supported on this platform");
+        #if __APPLE__
+            /* We don't really have any parent prep work to do (yet?)
+               All work happens in the child, instead. */
+        #else
+            throw Error("sandboxing builds is not supported on this platform");
+        #endif
 #endif
+    } else {
+        if (parsedDrv->useUidRange())
+            throw Error("feature 'uid-range' is only supported in sandboxed builds");
     }
 
     if (needsHashRewrite() && pathExists(homeDir))
@@ -913,14 +999,16 @@ void LocalDerivationGoal::startBuilder()
                the calling user (if build users are disabled). */
             uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
             uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
+            uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1;
 
             writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-                fmt("%d %d 1", sandboxUid(), hostUid));
+                fmt("%d %d %d", sandboxUid(), hostUid, nrIds));
 
-            writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
+            if (!buildUser || buildUser->getUIDCount() == 1)
+                writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
 
             writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-                fmt("%d %d 1", sandboxGid(), hostGid));
+                fmt("%d %d %d", sandboxGid(), hostGid, nrIds));
         } else {
             debug("note: not using a user namespace");
             if (!buildUser)
@@ -947,6 +1035,10 @@ void LocalDerivationGoal::startBuilder()
                 throw SysError("getting sandbox user namespace");
         }
 
+        /* Move the child into its own cgroup. */
+        if (cgroup)
+            writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
+
         /* Signal the builder that we've updated its user namespace. */
         writeFull(userNamespaceSync.writeSide.get(), "1");
 
@@ -1552,6 +1644,22 @@ void setupSeccomp()
         seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
         printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
 
+    if (nativeSystem == "mips64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPS) != 0)
+        printError("unable to add mips seccomp architecture");
+
+    if (nativeSystem == "mips64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPS64N32) != 0)
+        printError("unable to add mips64-*abin32 seccomp architecture");
+
+    if (nativeSystem == "mips64el-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL) != 0)
+        printError("unable to add mipsel seccomp architecture");
+
+    if (nativeSystem == "mips64el-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL64N32) != 0)
+        printError("unable to add mips64el-*abin32 seccomp architecture");
+
     /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
@@ -1763,6 +1871,13 @@ void LocalDerivationGoal::runChild()
             if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
                 throw SysError("mounting /proc");
 
+            /* Mount sysfs on /sys. */
+            if (buildUser && buildUser->getUIDCount() != 1) {
+                createDirs(chrootRootDir + "/sys");
+                if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1)
+                    throw SysError("mounting /sys");
+            }
+
             /* Mount a new tmpfs on /dev/shm to ensure that whatever
                the builder puts in /dev/shm is cleaned up automatically. */
             if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
@@ -1805,6 +1920,12 @@ void LocalDerivationGoal::runChild()
             if (unshare(CLONE_NEWNS) == -1)
                 throw SysError("unsharing mount namespace");
 
+            /* Unshare the cgroup namespace. This means
+               /proc/self/cgroup will show the child's cgroup as '/'
+               rather than whatever it is in the parent. */
+            if (cgroup && unshare(CLONE_NEWCGROUP) == -1)
+                throw SysError("unsharing cgroup namespace");
+
             /* Do the chroot(). */
             if (chdir(chrootRootDir.c_str()) == -1)
                 throw SysError("cannot change directory to '%1%'", chrootRootDir);
@@ -1842,33 +1963,7 @@ void LocalDerivationGoal::runChild()
         /* Close all other file descriptors. */
         closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
 
-#if __linux__
-        /* Change the personality to 32-bit if we're doing an
-           i686-linux build on an x86_64-linux machine. */
-        struct utsname utsbuf;
-        uname(&utsbuf);
-        if ((drv->platform == "i686-linux"
-                && (settings.thisSystem == "x86_64-linux"
-                    || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
-            || drv->platform == "armv7l-linux"
-            || drv->platform == "armv6l-linux")
-        {
-            if (personality(PER_LINUX32) == -1)
-                throw SysError("cannot set 32-bit personality");
-        }
-
-        /* Impersonate a Linux 2.6 machine to get some determinism in
-           builds that depend on the kernel version. */
-        if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
-            int cur = personality(0xffffffff);
-            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
-        }
-
-        /* Disable address space randomization for improved
-           determinism. */
-        int cur = personality(0xffffffff);
-        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
+        setPersonality(drv->platform);
 
         /* Disable core dumps by default. */
         struct rlimit limit = { 0, RLIM_INFINITY };
@@ -1890,9 +1985,8 @@ void LocalDerivationGoal::runChild()
         if (setUser && buildUser) {
             /* Preserve supplementary groups of the build user, to allow
                admins to specify groups such as "kvm".  */
-            if (!buildUser->getSupplementaryGIDs().empty() &&
-                setgroups(buildUser->getSupplementaryGIDs().size(),
-                          buildUser->getSupplementaryGIDs().data()) == -1)
+            auto gids = buildUser->getSupplementaryGIDs();
+            if (setgroups(gids.size(), gids.data()) == -1)
                 throw SysError("cannot set supplementary groups of build user");
 
             if (setgid(buildUser->getGID()) == -1 ||
@@ -2139,7 +2233,6 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
     InodesSeen inodesSeen;
 
     Path checkSuffix = ".check";
-    bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
 
     std::exception_ptr delayedException;
 
@@ -2221,7 +2314,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
         /* Canonicalise first.  This ensures that the path we're
            rewriting doesn't contain a hard link to /etc/shadow or
            something like that. */
-        canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
+        canonicalisePathMetaData(
+            actualPath,
+            buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt,
+            inodesSeen);
 
         debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
 
@@ -2314,6 +2410,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
                 sink.s = rewriteStrings(sink.s, outputRewrites);
                 StringSource source(sink.s);
                 restorePath(actualPath, source);
+
+                /* FIXME: set proper permissions in restorePath() so
+                   we don't have to do another traversal. */
+                canonicalisePathMetaData(actualPath, {}, inodesSeen);
             }
         };
 
@@ -2476,7 +2576,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
 
         /* FIXME: set proper permissions in restorePath() so
             we don't have to do another traversal. */
-        canonicalisePathMetaData(actualPath, -1, inodesSeen);
+        canonicalisePathMetaData(actualPath, {}, inodesSeen);
 
         /* Calculate where we'll move the output files. In the checking case we
            will leave leave them where they are, for now, rather than move to
@@ -2560,10 +2660,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
                 debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
         }
 
-        if (curRound == nrRounds) {
-            localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences()
-            worker.markContentsGood(newInfo.path);
-        }
+        localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences()
+        worker.markContentsGood(newInfo.path);
 
         newInfo.deriver = drvPath;
         newInfo.ultimate = true;
@@ -2592,61 +2690,6 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
     /* Apply output checks. */
     checkOutputs(infos);
 
-    /* Compare the result with the previous round, and report which
-       path is different, if any.*/
-    if (curRound > 1 && prevInfos != infos) {
-        assert(prevInfos.size() == infos.size());
-        for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
-            if (!(*i == *j)) {
-                buildResult.isNonDeterministic = true;
-                Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
-                bool prevExists = keepPreviousRound && pathExists(prev);
-                hintformat hint = prevExists
-                    ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
-                    : hintfmt("output '%s' of '%s' differs from previous round",
-                        worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
-                handleDiffHook(
-                    buildUser ? buildUser->getUID() : getuid(),
-                    buildUser ? buildUser->getGID() : getgid(),
-                    prev, worker.store.printStorePath(i->second.path),
-                    worker.store.printStorePath(drvPath), tmpDir);
-
-                if (settings.enforceDeterminism)
-                    throw NotDeterministic(hint);
-
-                printError(hint);
-
-                curRound = nrRounds; // we know enough, bail out early
-            }
-    }
-
-    /* If this is the first round of several, then move the output out of the way. */
-    if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            auto path = worker.store.printStorePath(outputStorePath);
-            Path prev = path + checkSuffix;
-            deletePath(prev);
-            Path dst = path + checkSuffix;
-            renameFile(path, dst);
-        }
-    }
-
-    if (curRound < nrRounds) {
-        prevInfos = std::move(infos);
-        return {};
-    }
-
-    /* Remove the .check directories if we're done. FIXME: keep them
-       if the result was not determistic? */
-    if (curRound == nrRounds) {
-        for (auto & [_, outputStorePath] : finalOutputs) {
-            Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
-            deletePath(prev);
-        }
-    }
-
     /* Register each output path as valid, and register the sets of
        paths referenced by each of them.  If there are cycles in the
        outputs, this will fail. */
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
index d456e9cae..34c4e9187 100644
--- a/src/libstore/build/local-derivation-goal.hh
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -15,6 +15,9 @@ struct LocalDerivationGoal : public DerivationGoal
     /* The process ID of the builder. */
     Pid pid;
 
+    /* The cgroup of the builder, if any. */
+    std::optional<Path> cgroup;
+
     /* The temporary directory. */
     Path tmpDir;
 
@@ -92,8 +95,8 @@ struct LocalDerivationGoal : public DerivationGoal
        result. */
     std::map<Path, ValidPathInfo> prevInfos;
 
-    uid_t sandboxUid() { return usingUserNamespace ? 1000 : buildUser->getUID(); }
-    gid_t sandboxGid() { return usingUserNamespace ?  100 : buildUser->getGID(); }
+    uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
+    gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100  : 0) : buildUser->getGID(); }
 
     const static Path homeDir;
 
@@ -197,6 +200,10 @@ struct LocalDerivationGoal : public DerivationGoal
     /* Forcibly kill the child process, if any. */
     void killChild() override;
 
+    /* Kill any processes running under the build user UID or in the
+       cgroup of the build. */
+    void killSandbox(bool getStats);
+
     /* Create alternative path calculated from but distinct from the
        input, so we can avoid overwriting outputs (or other store paths)
        that already exist. */
diff --git a/src/libstore/build/personality.cc b/src/libstore/build/personality.cc
new file mode 100644
index 000000000..4ad477869
--- /dev/null
+++ b/src/libstore/build/personality.cc
@@ -0,0 +1,44 @@
+#include "personality.hh"
+#include "globals.hh"
+
+#if __linux__
+#include <sys/utsname.h>
+#include <sys/personality.h>
+#endif
+
+#include <cstring>
+
+namespace nix {
+
+void setPersonality(std::string_view system)
+{
+#if __linux__
+        /* Change the personality to 32-bit if we're doing an
+           i686-linux build on an x86_64-linux machine. */
+        struct utsname utsbuf;
+        uname(&utsbuf);
+        if ((system == "i686-linux"
+                && (std::string_view(SYSTEM) == "x86_64-linux"
+                    || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64"))))
+            || system == "armv7l-linux"
+            || system == "armv6l-linux")
+        {
+            if (personality(PER_LINUX32) == -1)
+                throw SysError("cannot set 32-bit personality");
+        }
+
+        /* Impersonate a Linux 2.6 machine to get some determinism in
+           builds that depend on the kernel version. */
+        if ((system == "i686-linux" || system == "x86_64-linux") && settings.impersonateLinux26) {
+            int cur = personality(0xffffffff);
+            if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
+        }
+
+        /* Disable address space randomization for improved
+           determinism. */
+        int cur = personality(0xffffffff);
+        if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
+#endif
+}
+
+}
diff --git a/src/libstore/build/personality.hh b/src/libstore/build/personality.hh
new file mode 100644
index 000000000..30e4f4062
--- /dev/null
+++ b/src/libstore/build/personality.hh
@@ -0,0 +1,11 @@
+#pragma once
+
+#include <string>
+
+namespace nix {
+
+void setPersonality(std::string_view system);
+
+}
+
+