Dynamically allocate UIDs

Rather than rely on a nixbld group, we now allocate UIDs/GIDs
dynamically starting at a configurable ID (872415232 by default).

Also, we allocate 2^18 UIDs and GIDs per build, and run the build as
root in its UID namespace. (This should not be the default since it
breaks some builds. We probably should enable this conditional on a
requiredSystemFeature.) The goal is to be able to run (NixOS)
containers in a build. However, this will also require some cgroup
initialisation.

The 2^18 UIDs/GIDs is intended to provide enough ID space to run
multiple containers per build, e.g. for distributed NixOS tests.

2 files changed, 61 insertions, 8 deletions

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 347fe1b99..c853f609d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -513,7 +513,6 @@ private:
     AutoCloseFD fdUserLock;
 
     bool isEnabled = false;
-    string user;
     uid_t uid = 0;
     gid_t gid = 0;
     std::vector<gid_t> supplementaryGIDs;
@@ -523,9 +522,9 @@ public:
 
     void kill();
 
-    string getUser() { return user; }
     uid_t getUID() { assert(uid); return uid; }
-    uid_t getGID() { assert(gid); return gid; }
+    gid_t getGID() { assert(gid); return gid; }
+    uint32_t getIDCount() { return 1; }
     std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
 
     bool findFreeUser();
@@ -537,13 +536,16 @@ public:
 
 UserLock::UserLock()
 {
+#if 0
     assert(settings.buildUsersGroup != "");
     createDirs(settings.nixStateDir + "/userpool");
+#endif
 }
 
 bool UserLock::findFreeUser() {
     if (enabled()) return true;
 
+#if 0
     /* Get the members of the build-users-group. */
     struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
     if (!gr)
@@ -608,11 +610,45 @@ bool UserLock::findFreeUser() {
     }
 
     return false;
+#endif
+
+    assert(settings.startId > 0);
+    assert(settings.startId % settings.idsPerBuild == 0);
+    assert(settings.uidCount % settings.idsPerBuild == 0);
+    assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits<uid_t>::max());
+
+    // FIXME: check whether the id range overlaps any known users
+
+    size_t nrSlots = settings.uidCount / settings.idsPerBuild;
+
+    for (size_t i = 0; i < nrSlots; i++) {
+        debug("trying user slot '%d'", i);
+
+        createDirs(settings.nixStateDir + "/userpool");
+
+        fnUserLock = fmt("%s/userpool/slot-%d", settings.nixStateDir, i);
+
+        AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+        if (!fd)
+            throw SysError("opening user lock '%1%'", fnUserLock);
+
+        if (lockFile(fd.get(), ltWrite, false)) {
+            fdUserLock = std::move(fd);
+            uid = settings.startId + i * settings.idsPerBuild;
+            gid = settings.startId + i * settings.idsPerBuild;
+            return true;
+        }
+    }
+
+    return false;
 }
 
 void UserLock::kill()
 {
+    // FIXME: use a cgroup to kill all processes in the build?
+#if 0
     killUser(uid);
+#endif
 }
 
 
@@ -1523,7 +1559,7 @@ void DerivationGoal::tryLocalBuild() {
 
     /* If `build-users-group' is not empty, then we have to build as
        one of the members of that group. */
-    if (settings.buildUsersGroup != "" && getuid() == 0) {
+    if ((settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0) {
 #if defined(__linux__) || defined(__APPLE__)
         if (!buildUser) buildUser = std::make_unique<UserLock>();
 
@@ -2129,7 +2165,7 @@ void DerivationGoal::startBuilder()
 
         printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
 
-        if (mkdir(chrootRootDir.c_str(), 0750) == -1)
+        if (mkdir(chrootRootDir.c_str(), 0755) == -1)
             throw SysError("cannot create '%1%'", chrootRootDir);
 
         if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1)
@@ -2444,14 +2480,15 @@ void DerivationGoal::startBuilder()
            the calling user (if build users are disabled). */
         uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
         uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
+        uint32_t nrIds = settings.idsPerBuild; // FIXME
 
         writeFile("/proc/" + std::to_string(pid) + "/uid_map",
-            (format("%d %d 1") % sandboxUid % hostUid).str());
+            fmt("%d %d %d", /* sandboxUid */ 0, hostUid, nrIds));
 
-        writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
+        //writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
 
         writeFile("/proc/" + std::to_string(pid) + "/gid_map",
-            (format("%d %d 1") % sandboxGid % hostGid).str());
+            fmt("%d %d %d", /* sandboxGid */ 0, hostGid, nrIds));
 
         /* Save the mount namespace of the child. We have to do this
            *before* the child does a chroot. */
@@ -3306,10 +3343,16 @@ void DerivationGoal::runChild()
             /* Switch to the sandbox uid/gid in the user namespace,
                which corresponds to the build user or calling user in
                the parent namespace. */
+#if 0
             if (setgid(sandboxGid) == -1)
                 throw SysError("setgid failed");
             if (setuid(sandboxUid) == -1)
                 throw SysError("setuid failed");
+#endif
+            if (setgid(0) == -1)
+                throw SysError("setgid failed");
+            if (setuid(0) == -1)
+                throw SysError("setuid failed");
 
             setUser = false;
         }
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 58cf08763..7dc842bca 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -148,6 +148,16 @@ public:
     Setting<std::string> buildUsersGroup{this, "", "build-users-group",
         "The Unix group that contains the build users."};
 
+    #if __linux__
+    const uint32_t idsPerBuild = 1 << 18;
+
+    Setting<uint32_t> startId{this, 872415232, "start-id",
+        "The first UID and GID to use for dynamic ID allocation. (0 means disable.)"};
+
+    Setting<uint32_t> uidCount{this, idsPerBuild * 128, "id-count",
+        "The number of UIDs/GIDs to use for dynamic ID allocation."};
+    #endif
+
     Setting<bool> impersonateLinux26{this, false, "impersonate-linux-26",
         "Whether to impersonate a Linux 2.6 machine on newer kernels.",
         {"build-impersonate-linux-26"}};