From 836573a9a2d38935e254702826d250ea39824a1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Oct 2017 12:22:29 +0100 Subject: Dynamically allocate UIDs Rather than rely on a nixbld group, we now allocate UIDs/GIDs dynamically starting at a configurable ID (872415232 by default). Also, we allocate 2^18 UIDs and GIDs per build, and run the build as root in its UID namespace. (This should not be the default since it breaks some builds. We probably should enable this conditional on a requiredSystemFeature.) The goal is to be able to run (NixOS) containers in a build. However, this will also require some cgroup initialisation. The 2^18 UIDs/GIDs is intended to provide enough ID space to run multiple containers per build, e.g. for distributed NixOS tests. --- src/libstore/build.cc | 59 ++++++++++++++++++++++++++++++++++++++++++------- src/libstore/globals.hh | 10 +++++++++ 2 files changed, 61 insertions(+), 8 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 347fe1b99..c853f609d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -513,7 +513,6 @@ private: AutoCloseFD fdUserLock; bool isEnabled = false; - string user; uid_t uid = 0; gid_t gid = 0; std::vector supplementaryGIDs; @@ -523,9 +522,9 @@ public: void kill(); - string getUser() { return user; } uid_t getUID() { assert(uid); return uid; } - uid_t getGID() { assert(gid); return gid; } + gid_t getGID() { assert(gid); return gid; } + uint32_t getIDCount() { return 1; } std::vector getSupplementaryGIDs() { return supplementaryGIDs; } bool findFreeUser(); @@ -537,13 +536,16 @@ public: UserLock::UserLock() { +#if 0 assert(settings.buildUsersGroup != ""); createDirs(settings.nixStateDir + "/userpool"); +#endif } bool UserLock::findFreeUser() { if (enabled()) return true; +#if 0 /* Get the members of the build-users-group. */ struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) @@ -607,12 +609,46 @@ bool UserLock::findFreeUser() { } } + return false; +#endif + + assert(settings.startId > 0); + assert(settings.startId % settings.idsPerBuild == 0); + assert(settings.uidCount % settings.idsPerBuild == 0); + assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); + + // FIXME: check whether the id range overlaps any known users + + size_t nrSlots = settings.uidCount / settings.idsPerBuild; + + for (size_t i = 0; i < nrSlots; i++) { + debug("trying user slot '%d'", i); + + createDirs(settings.nixStateDir + "/userpool"); + + fnUserLock = fmt("%s/userpool/slot-%d", settings.nixStateDir, i); + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError("opening user lock '%1%'", fnUserLock); + + if (lockFile(fd.get(), ltWrite, false)) { + fdUserLock = std::move(fd); + uid = settings.startId + i * settings.idsPerBuild; + gid = settings.startId + i * settings.idsPerBuild; + return true; + } + } + return false; } void UserLock::kill() { + // FIXME: use a cgroup to kill all processes in the build? +#if 0 killUser(uid); +#endif } @@ -1523,7 +1559,7 @@ void DerivationGoal::tryLocalBuild() { /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { + if ((settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0) { #if defined(__linux__) || defined(__APPLE__) if (!buildUser) buildUser = std::make_unique(); @@ -2129,7 +2165,7 @@ void DerivationGoal::startBuilder() printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - if (mkdir(chrootRootDir.c_str(), 0750) == -1) + if (mkdir(chrootRootDir.c_str(), 0755) == -1) throw SysError("cannot create '%1%'", chrootRootDir); if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) @@ -2444,14 +2480,15 @@ void DerivationGoal::startBuilder() the calling user (if build users are disabled). */ uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + uint32_t nrIds = settings.idsPerBuild; // FIXME writeFile("/proc/" + std::to_string(pid) + "/uid_map", - (format("%d %d 1") % sandboxUid % hostUid).str()); + fmt("%d %d %d", /* sandboxUid */ 0, hostUid, nrIds)); - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + //writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); writeFile("/proc/" + std::to_string(pid) + "/gid_map", - (format("%d %d 1") % sandboxGid % hostGid).str()); + fmt("%d %d %d", /* sandboxGid */ 0, hostGid, nrIds)); /* Save the mount namespace of the child. We have to do this *before* the child does a chroot. */ @@ -3306,10 +3343,16 @@ void DerivationGoal::runChild() /* Switch to the sandbox uid/gid in the user namespace, which corresponds to the build user or calling user in the parent namespace. */ +#if 0 if (setgid(sandboxGid) == -1) throw SysError("setgid failed"); if (setuid(sandboxUid) == -1) throw SysError("setuid failed"); +#endif + if (setgid(0) == -1) + throw SysError("setgid failed"); + if (setuid(0) == -1) + throw SysError("setuid failed"); setUser = false; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 58cf08763..7dc842bca 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -148,6 +148,16 @@ public: Setting buildUsersGroup{this, "", "build-users-group", "The Unix group that contains the build users."}; + #if __linux__ + const uint32_t idsPerBuild = 1 << 18; + + Setting startId{this, 872415232, "start-id", + "The first UID and GID to use for dynamic ID allocation. (0 means disable.)"}; + + Setting uidCount{this, idsPerBuild * 128, "id-count", + "The number of UIDs/GIDs to use for dynamic ID allocation."}; + #endif + Setting impersonateLinux26{this, false, "impersonate-linux-26", "Whether to impersonate a Linux 2.6 machine on newer kernels.", {"build-impersonate-linux-26"}}; -- cgit v1.2.3 From c3e0a68c7eeeb4f491c0464392b2146ddec4305a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 May 2020 13:52:41 +0200 Subject: canonicalisePathMetaData(): Support a UID range --- src/libstore/build.cc | 12 +++++++++--- src/libstore/local-store.cc | 27 +++++++++++++++++---------- src/libstore/local-store.hh | 15 ++++++++++++--- 3 files changed, 38 insertions(+), 16 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index c853f609d..4e654e8ad 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -524,7 +524,7 @@ public: uid_t getUID() { assert(uid); return uid; } gid_t getGID() { assert(gid); return gid; } - uint32_t getIDCount() { return 1; } + uint32_t getIDCount() { return settings.idsPerBuild; } std::vector getSupplementaryGIDs() { return supplementaryGIDs; } bool findFreeUser(); @@ -3744,7 +3744,10 @@ void DerivationGoal::registerOutputs() /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or something like that. */ - canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); + canonicalisePathMetaData( + actualPath, + buildUser ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1)) : std::nullopt, + inodesSeen); /* FIXME: this is in-memory. */ StringSink sink; @@ -3819,7 +3822,10 @@ void DerivationGoal::registerOutputs() /* Get rid of all weird permissions. This also checks that all files are owned by the build user, if applicable. */ canonicalisePathMetaData(actualPath, - buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen); + buildUser && !rewritten + ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1)) + : std::nullopt, + inodesSeen); /* For this output path, find the references to other paths contained in it. Compute the SHA-256 NAR hash at the same diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eed225349..80ebe903f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -424,7 +424,10 @@ void canonicaliseTimestampAndPermissions(const Path & path) } -static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +static void canonicalisePathMetaData_( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) { checkInterrupt(); @@ -475,7 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe However, ignore files that we chown'ed ourselves previously to ensure that we don't fail on hard links within the same build (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { + if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { assert(!S_ISDIR(st.st_mode)); if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) throw BuildError("invalid ownership on file '%1%'", path); @@ -509,14 +512,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (S_ISDIR(st.st_mode)) { DirEntries entries = readDirectory(path); for (auto & i : entries) - canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); + canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); } } -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) { - canonicalisePathMetaData_(path, fromUid, inodesSeen); + canonicalisePathMetaData_(path, uidRange, inodesSeen); /* On platforms that don't have lchown(), the top-level path can't be a symlink, since we can't change its ownership. */ @@ -531,10 +537,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino } -void canonicalisePathMetaData(const Path & path, uid_t fromUid) +void canonicalisePathMetaData(const Path & path, + std::optional> uidRange) { InodesSeen inodesSeen; - canonicalisePathMetaData(path, fromUid, inodesSeen); + canonicalisePathMetaData(path, uidRange, inodesSeen); } @@ -1021,7 +1028,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, autoGC(); - canonicalisePathMetaData(realPath, -1); + canonicalisePathMetaData(realPath, {}); optimisePath(realPath); // FIXME: combine with hashPath() @@ -1064,7 +1071,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam } else writeFile(realPath, dump); - canonicalisePathMetaData(realPath, -1); + canonicalisePathMetaData(realPath, {}); /* Register the SHA-256 hash of the NAR serialisation of the path in the database. We may just have computed it @@ -1134,7 +1141,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, writeFile(realPath, s); - canonicalisePathMetaData(realPath, -1); + canonicalisePathMetaData(realPath, {}); StringSink sink; dumpString(s, sink); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ff36cb00e..79b415875 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -311,9 +311,18 @@ typedef set InodesSeen; - the permissions are set of 444 or 555 (i.e., read-only with or without execute permission; setuid bits etc. are cleared) - the owner and group are set to the Nix user and group, if we're - running as root. */ -void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); -void canonicalisePathMetaData(const Path & path, uid_t fromUid); + running as root. + If uidRange is not empty, this function will throw an error if it + encounters files owned by a user outside of the closed interval + [uidRange->first, uidRange->second]. +*/ +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen); +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange); void canonicaliseTimestampAndPermissions(const Path & path); -- cgit v1.2.3 From f5fa3de759a2b4c1d0107a4304a0b3f9571c87b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 15 May 2020 00:11:59 +0200 Subject: Run builds in their own cgroup Also, run builds in a cgroup namespace (ensuring /proc/self/cgroup doesn't leak information about the outside world) and mount /sys. This enables running systemd-nspawn and thus NixOS containers in a Nix build. --- src/libstore/build.cc | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 4e654e8ad..816d695a5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2168,7 +2168,8 @@ void DerivationGoal::startBuilder() if (mkdir(chrootRootDir.c_str(), 0755) == -1) throw SysError("cannot create '%1%'", chrootRootDir); - if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) + // FIXME: only make root writable for user namespace builds. + if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) throw SysError("cannot change ownership of '%1%'", chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need @@ -2182,6 +2183,7 @@ void DerivationGoal::startBuilder() nobody account. The latter is kind of a hack to support Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); + chownToBuilder(chrootRootDir + "/etc"); writeFile(chrootRootDir + "/etc/passwd", fmt( "root:x:0:0:Nix build user:%3%:/noshell\n" @@ -2372,6 +2374,52 @@ void DerivationGoal::startBuilder() #if __linux__ if (useChroot) { + /* Create a cgroup. */ + // FIXME: do we want to use the parent cgroup? We should + // always use the same cgroup regardless of whether we're the + // daemon or run from a user session via sudo. + std::string msg; + std::vector cgroups; + for (auto & line : tokenizeString>(readFile("/proc/self/cgroup"), "\n")) { + static std::regex regex("([0-9]+):([^:]*):(.*)"); + std::smatch match; + if (!std::regex_match(line, match, regex)) + throw Error("invalid line '%s' in '/proc/self/cgroup'", line); + + /* We only create a systemd cgroup, since that's enough + for running systemd-nspawn. */ + std::string name; + if (match[2] == "name=systemd") + name = "systemd"; + //else if (match[2] == "") + // name = "unified"; + else continue; + + std::string cgroup = match[3]; + + auto hostCgroup = canonPath("/sys/fs/cgroup/" + name + "/" + cgroup); + + if (!pathExists(hostCgroup)) + throw Error("expected unified cgroup directory '%s'", hostCgroup); + + auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID()); + + // FIXME: if the cgroup already exists, kill all processes + // in it and destroy it. + + if (mkdir(childCgroup.c_str(), 0755) == -1 && errno != EEXIST) + throw SysError("creating cgroup '%s'", childCgroup); + + chownToBuilder(childCgroup); + chownToBuilder(childCgroup + "/cgroup.procs"); + if (name == "unified") { + chownToBuilder(childCgroup + "/cgroup.threads"); + chownToBuilder(childCgroup + "/cgroup.subtree_control"); + } + + cgroups.push_back(childCgroup); + } + /* Set up private namespaces for the build: - The PID namespace causes the build to start as PID 1. @@ -2496,6 +2544,10 @@ void DerivationGoal::startBuilder() if (sandboxMountNamespace.get() == -1) throw SysError("getting sandbox mount namespace"); + /* Move the child into its own cgroup. */ + for (auto & childCgroup : cgroups) + writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); userNamespaceSync.writeSide = -1; @@ -3279,6 +3331,12 @@ void DerivationGoal::runChild() if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) throw SysError("mounting /proc"); + /* Mount sysfs on /sys. FIXME: only in user namespace + builds. */ + createDirs(chrootRootDir + "/sys"); + if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) + throw SysError("mounting /sys"); + /* Mount a new tmpfs on /dev/shm to ensure that whatever the builder puts in /dev/shm is cleaned up automatically. */ if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, @@ -3321,6 +3379,12 @@ void DerivationGoal::runChild() if (unshare(CLONE_NEWNS) == -1) throw SysError("unsharing mount namespace"); + /* Unshare the cgroup namespace. This means + /proc/self/cgroup will show the child's cgroup as '/' + rather than whatever it is in the parent. */ + if (unshare(CLONE_NEWCGROUP) == -1) + throw SysError("unsharing cgroup namespace"); + /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) throw SysError("cannot change directory to '%1%'", chrootRootDir); -- cgit v1.2.3 From ca2f64bcdaef5915f5147eac935ecb770511e438 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 15 May 2020 00:32:44 +0200 Subject: Reduce # of UIDs per build to 65536 2^18 was overkill. The idea was to enable multiple containers to run inside a build. However, those containers can use the same UID range - we don't really care about perfect isolation between containers inside a build. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7dc842bca..89db072b0 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -149,7 +149,7 @@ public: "The Unix group that contains the build users."}; #if __linux__ - const uint32_t idsPerBuild = 1 << 18; + const uint32_t idsPerBuild = 1 << 16; Setting startId{this, 872415232, "start-id", "The first UID and GID to use for dynamic ID allocation. (0 means disable.)"}; -- cgit v1.2.3 From 7bdcf43b401eba6aee29a359c5bce1f9cc01ce52 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 16 May 2020 21:09:48 +0200 Subject: Destroy the cgroup prior to building --- src/libstore/build.cc | 8 ++++---- src/libstore/cgroup.cc | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ src/libstore/cgroup.hh | 13 +++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 src/libstore/cgroup.cc create mode 100644 src/libstore/cgroup.hh (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 816d695a5..1f1468d97 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -16,6 +16,7 @@ #include "machines.hh" #include "daemon.hh" #include "worker-protocol.hh" +#include "cgroup.hh" #include #include @@ -2400,14 +2401,13 @@ void DerivationGoal::startBuilder() auto hostCgroup = canonPath("/sys/fs/cgroup/" + name + "/" + cgroup); if (!pathExists(hostCgroup)) - throw Error("expected unified cgroup directory '%s'", hostCgroup); + throw Error("expected cgroup directory '%s'", hostCgroup); auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID()); - // FIXME: if the cgroup already exists, kill all processes - // in it and destroy it. + destroyCgroup(childCgroup); - if (mkdir(childCgroup.c_str(), 0755) == -1 && errno != EEXIST) + if (mkdir(childCgroup.c_str(), 0755) == -1) throw SysError("creating cgroup '%s'", childCgroup); chownToBuilder(childCgroup); diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc new file mode 100644 index 000000000..8cd682e68 --- /dev/null +++ b/src/libstore/cgroup.cc @@ -0,0 +1,49 @@ +#if __linux__ + +#include "cgroup.hh" +#include "util.hh" + +#include + +#include + +namespace nix { + +void destroyCgroup(const Path & cgroup) +{ + for (auto & entry : readDirectory(cgroup)) { + if (entry.type != DT_DIR) continue; + destroyCgroup(cgroup + "/" + entry.name); + } + + int round = 1; + + while (true) { + auto pids = tokenizeString>(readFile(cgroup + "/cgroup.procs")); + + if (pids.empty()) break; + + if (round > 20) + throw Error("cannot kill cgroup '%s'", cgroup); + + for (auto & pid_s : pids) { + pid_t pid; + if (!string2Int(pid_s, pid)) throw Error("invalid pid '%s'", pid); + // FIXME: pid wraparound + if (kill(pid, SIGKILL) == -1 && errno != ESRCH) + throw SysError("killing member %d of cgroup '%s'", pid, cgroup); + } + + auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10))); + printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup); + std::this_thread::sleep_for(sleep); + round++; + } + + if (rmdir(cgroup.c_str()) == -1) + throw SysError("deleting cgroup '%s'", cgroup); +} + +} + +#endif diff --git a/src/libstore/cgroup.hh b/src/libstore/cgroup.hh new file mode 100644 index 000000000..c7b09398e --- /dev/null +++ b/src/libstore/cgroup.hh @@ -0,0 +1,13 @@ +#pragma once + +#if __linux__ + +#include "types.hh" + +namespace nix { + +void destroyCgroup(const Path & cgroup); + +} + +#endif -- cgit v1.2.3 From 570c443f560e015cf02e4f96102eaaa0e6853562 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 16 May 2020 21:21:41 +0200 Subject: Simplify cgroup creation --- src/libstore/build.cc | 63 +++++++++++++++++--------------------------------- src/libstore/cgroup.cc | 17 ++++++++++++++ src/libstore/cgroup.hh | 2 ++ 3 files changed, 40 insertions(+), 42 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1f1468d97..97554e9cf 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2375,50 +2375,30 @@ void DerivationGoal::startBuilder() #if __linux__ if (useChroot) { - /* Create a cgroup. */ + /* Create a systemd cgroup since that's the minimum required + by systemd-nspawn. */ // FIXME: do we want to use the parent cgroup? We should // always use the same cgroup regardless of whether we're the // daemon or run from a user session via sudo. - std::string msg; - std::vector cgroups; - for (auto & line : tokenizeString>(readFile("/proc/self/cgroup"), "\n")) { - static std::regex regex("([0-9]+):([^:]*):(.*)"); - std::smatch match; - if (!std::regex_match(line, match, regex)) - throw Error("invalid line '%s' in '/proc/self/cgroup'", line); - - /* We only create a systemd cgroup, since that's enough - for running systemd-nspawn. */ - std::string name; - if (match[2] == "name=systemd") - name = "systemd"; - //else if (match[2] == "") - // name = "unified"; - else continue; - - std::string cgroup = match[3]; - - auto hostCgroup = canonPath("/sys/fs/cgroup/" + name + "/" + cgroup); - - if (!pathExists(hostCgroup)) - throw Error("expected cgroup directory '%s'", hostCgroup); - - auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID()); - - destroyCgroup(childCgroup); - - if (mkdir(childCgroup.c_str(), 0755) == -1) - throw SysError("creating cgroup '%s'", childCgroup); - - chownToBuilder(childCgroup); - chownToBuilder(childCgroup + "/cgroup.procs"); - if (name == "unified") { - chownToBuilder(childCgroup + "/cgroup.threads"); - chownToBuilder(childCgroup + "/cgroup.subtree_control"); - } + auto ourCgroups = getCgroups("/proc/self/cgroup"); + auto systemdCgroup = ourCgroups["systemd"]; + if (systemdCgroup == "") + throw Error("'systemd' cgroup does not exist"); - cgroups.push_back(childCgroup); - } + auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup); + + if (!pathExists(hostCgroup)) + throw Error("expected cgroup directory '%s'", hostCgroup); + + auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID()); + + destroyCgroup(childCgroup); + + if (mkdir(childCgroup.c_str(), 0755) == -1) + throw SysError("creating cgroup '%s'", childCgroup); + + chownToBuilder(childCgroup); + chownToBuilder(childCgroup + "/cgroup.procs"); /* Set up private namespaces for the build: @@ -2545,8 +2525,7 @@ void DerivationGoal::startBuilder() throw SysError("getting sandbox mount namespace"); /* Move the child into its own cgroup. */ - for (auto & childCgroup : cgroups) - writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 8cd682e68..887facdca 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -9,6 +9,23 @@ namespace nix { +std::map getCgroups(const Path & cgroupFile) +{ + std::map cgroups; + + for (auto & line : tokenizeString>(readFile(cgroupFile), "\n")) { + static std::regex regex("([0-9]+):([^:]*):(.*)"); + std::smatch match; + if (!std::regex_match(line, match, regex)) + throw Error("invalid line '%s' in '%s'", line, cgroupFile); + + std::string name = hasPrefix(match[2], "name=") ? std::string(match[2], 5) : match[2]; + cgroups.insert_or_assign(name, match[3]); + } + + return cgroups; +} + void destroyCgroup(const Path & cgroup) { for (auto & entry : readDirectory(cgroup)) { diff --git a/src/libstore/cgroup.hh b/src/libstore/cgroup.hh index c7b09398e..dc6758957 100644 --- a/src/libstore/cgroup.hh +++ b/src/libstore/cgroup.hh @@ -6,6 +6,8 @@ namespace nix { +std::map getCgroups(const Path & cgroupFile); + void destroyCgroup(const Path & cgroup); } -- cgit v1.2.3 From ba50c3efa3b2394f5a8372939bc600008cd25e7e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 May 2020 23:25:44 +0200 Subject: Add "uid-range" and "systemd-cgroup" system features "uid-range" provides 65536 UIDs to a build and runs the build as root in its user namespace. "systemd-cgroup" allows the build to mount the systemd cgroup controller (needed for running systemd-nspawn and NixOS containers). Also, add a configuration option "auto-allocate-uids" which is needed to enable these features, and some experimental feature gates. So to enable support for containers you need the following in nix.conf: experimental-features = auto-allocate-uids systemd-cgroup auto-allocate-uids = true system-features = uid-range systemd-cgroup --- src/libstore/build.cc | 270 ++++++++++++---------------------------------- src/libstore/cgroup.cc | 19 +++- src/libstore/globals.hh | 5 +- src/libstore/user-lock.cc | 212 ++++++++++++++++++++++++++++++++++++ src/libstore/user-lock.hh | 39 +++++++ 5 files changed, 340 insertions(+), 205 deletions(-) create mode 100644 src/libstore/user-lock.cc create mode 100644 src/libstore/user-lock.hh (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 97554e9cf..1f79a8d2d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -16,7 +16,7 @@ #include "machines.hh" #include "daemon.hh" #include "worker-protocol.hh" -#include "cgroup.hh" +#include "user-lock.hh" #include #include @@ -504,154 +504,6 @@ void handleDiffHook( } } -////////////////////////////////////////////////////////////////////// - - -class UserLock -{ -private: - Path fnUserLock; - AutoCloseFD fdUserLock; - - bool isEnabled = false; - uid_t uid = 0; - gid_t gid = 0; - std::vector supplementaryGIDs; - -public: - UserLock(); - - void kill(); - - uid_t getUID() { assert(uid); return uid; } - gid_t getGID() { assert(gid); return gid; } - uint32_t getIDCount() { return settings.idsPerBuild; } - std::vector getSupplementaryGIDs() { return supplementaryGIDs; } - - bool findFreeUser(); - - bool enabled() { return isEnabled; } - -}; - - -UserLock::UserLock() -{ -#if 0 - assert(settings.buildUsersGroup != ""); - createDirs(settings.nixStateDir + "/userpool"); -#endif -} - -bool UserLock::findFreeUser() { - if (enabled()) return true; - -#if 0 - /* Get the members of the build-users-group. */ - struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); - if (!gr) - throw Error("the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup); - gid = gr->gr_gid; - - /* Copy the result of getgrnam. */ - Strings users; - for (char * * p = gr->gr_mem; *p; ++p) { - debug("found build user '%1%'", *p); - users.push_back(*p); - } - - if (users.empty()) - throw Error("the build users group '%1%' has no members", - settings.buildUsersGroup); - - /* Find a user account that isn't currently in use for another - build. */ - for (auto & i : users) { - debug("trying user '%1%'", i); - - struct passwd * pw = getpwnam(i.c_str()); - if (!pw) - throw Error("the user '%1%' in the group '%2%' does not exist", - i, settings.buildUsersGroup); - - - fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error("the Nix user should not be a member of '%1%'", - settings.buildUsersGroup); - -#if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); - - supplementaryGIDs.resize(ngroups); -#endif - - isEnabled = true; - return true; - } - } - - return false; -#endif - - assert(settings.startId > 0); - assert(settings.startId % settings.idsPerBuild == 0); - assert(settings.uidCount % settings.idsPerBuild == 0); - assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); - - // FIXME: check whether the id range overlaps any known users - - size_t nrSlots = settings.uidCount / settings.idsPerBuild; - - for (size_t i = 0; i < nrSlots; i++) { - debug("trying user slot '%d'", i); - - createDirs(settings.nixStateDir + "/userpool"); - - fnUserLock = fmt("%s/userpool/slot-%d", settings.nixStateDir, i); - - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError("opening user lock '%1%'", fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - uid = settings.startId + i * settings.idsPerBuild; - gid = settings.startId + i * settings.idsPerBuild; - return true; - } - } - - return false; -} - -void UserLock::kill() -{ - // FIXME: use a cgroup to kill all processes in the build? -#if 0 - killUser(uid); -#endif -} - ////////////////////////////////////////////////////////////////////// @@ -840,6 +692,13 @@ private: Path chrootRootDir; + /* Whether to give the build more than 1 UID. */ + bool useUidRange = false; + + /* Whether to make the 'systemd' cgroup controller available to + the build. */ + bool useSystemdCgroup = false; + /* RAII object to delete the chroot directory. */ std::shared_ptr autoDelChroot; @@ -896,8 +755,8 @@ private: result. */ std::map prevInfos; - const uid_t sandboxUid = 1000; - const gid_t sandboxGid = 100; + uid_t sandboxUid = -1; + gid_t sandboxGid = -1; const static Path homeDir; @@ -1445,6 +1304,7 @@ void DerivationGoal::inputsRealised() result = BuildResult(); } + void DerivationGoal::started() { auto msg = fmt( buildMode == bmRepair ? "repairing outputs of '%s'" : @@ -1459,6 +1319,7 @@ void DerivationGoal::started() { worker.updateProgress(); } + void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -1556,25 +1417,28 @@ void DerivationGoal::tryToBuild() worker.wakeUp(shared_from_this()); } + void DerivationGoal::tryLocalBuild() { /* If `build-users-group' is not empty, then we have to build as one of the members of that group. */ - if ((settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0) { + static bool useBuildUsers = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0; + if (useBuildUsers) { #if defined(__linux__) || defined(__APPLE__) - if (!buildUser) buildUser = std::make_unique(); + if (!buildUser) + buildUser = acquireUserLock(); - if (buildUser->findFreeUser()) { - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); - } else { + if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } + + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); #else /* Don't know how to block the creation of setuid/setgid binaries on this platform. */ @@ -2087,6 +1951,9 @@ void DerivationGoal::startBuilder() } } + useUidRange = parsedDrv->getRequiredSystemFeatures().count("uid-range"); + useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("systemd-cgroup"); + if (useChroot) { /* Allow a user-configurable set of directories from the @@ -2166,7 +2033,7 @@ void DerivationGoal::startBuilder() printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - if (mkdir(chrootRootDir.c_str(), 0755) == -1) + if (mkdir(chrootRootDir.c_str(), useUidRange ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); // FIXME: only make root writable for user namespace builds. @@ -2186,6 +2053,12 @@ void DerivationGoal::startBuilder() createDirs(chrootRootDir + "/etc"); chownToBuilder(chrootRootDir + "/etc"); + if (useUidRange && (!buildUser || buildUser->getUIDCount() < 65536)) + throw Error("feature 'uid-range' requires '%s' to be enabled", settings.autoAllocateUids.name); + + sandboxUid = useUidRange ? 0 : 1000; + sandboxGid = useUidRange ? 0 : 100; + writeFile(chrootRootDir + "/etc/passwd", fmt( "root:x:0:0:Nix build user:%3%:/noshell\n" "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" @@ -2238,12 +2111,32 @@ void DerivationGoal::startBuilder() for (auto & i : drv->outputs) dirsInChroot.erase(worker.store.printStorePath(i.second.path)); -#elif __APPLE__ - /* We don't really have any parent prep work to do (yet?) - All work happens in the child, instead. */ + if (useSystemdCgroup) { + settings.requireExperimentalFeature("systemd-cgroup"); + std::optional cgroup; + if (!buildUser || !(cgroup = buildUser->getCgroup())) + throw Error("feature 'systemd-cgroup' requires 'auto-allocate-uids = true' in nix.conf"); + chownToBuilder(*cgroup); + chownToBuilder(*cgroup + "/cgroup.procs"); + } + #else - throw Error("sandboxing builds is not supported on this platform"); + if (useUidRange) + throw Error("feature 'uid-range' is not supported on this platform"); + if (useSystemdCgroup) + throw Error("feature 'systemd-cgroup' is not supported on this platform"); + #if __APPLE__ + /* We don't really have any parent prep work to do (yet?) + All work happens in the child, instead. */ + #else + throw Error("sandboxing builds is not supported on this platform"); + #endif #endif + } else { + if (useUidRange) + throw Error("feature 'uid-range' is only supported in sandboxed builds"); + if (useSystemdCgroup) + throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds"); } if (needsHashRewrite()) { @@ -2375,31 +2268,6 @@ void DerivationGoal::startBuilder() #if __linux__ if (useChroot) { - /* Create a systemd cgroup since that's the minimum required - by systemd-nspawn. */ - // FIXME: do we want to use the parent cgroup? We should - // always use the same cgroup regardless of whether we're the - // daemon or run from a user session via sudo. - auto ourCgroups = getCgroups("/proc/self/cgroup"); - auto systemdCgroup = ourCgroups["systemd"]; - if (systemdCgroup == "") - throw Error("'systemd' cgroup does not exist"); - - auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup); - - if (!pathExists(hostCgroup)) - throw Error("expected cgroup directory '%s'", hostCgroup); - - auto childCgroup = fmt("%s/nix-%d", hostCgroup, buildUser->getUID()); - - destroyCgroup(childCgroup); - - if (mkdir(childCgroup.c_str(), 0755) == -1) - throw SysError("creating cgroup '%s'", childCgroup); - - chownToBuilder(childCgroup); - chownToBuilder(childCgroup + "/cgroup.procs"); - /* Set up private namespaces for the build: - The PID namespace causes the build to start as PID 1. @@ -2508,15 +2376,16 @@ void DerivationGoal::startBuilder() the calling user (if build users are disabled). */ uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - uint32_t nrIds = settings.idsPerBuild; // FIXME + uint32_t nrIds = buildUser && useUidRange ? buildUser->getUIDCount() : 1; writeFile("/proc/" + std::to_string(pid) + "/uid_map", - fmt("%d %d %d", /* sandboxUid */ 0, hostUid, nrIds)); + fmt("%d %d %d", sandboxUid, hostUid, nrIds)); - //writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + if (!useUidRange) + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); writeFile("/proc/" + std::to_string(pid) + "/gid_map", - fmt("%d %d %d", /* sandboxGid */ 0, hostGid, nrIds)); + fmt("%d %d %d", sandboxGid, hostGid, nrIds)); /* Save the mount namespace of the child. We have to do this *before* the child does a chroot. */ @@ -2525,7 +2394,10 @@ void DerivationGoal::startBuilder() throw SysError("getting sandbox mount namespace"); /* Move the child into its own cgroup. */ - writeFile(childCgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + if (buildUser) { + if (auto cgroup = buildUser->getCgroup()) + writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + } /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); @@ -3361,7 +3233,7 @@ void DerivationGoal::runChild() /* Unshare the cgroup namespace. This means /proc/self/cgroup will show the child's cgroup as '/' rather than whatever it is in the parent. */ - if (unshare(CLONE_NEWCGROUP) == -1) + if (useSystemdCgroup && unshare(CLONE_NEWCGROUP) == -1) throw SysError("unsharing cgroup namespace"); /* Do the chroot(). */ @@ -3386,16 +3258,10 @@ void DerivationGoal::runChild() /* Switch to the sandbox uid/gid in the user namespace, which corresponds to the build user or calling user in the parent namespace. */ -#if 0 if (setgid(sandboxGid) == -1) throw SysError("setgid failed"); if (setuid(sandboxUid) == -1) throw SysError("setuid failed"); -#endif - if (setgid(0) == -1) - throw SysError("setgid failed"); - if (setuid(0) == -1) - throw SysError("setuid failed"); setUser = false; } @@ -3789,7 +3655,7 @@ void DerivationGoal::registerOutputs() something like that. */ canonicalisePathMetaData( actualPath, - buildUser ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1)) : std::nullopt, + buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); /* FIXME: this is in-memory. */ @@ -3866,7 +3732,7 @@ void DerivationGoal::registerOutputs() all files are owned by the build user, if applicable. */ canonicalisePathMetaData(actualPath, buildUser && !rewritten - ? std::optional(std::make_pair(buildUser->getUID(), buildUser->getUID() + buildUser->getIDCount() - 1)) + ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 887facdca..9e5e937df 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -4,6 +4,7 @@ #include "util.hh" #include +#include #include @@ -19,7 +20,7 @@ std::map getCgroups(const Path & cgroupFile) if (!std::regex_match(line, match, regex)) throw Error("invalid line '%s' in '%s'", line, cgroupFile); - std::string name = hasPrefix(match[2], "name=") ? std::string(match[2], 5) : match[2]; + std::string name = hasPrefix(std::string(match[2]), "name=") ? std::string(match[2], 5) : match[2]; cgroups.insert_or_assign(name, match[3]); } @@ -28,6 +29,8 @@ std::map getCgroups(const Path & cgroupFile) void destroyCgroup(const Path & cgroup) { + if (!pathExists(cgroup)) return; + for (auto & entry : readDirectory(cgroup)) { if (entry.type != DT_DIR) continue; destroyCgroup(cgroup + "/" + entry.name); @@ -35,6 +38,8 @@ void destroyCgroup(const Path & cgroup) int round = 1; + std::unordered_set pidsShown; + while (true) { auto pids = tokenizeString>(readFile(cgroup + "/cgroup.procs")); @@ -46,13 +51,23 @@ void destroyCgroup(const Path & cgroup) for (auto & pid_s : pids) { pid_t pid; if (!string2Int(pid_s, pid)) throw Error("invalid pid '%s'", pid); + if (pidsShown.insert(pid).second) { + try { + auto cmdline = readFile(fmt("/proc/%d/cmdline", pid)); + using namespace std::string_literals; + warn("killing stray builder process %d (%s)...", + pid, trim(replaceStrings(cmdline, "\0"s, " "))); + } catch (SysError &) { + } + } // FIXME: pid wraparound if (kill(pid, SIGKILL) == -1 && errno != ESRCH) throw SysError("killing member %d of cgroup '%s'", pid, cgroup); } auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10))); - printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup); + if (sleep.count() > 100) + printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup); std::this_thread::sleep_for(sleep); round++; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 89db072b0..5cf73c7b4 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -149,10 +149,13 @@ public: "The Unix group that contains the build users."}; #if __linux__ + Setting autoAllocateUids{this, false, "auto-allocate-uids", + "Whether to allocate UIDs for builders automatically."}; + const uint32_t idsPerBuild = 1 << 16; Setting startId{this, 872415232, "start-id", - "The first UID and GID to use for dynamic ID allocation. (0 means disable.)"}; + "The first UID and GID to use for dynamic ID allocation."}; Setting uidCount{this, idsPerBuild * 128, "id-count", "The number of UIDs/GIDs to use for dynamic ID allocation."}; diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc new file mode 100644 index 000000000..8a09df4d1 --- /dev/null +++ b/src/libstore/user-lock.cc @@ -0,0 +1,212 @@ +#include "user-lock.hh" +#include "globals.hh" +#include "pathlocks.hh" +#include "cgroup.hh" + +namespace nix { + +struct SimpleUserLock : UserLock +{ + AutoCloseFD fdUserLock; + uid_t uid; + gid_t gid; + std::vector supplementaryGIDs; + + void kill() override + { + killUser(uid); + } + + std::pair getUIDRange() override + { + assert(uid); + return {uid, uid}; + } + + gid_t getGID() override { assert(gid); return gid; } + + std::vector getSupplementaryGIDs() override { return supplementaryGIDs; } + + static std::unique_ptr acquire() + { + assert(settings.buildUsersGroup != ""); + createDirs(settings.nixStateDir + "/userpool"); + + /* Get the members of the build-users-group. */ + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (!gr) + throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup); + + /* Copy the result of getgrnam. */ + Strings users; + for (char * * p = gr->gr_mem; *p; ++p) { + debug("found build user '%s'", *p); + users.push_back(*p); + } + + if (users.empty()) + throw Error("the build users group '%s' has no members", settings.buildUsersGroup); + + /* Find a user account that isn't currently in use for another + build. */ + for (auto & i : users) { + debug("trying user '%s'", i); + + struct passwd * pw = getpwnam(i.c_str()); + if (!pw) + throw Error("the user '%s' in the group '%s' does not exist", i, settings.buildUsersGroup); + + auto fnUserLock = fmt("%s/userpool/%s", settings.nixStateDir,pw->pw_uid); + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError("opening user lock '%s'", fnUserLock); + + if (lockFile(fd.get(), ltWrite, false)) { + auto lock = std::make_unique(); + + lock->fdUserLock = std::move(fd); + lock->uid = pw->pw_uid; + lock->gid = gr->gr_gid; + + /* Sanity check... */ + if (lock->uid == getuid() || lock->uid == geteuid()) + throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); + + #if __linux__ + /* Get the list of supplementary groups of this build + user. This is usually either empty or contains a + group such as "kvm". */ + lock->supplementaryGIDs.resize(10); + int ngroups = lock->supplementaryGIDs.size(); + int err = getgrouplist(pw->pw_name, pw->pw_gid, + lock->supplementaryGIDs.data(), &ngroups); + if (err == -1) + throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); + + lock->supplementaryGIDs.resize(ngroups); + #endif + + return lock; + } + } + + return nullptr; + } +}; + +#if __linux__ +struct CgroupUserLock : UserLock +{ + AutoCloseFD fdUserLock; + uid_t uid; + + void kill() override + { + if (cgroup) { + destroyCgroup(*cgroup); + cgroup.reset(); + } + } + + std::pair getUIDRange() override + { + assert(uid); + return {uid, uid + settings.idsPerBuild - 1}; + } + + gid_t getGID() override + { + // We use the same GID ranges as for the UIDs. + assert(uid); + return uid; + } + + std::vector getSupplementaryGIDs() override { return {}; } // FIXME + + static std::unique_ptr acquire() + { + settings.requireExperimentalFeature("auto-allocate-uids"); + assert(settings.startId > 0); + assert(settings.startId % settings.idsPerBuild == 0); + assert(settings.uidCount % settings.idsPerBuild == 0); + assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); + + // FIXME: check whether the id range overlaps any known users + + createDirs(settings.nixStateDir + "/userpool2"); + + size_t nrSlots = settings.uidCount / settings.idsPerBuild; + + for (size_t i = 0; i < nrSlots; i++) { + debug("trying user slot '%d'", i); + + createDirs(settings.nixStateDir + "/userpool2"); + + auto fnUserLock = fmt("%s/userpool2/slot-%d", settings.nixStateDir, i); + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError("opening user lock '%s'", fnUserLock); + + if (lockFile(fd.get(), ltWrite, false)) { + auto lock = std::make_unique(); + lock->fdUserLock = std::move(fd); + lock->uid = settings.startId + i * settings.idsPerBuild; + auto s = drainFD(lock->fdUserLock.get()); + if (s != "") lock->cgroup = s; + return lock; + } + } + + return nullptr; + } + + std::optional cgroup; + + std::optional getCgroup() override + { + if (!cgroup) { + /* Create a systemd cgroup since that's the minimum + required by systemd-nspawn. */ + auto ourCgroups = getCgroups("/proc/self/cgroup"); + auto systemdCgroup = ourCgroups["systemd"]; + if (systemdCgroup == "") + throw Error("'systemd' cgroup does not exist"); + + auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup); + + if (!pathExists(hostCgroup)) + throw Error("expected cgroup directory '%s'", hostCgroup); + + cgroup = fmt("%s/nix-%d", hostCgroup, uid); + + destroyCgroup(*cgroup); + + if (mkdir(cgroup->c_str(), 0755) == -1) + throw SysError("creating cgroup '%s'", *cgroup); + + /* Record the cgroup in the lock file. This ensures that + if we subsequently get executed under a different parent + cgroup, we kill the previous cgroup first. */ + if (ftruncate(fdUserLock.get(), 0) == -1) + throw Error("truncating user lock"); + writeFull(fdUserLock.get(), *cgroup); + } + + return cgroup; + }; +}; +#endif + +std::unique_ptr acquireUserLock() +{ + #if __linux__ + if (settings.autoAllocateUids) + return CgroupUserLock::acquire(); + else + #endif + return SimpleUserLock::acquire(); +} + +} diff --git a/src/libstore/user-lock.hh b/src/libstore/user-lock.hh new file mode 100644 index 000000000..88d068689 --- /dev/null +++ b/src/libstore/user-lock.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +struct UserLock +{ + virtual ~UserLock() { } + + /* Get the first and last UID. */ + virtual std::pair getUIDRange() = 0; + + /* Get the first UID. */ + uid_t getUID() + { + return getUIDRange().first; + } + + uid_t getUIDCount() + { + return getUIDRange().second - getUIDRange().first + 1; + } + + virtual gid_t getGID() = 0; + + virtual std::vector getSupplementaryGIDs() = 0; + + /* Kill any processes currently executing as this user. */ + virtual void kill() = 0; + + virtual std::optional getCgroup() { return {}; }; +}; + +/* Acquire a user lock. Note that this may return nullptr if no user + is available. */ +std::unique_ptr acquireUserLock(); + +} -- cgit v1.2.3 From 8c4cce553c16438f0ccbbaea6d77f2bd64306dfe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 May 2020 11:24:21 +0200 Subject: Fix macOS build --- src/libstore/build.cc | 11 +---------- src/libstore/user-lock.cc | 13 +++++++++++++ src/libstore/user-lock.hh | 2 ++ 3 files changed, 16 insertions(+), 10 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 1f79a8d2d..6c3f94a76 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1420,11 +1420,7 @@ void DerivationGoal::tryToBuild() void DerivationGoal::tryLocalBuild() { - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - static bool useBuildUsers = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0; - if (useBuildUsers) { -#if defined(__linux__) || defined(__APPLE__) + if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(); @@ -1439,11 +1435,6 @@ void DerivationGoal::tryLocalBuild() { /* Make sure that no other processes are executing under this uid. */ buildUser->kill(); -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif } actLock.reset(); diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc index 8a09df4d1..2254386da 100644 --- a/src/libstore/user-lock.cc +++ b/src/libstore/user-lock.cc @@ -209,4 +209,17 @@ std::unique_ptr acquireUserLock() return SimpleUserLock::acquire(); } +bool useBuildUsers() +{ + #if __linux__ + static bool b = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0; + return b; + #elif __APPLE__ + static bool b = settings.buildUsersGroup != "" && getuid() == 0; + return b; + #else + return false; + #endif +} + } diff --git a/src/libstore/user-lock.hh b/src/libstore/user-lock.hh index 88d068689..bfb55b0d9 100644 --- a/src/libstore/user-lock.hh +++ b/src/libstore/user-lock.hh @@ -36,4 +36,6 @@ struct UserLock is available. */ std::unique_ptr acquireUserLock(); +bool useBuildUsers(); + } -- cgit v1.2.3 From 7349f257da8278af9aae35544b15c9a204e2a57b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 May 2020 11:57:33 +0200 Subject: Only mount /sys in uid-range builds Maybe this should be a separate system feature... /sys exposes a lot of impure info about the host system. --- src/libstore/build.cc | 11 ++++++----- src/libstore/user-lock.cc | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 6c3f94a76..e927a65f0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3173,11 +3173,12 @@ void DerivationGoal::runChild() if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) throw SysError("mounting /proc"); - /* Mount sysfs on /sys. FIXME: only in user namespace - builds. */ - createDirs(chrootRootDir + "/sys"); - if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) - throw SysError("mounting /sys"); + /* Mount sysfs on /sys. */ + if (useUidRange) { + createDirs(chrootRootDir + "/sys"); + if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) + throw SysError("mounting /sys"); + } /* Mount a new tmpfs on /dev/shm to ensure that whatever the builder puts in /dev/shm is cleaned up automatically. */ diff --git a/src/libstore/user-lock.cc b/src/libstore/user-lock.cc index 2254386da..fb2a45f48 100644 --- a/src/libstore/user-lock.cc +++ b/src/libstore/user-lock.cc @@ -122,7 +122,7 @@ struct CgroupUserLock : UserLock return uid; } - std::vector getSupplementaryGIDs() override { return {}; } // FIXME + std::vector getSupplementaryGIDs() override { return {}; } static std::unique_ptr acquire() { -- cgit v1.2.3 From 417aaf4ff7ac1ca501c5a460775fa25d8e078c8a Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 22 Dec 2021 11:31:14 +0100 Subject: Correctly hijack the `file://` uri scheme with `_NIX_FORCE_HTTP` Setting the `_NIX_FORCE_HTTP` environment variable is supposed to force `file://` store urls to use the `HttpBinaryCacheStore` implementation rather than the `LocalBinaryCacheStore` one (very useful for testing). However because of a name mismatch, the `LocalBinaryCacheStore` was still registering the `file` scheme when this variable was set, meaning that the actual store implementation picked up on `file://` uris was dependent on the registration order of the stores (itself dependent on the link order of the object files). Fix this by making the `LocalBinaryCacheStore` gracefully not register the `file` uri scheme when the variable is set. --- src/libstore/local-binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index f754770f9..a3c3e4806 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -107,7 +107,7 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) std::set LocalBinaryCacheStore::uriSchemes() { - if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1") + if (getEnv("_NIX_FORCE_HTTP") == "1") return {}; else return {"file"}; -- cgit v1.2.3 From 75b62e52600a44b42693944b50638bf580a2c86e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jun 2020 17:54:16 +0000 Subject: Avoid `fmt` when constructor already does it There is a correctnes issue here, but #3724 will fix that. This is just a cleanup for brevity's sake. --- src/libstore/filetransfer.cc | 13 +++++----- src/libstore/local-store.cc | 6 ++--- src/libstore/sqlite.cc | 57 +++++++++++++++++++++++++++----------------- src/libstore/sqlite.hh | 16 ++++++++++--- 4 files changed, 57 insertions(+), 35 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c46262299..529a41891 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -443,14 +443,13 @@ struct curlFileTransfer : public FileTransfer : httpStatus != 0 ? FileTransferError(err, std::move(response), - fmt("unable to %s '%s': HTTP error %d ('%s')", - request.verb(), request.uri, httpStatus, statusMsg) - + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) - ) + "unable to %s '%s': HTTP error %d%s", + request.verb(), request.uri, httpStatus, + code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError(err, std::move(response), - fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code)); + "unable to %s '%s': %s (%d)", + request.verb(), request.uri, curl_easy_strerror(code), code); /* If this is a transient error, then maybe retry the download after a while. If we're writing to a @@ -704,7 +703,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri); + throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri); res.data = std::move(*s3Res.data); callback(std::move(res)); #else diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d77fff963..ece5bb5ef 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -482,18 +482,18 @@ void LocalStore::openDB(State & state, bool create) SQLiteStmt stmt; stmt.create(db, "pragma main.journal_mode;"); if (sqlite3_step(stmt) != SQLITE_ROW) - throwSQLiteError(db, "querying journal mode"); + SQLiteError::throw_(db, "querying journal mode"); prevMode = std::string((const char *) sqlite3_column_text(stmt, 0)); } if (prevMode != mode && sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting journal mode"); + SQLiteError::throw_(db, "setting journal mode"); /* Increase the auto-checkpoint interval to 40000 pages. This seems enough to ensure that instantiating the NixOS system derivation is done in a single fsync(). */ if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting autocheckpoint interval"); + SQLiteError::throw_(db, "setting autocheckpoint interval"); /* Initialise the database schema, if necessary. */ if (create) { diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 1d82b4ab1..32d2fc021 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -8,22 +8,35 @@ namespace nix { -[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) +template +SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, const Args & ... args) + : Error(""), path(path), errNo(errNo), extendedErrNo(extendedErrNo) +{ + auto hf = hintfmt(args...); + err.msg = hintfmt("%s: %s (in '%s')", + normaltxt(hf.str()), + sqlite3_errstr(extendedErrNo), + path ? path : "(in-memory)"); +} + +template +[[noreturn]] void SQLiteError::throw_(sqlite3 * db, const std::string & fs, const Args & ... args) { int err = sqlite3_errcode(db); int exterr = sqlite3_extended_errcode(db); auto path = sqlite3_db_filename(db, nullptr); - if (!path) path = "(in-memory)"; if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - throw SQLiteBusy( + auto exp = SQLiteBusy(path, err, exterr, fs, args...); + exp.err.msg = hintfmt( err == SQLITE_PROTOCOL - ? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path) - : fmt("SQLite database '%s' is busy", path)); - } - else - throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); + ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" + : "SQLite database '%s' is busy", + path ? path : "(in-memory)"); + throw exp; + } else + throw SQLiteError(path, err, exterr, fs, args...); } SQLite::SQLite(const Path & path, bool create) @@ -37,7 +50,7 @@ SQLite::SQLite(const Path & path, bool create) throw Error("cannot open SQLite database '%s'", path); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); + SQLiteError::throw_(db, "setting timeout"); exec("pragma foreign_keys = 1"); } @@ -46,7 +59,7 @@ SQLite::~SQLite() { try { if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); + SQLiteError::throw_(db, "closing database"); } catch (...) { ignoreException(); } @@ -62,7 +75,7 @@ void SQLite::exec(const std::string & stmt) { retrySQLite([&]() { if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt); + SQLiteError::throw_(db, "executing SQLite statement '%s'", stmt); }); } @@ -76,7 +89,7 @@ void SQLiteStmt::create(sqlite3 * db, const std::string & sql) checkInterrupt(); assert(!stmt); if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, fmt("creating statement '%s'", sql)); + SQLiteError::throw_(db, "creating statement '%s'", sql); this->db = db; this->sql = sql; } @@ -85,7 +98,7 @@ SQLiteStmt::~SQLiteStmt() { try { if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, fmt("finalizing statement '%s'", sql)); + SQLiteError::throw_(db, "finalizing statement '%s'", sql); } catch (...) { ignoreException(); } @@ -109,7 +122,7 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool not { if (notNull) { if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); return *this; @@ -119,7 +132,7 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size { if (notNull) { if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); return *this; @@ -129,7 +142,7 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { if (notNull) { if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); return *this; @@ -138,7 +151,7 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) SQLiteStmt::Use & SQLiteStmt::Use::bind() { if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); return *this; } @@ -152,14 +165,14 @@ void SQLiteStmt::Use::exec() int r = step(); assert(r != SQLITE_ROW); if (r != SQLITE_DONE) - throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt))); + SQLiteError::throw_(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt))); } bool SQLiteStmt::Use::next() { int r = step(); if (r != SQLITE_DONE && r != SQLITE_ROW) - throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt))); + SQLiteError::throw_(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt))); return r == SQLITE_ROW; } @@ -185,14 +198,14 @@ SQLiteTxn::SQLiteTxn(sqlite3 * db) { this->db = db; if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); + SQLiteError::throw_(db, "starting transaction"); active = true; } void SQLiteTxn::commit() { if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); + SQLiteError::throw_(db, "committing transaction"); active = false; } @@ -200,7 +213,7 @@ SQLiteTxn::~SQLiteTxn() { try { if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); + SQLiteError::throw_(db, "aborting transaction"); } catch (...) { ignoreException(); } diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 99f0d56ce..72ec302e1 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -96,10 +96,20 @@ struct SQLiteTxn }; -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); +struct SQLiteError : Error +{ + const char *path; + int errNo, extendedErrNo; + + template + [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args); -[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs); +protected: + template + SQLiteError(const char *path, int errNo, int extendedErrNo, const Args & ... args); +}; + +MakeError(SQLiteBusy, SQLiteError); void handleSQLiteBusy(const SQLiteBusy & e); -- cgit v1.2.3 From 05ec0beb40f5e4e162903570b68837b34811a02d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 20 Apr 2022 16:57:06 +0000 Subject: Move templated functions to `sqlite-impl.hh` This ensures that use-sites properly trigger new monomorphisations on one hand, and on the other hand keeps the main `sqlite.hh` clean and interface-only. I think that is good practice in general, but in this situation in particular we do indeed have `sqlite.hh` users that don't need the `throw_` function. --- src/libstore/local-store.cc | 1 + src/libstore/sqlite-impl.hh | 42 ++++++++++++++++++++++++++++++++++++++++++ src/libstore/sqlite.cc | 32 +------------------------------- 3 files changed, 44 insertions(+), 31 deletions(-) create mode 100644 src/libstore/sqlite-impl.hh (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ece5bb5ef..42cc30cbf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -10,6 +10,7 @@ #include "topo-sort.hh" #include "finally.hh" #include "compression.hh" +#include "sqlite-impl.hh" #include #include diff --git a/src/libstore/sqlite-impl.hh b/src/libstore/sqlite-impl.hh new file mode 100644 index 000000000..c0a99403b --- /dev/null +++ b/src/libstore/sqlite-impl.hh @@ -0,0 +1,42 @@ +#include "sqlite.hh" +#include "globals.hh" +#include "util.hh" + +#include + +#include + +namespace nix { + +template +SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, const Args & ... args) + : Error(""), path(path), errNo(errNo), extendedErrNo(extendedErrNo) +{ + auto hf = hintfmt(args...); + err.msg = hintfmt("%s: %s (in '%s')", + normaltxt(hf.str()), + sqlite3_errstr(extendedErrNo), + path ? path : "(in-memory)"); +} + +template +[[noreturn]] void SQLiteError::throw_(sqlite3 * db, const std::string & fs, const Args & ... args) +{ + int err = sqlite3_errcode(db); + int exterr = sqlite3_extended_errcode(db); + + auto path = sqlite3_db_filename(db, nullptr); + + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + auto exp = SQLiteBusy(path, err, exterr, fs, args...); + exp.err.msg = hintfmt( + err == SQLITE_PROTOCOL + ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" + : "SQLite database '%s' is busy", + path ? path : "(in-memory)"); + throw exp; + } else + throw SQLiteError(path, err, exterr, fs, args...); +} + +} diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 32d2fc021..80290fa87 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,4 +1,5 @@ #include "sqlite.hh" +#include "sqlite-impl.hh" #include "globals.hh" #include "util.hh" @@ -8,37 +9,6 @@ namespace nix { -template -SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, const Args & ... args) - : Error(""), path(path), errNo(errNo), extendedErrNo(extendedErrNo) -{ - auto hf = hintfmt(args...); - err.msg = hintfmt("%s: %s (in '%s')", - normaltxt(hf.str()), - sqlite3_errstr(extendedErrNo), - path ? path : "(in-memory)"); -} - -template -[[noreturn]] void SQLiteError::throw_(sqlite3 * db, const std::string & fs, const Args & ... args) -{ - int err = sqlite3_errcode(db); - int exterr = sqlite3_extended_errcode(db); - - auto path = sqlite3_db_filename(db, nullptr); - - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - auto exp = SQLiteBusy(path, err, exterr, fs, args...); - exp.err.msg = hintfmt( - err == SQLITE_PROTOCOL - ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" - : "SQLite database '%s' is busy", - path ? path : "(in-memory)"); - throw exp; - } else - throw SQLiteError(path, err, exterr, fs, args...); -} - SQLite::SQLite(const Path & path, bool create) { // useSQLiteWAL also indicates what virtual file system we need. Using -- cgit v1.2.3 From f63b0f4540b61d8cdac7a3c52ca6e190f7c1b8cf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 20 Apr 2022 17:37:59 +0000 Subject: Actually, solve this in a lighter-weight way The templating is very superficial --- src/libstore/local-store.cc | 1 - src/libstore/sqlite-impl.hh | 42 ------------------------------------------ src/libstore/sqlite.cc | 29 ++++++++++++++++++++++++++++- src/libstore/sqlite.hh | 14 ++++++++++++-- 4 files changed, 40 insertions(+), 46 deletions(-) delete mode 100644 src/libstore/sqlite-impl.hh (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 42cc30cbf..ece5bb5ef 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -10,7 +10,6 @@ #include "topo-sort.hh" #include "finally.hh" #include "compression.hh" -#include "sqlite-impl.hh" #include #include diff --git a/src/libstore/sqlite-impl.hh b/src/libstore/sqlite-impl.hh deleted file mode 100644 index c0a99403b..000000000 --- a/src/libstore/sqlite-impl.hh +++ /dev/null @@ -1,42 +0,0 @@ -#include "sqlite.hh" -#include "globals.hh" -#include "util.hh" - -#include - -#include - -namespace nix { - -template -SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, const Args & ... args) - : Error(""), path(path), errNo(errNo), extendedErrNo(extendedErrNo) -{ - auto hf = hintfmt(args...); - err.msg = hintfmt("%s: %s (in '%s')", - normaltxt(hf.str()), - sqlite3_errstr(extendedErrNo), - path ? path : "(in-memory)"); -} - -template -[[noreturn]] void SQLiteError::throw_(sqlite3 * db, const std::string & fs, const Args & ... args) -{ - int err = sqlite3_errcode(db); - int exterr = sqlite3_extended_errcode(db); - - auto path = sqlite3_db_filename(db, nullptr); - - if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - auto exp = SQLiteBusy(path, err, exterr, fs, args...); - exp.err.msg = hintfmt( - err == SQLITE_PROTOCOL - ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" - : "SQLite database '%s' is busy", - path ? path : "(in-memory)"); - throw exp; - } else - throw SQLiteError(path, err, exterr, fs, args...); -} - -} diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 80290fa87..2090beabd 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,5 +1,4 @@ #include "sqlite.hh" -#include "sqlite-impl.hh" #include "globals.hh" #include "util.hh" @@ -9,6 +8,34 @@ namespace nix { +SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf) + : Error(""), path(path), errNo(errNo), extendedErrNo(extendedErrNo) +{ + err.msg = hintfmt("%s: %s (in '%s')", + normaltxt(hf.str()), + sqlite3_errstr(extendedErrNo), + path ? path : "(in-memory)"); +} + +[[noreturn]] void SQLiteError::throw_(sqlite3 * db, hintformat && hf) +{ + int err = sqlite3_errcode(db); + int exterr = sqlite3_extended_errcode(db); + + auto path = sqlite3_db_filename(db, nullptr); + + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + auto exp = SQLiteBusy(path, err, exterr, std::move(hf)); + exp.err.msg = hintfmt( + err == SQLITE_PROTOCOL + ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" + : "SQLite database '%s' is busy", + path ? path : "(in-memory)"); + throw exp; + } else + throw SQLiteError(path, err, exterr, std::move(hf)); +} + SQLite::SQLite(const Path & path, bool create) { // useSQLiteWAL also indicates what virtual file system we need. Using diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 72ec302e1..3a4ad8633 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -102,11 +102,21 @@ struct SQLiteError : Error int errNo, extendedErrNo; template - [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args); + [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args) { + throw_(db, hintfmt(fs, args...)); + } protected: + + SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf); + template - SQLiteError(const char *path, int errNo, int extendedErrNo, const Args & ... args); + SQLiteError(const char *path, int errNo, int extendedErrNo, const std::string & fs, const Args & ... args) + : SQLiteError(path, errNo, extendedErrNo, hintfmt(fs, args...)) + { } + + [[noreturn]] static void throw_(sqlite3 * db, hintformat && hf); + }; MakeError(SQLiteBusy, SQLiteError); -- cgit v1.2.3 From e7d79c78616425cbbea6619ea28ea9a5ec75cabe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 21 Apr 2022 09:40:55 +0200 Subject: Make the default SQLiteError constructor public Otherwise the clang builds fail because the constructor of `SQLiteBusy` inherits it, `SQLiteError::_throw` tries to call it, which fails. Strangely, gcc works fine with it. Not sure what the correct behavior is and who is buggy here, but either way, making it public is at the worst a reasonable workaround --- src/libstore/sqlite.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 3a4ad8633..1d1c553ea 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -106,10 +106,10 @@ struct SQLiteError : Error throw_(db, hintfmt(fs, args...)); } -protected: - SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf); +protected: + template SQLiteError(const char *path, int errNo, int extendedErrNo, const std::string & fs, const Args & ... args) : SQLiteError(path, errNo, extendedErrNo, hintfmt(fs, args...)) -- cgit v1.2.3 From 92656da0b953b92228ef4a252d504c07710e4b47 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 3 Nov 2021 16:30:22 +0100 Subject: Fix the gc with indirect self-references via the realisations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the derivation `foo` depends on `bar`, and they both have the same output path (because they are CA derivations), then this output path will depend both on the realisation of `foo` and of `bar`, which themselves depend on each other. This confuses SQLite which isn’t able to automatically solve this diamond dependency scheme. Help it by adding a trigger to delete all the references between the relevant realisations. Fix #5320 --- src/libstore/ca-specific-schema.sql | 13 +++++++++++++ src/libstore/local-store.cc | 15 ++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 64cc97fde..d2ea347fb 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -13,6 +13,19 @@ create table if not exists Realisations ( create index if not exists IndexRealisations on Realisations(drvPath, outputName); +-- We can end-up in a weird edge-case where a path depends on itself because +-- it’s an output of a CA derivation, that happens to be the same as one of its +-- dependencies. +-- In that case we have a dependency loop (path -> realisation1 -> realisation2 +-- -> path) that we need to break by removing the dependencies between the +-- realisations +create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths + begin + delete from RealisationsRefs where realisationReference = ( + select id from Realisations where outputPath = old.id + ); + end; + create table if not exists RealisationsRefs ( referrer integer not null, realisationReference integer, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ece5bb5ef..e64917956 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -81,7 +81,7 @@ int getSchema(Path schemaPath) void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) { - const int nixCASchemaVersion = 3; + const int nixCASchemaVersion = 4; int curCASchema = getSchema(schemaPath); if (curCASchema != nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) { @@ -143,6 +143,19 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) )"); txn.commit(); } + if (curCASchema < 4) { + SQLiteTxn txn(db); + db.exec(R"( + create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths + begin + delete from RealisationsRefs where realisationReference = ( + select id from Realisations where outputPath = old.id + ); + end; + )"); + txn.commit(); + } + writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); lockFile(lockFd.get(), ltRead, true); } -- cgit v1.2.3 From 86d7a11c6b338a73c30476be13a6cac562e309c3 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 5 Nov 2021 11:17:22 +0100 Subject: Make sure to delete all the realisation refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting just one will only work in the test cases where I didn’t bother creating too many of them :p --- src/libstore/ca-specific-schema.sql | 2 +- src/libstore/local-store.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index d2ea347fb..c01dc5a76 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -21,7 +21,7 @@ create index if not exists IndexRealisations on Realisations(drvPath, outputName -- realisations create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths begin - delete from RealisationsRefs where realisationReference = ( + delete from RealisationsRefs where realisationReference in ( select id from Realisations where outputPath = old.id ); end; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e64917956..76524b01c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -148,7 +148,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) db.exec(R"( create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths begin - delete from RealisationsRefs where realisationReference = ( + delete from RealisationsRefs where realisationReference in ( select id from Realisations where outputPath = old.id ); end; -- cgit v1.2.3 From 975b0b52e74168b034ccff23827eff4d626f1c46 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Sat, 26 Mar 2022 16:18:51 +0000 Subject: ca: add sqlite index on `RealisationsRefs(realisationReference)` Without the change any CA deletion triggers linear scan on large RealisationsRefs table: sqlite>.eqp full sqlite> delete from RealisationsRefs where realisationReference IN ( select id from Realisations where outputPath = 1234567890 ); QUERY PLAN |--SCAN RealisationsRefs `--LIST SUBQUERY 1 `--SEARCH Realisations USING COVERING INDEX IndexRealisationsRefsOnOutputPath (outputPath=?) With the change it gets turned into a lookup: sqlite> CREATE INDEX IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); sqlite> delete from RealisationsRefs where realisationReference IN ( select id from Realisations where outputPath = 1234567890 ); QUERY PLAN |--SEARCH RealisationsRefs USING INDEX IndexRealisationsRefsRealisationReference (realisationReference=?) `--LIST SUBQUERY 1 `--SEARCH Realisations USING COVERING INDEX IndexRealisationsRefsOnOutputPath (outputPath=?) --- src/libstore/ca-specific-schema.sql | 2 ++ src/libstore/local-store.cc | 2 ++ 2 files changed, 4 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index c01dc5a76..4ca91f585 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -32,6 +32,8 @@ create table if not exists RealisationsRefs ( foreign key (referrer) references Realisations(id) on delete cascade, foreign key (realisationReference) references Realisations(id) on delete restrict ); +-- used by deletion trigger +create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); -- used by QueryRealisationReferences create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 76524b01c..5cc5c91cc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -152,6 +152,8 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) select id from Realisations where outputPath = old.id ); end; + -- used by deletion trigger + create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); )"); txn.commit(); } -- cgit v1.2.3 From 49119072e72ae160c22a2c0b963cfe732b2819e4 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Wed, 27 Apr 2022 17:55:04 -0700 Subject: local-derivation-goal.cc: seccomp filters for MIPS secondary arch/abi A mips64el Linux MIPS kernel can execute userspace code using any of three ABIs: mips64el-linux-*abin64 mips64el-linux-*abin32 mipsel-linux-* The first of these is the native 64-bit ABI, and the only ABI with 64-bit pointers; this is sometimes called "n64". The last of these is the old legacy 32-bit ABI, whose binaries can execute natively on 32-bit MIPS hardware; this is sometimes called "o32". The second ABI, "n32" is essentially the 64-bit ABI with 32-bit pointers and address space. Hardware 64-bit integer/floating arithmetic is still allowed, as well as the much larger mips64 register set and more-efficient calling convention. Let's enable seccomp filters for all of these. Likewise for big endian (mips64-linux-*). --- src/libstore/build/local-derivation-goal.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4c91fa4fb..f80a4678b 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1529,6 +1529,22 @@ void setupSeccomp() seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); + if (nativeSystem == "mips64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPS) != 0) + printError("unable to add mips seccomp architecture"); + + if (nativeSystem == "mips64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPS64N32) != 0) + printError("unable to add mips64-*abin32 seccomp architecture"); + + if (nativeSystem == "mips64el-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL) != 0) + printError("unable to add mipsel seccomp architecture"); + + if (nativeSystem == "mips64el-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL64N32) != 0) + printError("unable to add mips64el-*abin32 seccomp architecture"); + /* Prevent builders from creating setuid/setgid binaries. */ for (int perm : { S_ISUID, S_ISGID }) { if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, -- cgit v1.2.3 From 4a9623b129730d3af3e99773a9f2a53395435311 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Apr 2022 13:36:01 +0200 Subject: Fix passing $OUT_PATHS to the post-build hook Fixes #6446. --- src/libstore/build/derivation-goal.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 53f212c1d..d095a0f02 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -786,8 +786,7 @@ void runPostBuildHook( Store & store, Logger & logger, const StorePath & drvPath, - StorePathSet outputPaths -) + const StorePathSet & outputPaths) { auto hook = settings.postBuildHook; if (hook == "") @@ -906,7 +905,7 @@ void DerivationGoal::buildDone() auto builtOutputs = registerOutputs(); StorePathSet outputPaths; - for (auto & [_, output] : buildResult.builtOutputs) + for (auto & [_, output] : builtOutputs) outputPaths.insert(output.outPath); runPostBuildHook( worker.store, -- cgit v1.2.3 From 70a30dbc00bfeb8b3f5f9b137bf3176e8a0b9298 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 28 Apr 2022 14:37:05 +0200 Subject: Fix libcxx build Fixes #6458. --- src/libstore/s3.hh | 1 + 1 file changed, 1 insertion(+) (limited to 'src/libstore') diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index 3f55c74db..cdb3e5908 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -5,6 +5,7 @@ #include "ref.hh" #include +#include namespace Aws { namespace Client { class ClientConfiguration; } } namespace Aws { namespace S3 { class S3Client; } } -- cgit v1.2.3 From 4a79cba5118f29b896f3d50164beacd4901ab01f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 22 Apr 2022 15:17:01 +0200 Subject: Allow selecting derivation outputs using 'installable!outputs' E.g. 'nixpkgs#glibc^dev,static' or 'nixpkgs#glibc^*'. --- src/libstore/path-with-outputs.cc | 16 ++++++++++++ src/libstore/path-with-outputs.hh | 12 +++++++++ src/libstore/tests/path-with-outputs.cc | 46 +++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 src/libstore/tests/path-with-outputs.cc (limited to 'src/libstore') diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 078c117bd..7d180a0f6 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,6 +1,8 @@ #include "path-with-outputs.hh" #include "store-api.hh" +#include + namespace nix { std::string StorePathWithOutputs::to_string(const Store & store) const @@ -68,4 +70,18 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std: return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) }; } +std::pair parseOutputsSpec(const std::string & s) +{ + static std::regex regex(R"((.*)\^((\*)|([a-z]+(,[a-z]+)*)))"); + + std::smatch match; + if (!std::regex_match(s, match, regex)) + return {s, DefaultOutputs()}; + + if (match[3].matched) + return {match[1], AllOutputs()}; + + return {match[1], tokenizeString(match[4].str(), ",")}; +} + } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 4c4023dcb..e4235d197 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -32,4 +32,16 @@ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); +typedef std::set OutputNames; + +struct AllOutputs { }; + +struct DefaultOutputs { }; + +typedef std::variant OutputsSpec; + +/* Parse a string of the form 'prefix^output1,...outputN' or + 'prefix^*', returning the prefix and the outputs spec. */ +std::pair parseOutputsSpec(const std::string & s); + } diff --git a/src/libstore/tests/path-with-outputs.cc b/src/libstore/tests/path-with-outputs.cc new file mode 100644 index 000000000..350ea7ffd --- /dev/null +++ b/src/libstore/tests/path-with-outputs.cc @@ -0,0 +1,46 @@ +#include "path-with-outputs.hh" + +#include + +namespace nix { + +TEST(parseOutputsSpec, basic) +{ + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo"); + ASSERT_EQ(prefix, "foo"); + ASSERT_TRUE(std::get_if(&outputsSpec)); + } + + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo^*"); + ASSERT_EQ(prefix, "foo"); + ASSERT_TRUE(std::get_if(&outputsSpec)); + } + + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo^out"); + ASSERT_EQ(prefix, "foo"); + ASSERT_TRUE(std::get(outputsSpec) == OutputNames({"out"})); + } + + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo^out,bin"); + ASSERT_EQ(prefix, "foo"); + ASSERT_TRUE(std::get(outputsSpec) == OutputNames({"out", "bin"})); + } + + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo^bar^out,bin"); + ASSERT_EQ(prefix, "foo^bar"); + ASSERT_TRUE(std::get(outputsSpec) == OutputNames({"out", "bin"})); + } + + { + auto [prefix, outputsSpec] = parseOutputsSpec("foo^&*()"); + ASSERT_EQ(prefix, "foo^&*()"); + ASSERT_TRUE(std::get_if(&outputsSpec)); + } +} + +} -- cgit v1.2.3 From a3c6c5b1c745a72a6a46bdf1a1de7a51a53f76b0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 May 2022 14:37:28 +0200 Subject: nix profile: Support overriding outputs --- src/libstore/path-with-outputs.cc | 40 +++++++++++++++++++++++++++++++++++++++ src/libstore/path-with-outputs.hh | 14 ++++++++++++-- 2 files changed, 52 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 7d180a0f6..d6d67ea05 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,5 +1,6 @@ #include "path-with-outputs.hh" #include "store-api.hh" +#include "nlohmann/json.hpp" #include @@ -84,4 +85,43 @@ std::pair parseOutputsSpec(const std::string & s) return {match[1], tokenizeString(match[4].str(), ",")}; } +std::string printOutputsSpec(const OutputsSpec & outputsSpec) +{ + if (std::get_if(&outputsSpec)) + return ""; + + if (std::get_if(&outputsSpec)) + return "^*"; + + if (auto outputNames = std::get_if(&outputsSpec)) + return "^" + concatStringsSep(",", *outputNames); + + assert(false); +} + +void to_json(nlohmann::json & json, const OutputsSpec & outputsSpec) +{ + if (std::get_if(&outputsSpec)) + json = nullptr; + + else if (std::get_if(&outputsSpec)) + json = std::vector({"*"}); + + else if (auto outputNames = std::get_if(&outputsSpec)) + json = *outputNames; +} + +void from_json(const nlohmann::json & json, OutputsSpec & outputsSpec) +{ + if (json.is_null()) + outputsSpec = DefaultOutputs(); + else { + auto names = json.get(); + if (names == OutputNames({"*"})) + outputsSpec = AllOutputs(); + else + outputsSpec = names; + } +} + } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index e4235d197..0cb5eb223 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -4,6 +4,7 @@ #include "path.hh" #include "derived-path.hh" +#include "nlohmann/json_fwd.hpp" namespace nix { @@ -34,9 +35,13 @@ StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std: typedef std::set OutputNames; -struct AllOutputs { }; +struct AllOutputs { + bool operator < (const AllOutputs & _) const { return false; } +}; -struct DefaultOutputs { }; +struct DefaultOutputs { + bool operator < (const DefaultOutputs & _) const { return false; } +}; typedef std::variant OutputsSpec; @@ -44,4 +49,9 @@ typedef std::variant OutputsSpec; 'prefix^*', returning the prefix and the outputs spec. */ std::pair parseOutputsSpec(const std::string & s); +std::string printOutputsSpec(const OutputsSpec & outputsSpec); + +void to_json(nlohmann::json &, const OutputsSpec &); +void from_json(const nlohmann::json &, OutputsSpec &); + } -- cgit v1.2.3 From 1385b2007804c8a0370f2a6555045a00e34b07c7 Mon Sep 17 00:00:00 2001 From: Alain Zscheile Date: Wed, 4 May 2022 07:44:32 +0200 Subject: Get rid of most `.at` calls (#6393) Use one of `get` or `getOr` instead which will either return a null-pointer (with a nicer error message) or a default value when the key is missing. --- src/libstore/build/derivation-goal.cc | 33 +++++--- src/libstore/build/local-derivation-goal.cc | 115 ++++++++++++++++------------ src/libstore/build/worker.cc | 5 +- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/derivations.cc | 13 +++- src/libstore/derived-path.cc | 23 ++++-- src/libstore/filetransfer.cc | 8 +- src/libstore/local-store.cc | 6 +- src/libstore/misc.cc | 9 ++- src/libstore/remote-store.cc | 17 ++-- src/libstore/store-api.cc | 2 +- 11 files changed, 144 insertions(+), 89 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d095a0f02..3fff2385f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -984,21 +984,28 @@ void DerivationGoal::resolvedFinished() realWantedOutputs = resolvedDrv.outputNames(); for (auto & wantedOutput : realWantedOutputs) { - assert(initialOutputs.count(wantedOutput) != 0); - assert(resolvedHashes.count(wantedOutput) != 0); - auto realisation = resolvedResult.builtOutputs.at( - DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput }); + auto initialOutput = get(initialOutputs, wantedOutput); + auto resolvedHash = get(resolvedHashes, wantedOutput); + if ((!initialOutput) || (!resolvedHash)) + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", + worker.store.printStorePath(drvPath), wantedOutput); + auto realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); + if (!realisation) + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", + worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); if (drv->type().isPure()) { - auto newRealisation = realisation; - newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput }; + auto newRealisation = *realisation; + newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput }; newRealisation.signatures.clear(); if (!drv->type().isFixed()) - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } - outputPaths.insert(realisation.outPath); - builtOutputs.emplace(realisation.id, realisation); + outputPaths.insert(realisation->outPath); + builtOutputs.emplace(realisation->id, *realisation); } runPostBuildHook( @@ -1294,7 +1301,11 @@ std::pair DerivationGoal::checkPathValidity() DrvOutputs validOutputs; for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput & info = initialOutputs.at(i.first); + auto initialOutput = get(initialOutputs, i.first); + if (!initialOutput) + // this is an invalid output, gets catched with (!wantedOutputsLeft.empty()) + continue; + auto & info = *initialOutput; info.wanted = wantOutput(i.first, wantedOutputs); if (info.wanted) wantedOutputsLeft.erase(i.first); @@ -1309,7 +1320,7 @@ std::pair DerivationGoal::checkPathValidity() : PathStatus::Corrupt, }; } - auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first}; + auto drvOutput = DrvOutput{info.outputHash, i.first}; if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4c91fa4fb..7f44276bd 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -482,7 +482,7 @@ void LocalDerivationGoal::startBuilder() temporary build directory. The text files have the format used by `nix-store --register-validity'. However, the deriver fields are left empty. */ - auto s = get(drv->env, "exportReferencesGraph").value_or(""); + auto s = getOr(drv->env, "exportReferencesGraph", ""); Strings ss = tokenizeString(s); if (ss.size() % 2 != 0) throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); @@ -989,7 +989,7 @@ void LocalDerivationGoal::initTmpDir() { there is no size constraint). */ if (!parsedDrv->getStructuredAttrs()) { - StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile").value_or("")); + StringSet passAsFile = tokenizeString(getOr(drv->env, "passAsFile", "")); for (auto & i : drv->env) { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; @@ -2128,12 +2128,22 @@ DrvOutputs LocalDerivationGoal::registerOutputs() std::map> outputReferencesIfUnregistered; std::map outputStats; for (auto & [outputName, _] : drv->outputs) { - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName))); + auto scratchOutput = get(scratchOutputs, outputName); + if (!scratchOutput) + throw BuildError( + "builder for '%s' has no scratch output for '%s'", + worker.store.printStorePath(drvPath), outputName); + auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchOutput)); outputsToSort.insert(outputName); /* Updated wanted info to remove the outputs we definitely don't need to register */ - auto & initialInfo = initialOutputs.at(outputName); + auto initialOutput = get(initialOutputs, outputName); + if (!initialOutput) + throw BuildError( + "builder for '%s' has no initial output for '%s'", + worker.store.printStorePath(drvPath), outputName); + auto & initialInfo = *initialOutput; /* Don't register if already valid, and not checking */ initialInfo.wanted = buildMode == bmCheck @@ -2185,6 +2195,11 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto sortedOutputNames = topoSort(outputsToSort, {[&](const std::string & name) { + auto orifu = get(outputReferencesIfUnregistered, name); + if (!orifu) + throw BuildError( + "no output reference for '%s' in build of '%s'", + name, worker.store.printStorePath(drvPath)); return std::visit(overloaded { /* Since we'll use the already installed versions of these, we can treat them as leaves and ignore any references they @@ -2199,7 +2214,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() referencedOutputs.insert(o); return referencedOutputs; }, - }, outputReferencesIfUnregistered.at(name)); + }, *orifu); }}, {[&](const std::string & path, const std::string & parent) { // TODO with more -vvvv also show the temporary paths for manual inspection. @@ -2213,9 +2228,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs() OutputPathMap finalOutputs; for (auto & outputName : sortedOutputNames) { - auto output = drv->outputs.at(outputName); - auto & scratchPath = scratchOutputs.at(outputName); - auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath)); + auto output = get(drv->outputs, outputName); + auto scratchPath = get(scratchOutputs, outputName); + assert(output && scratchPath); + auto actualPath = toRealPathChroot(worker.store.printStorePath(*scratchPath)); auto finish = [&](StorePath finalStorePath) { /* Store the final path */ @@ -2223,10 +2239,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs() /* The rewrite rule will be used in downstream outputs that refer to use. This is why the topological sort is essential to do first before this for loop. */ - if (scratchPath != finalStorePath) - outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() }; + if (*scratchPath != finalStorePath) + outputRewrites[std::string { scratchPath->hashPart() }] = std::string { finalStorePath.hashPart() }; }; + auto orifu = get(outputReferencesIfUnregistered, outputName); + assert(orifu); + std::optional referencesOpt = std::visit(overloaded { [&](const AlreadyRegistered & skippedFinalPath) -> std::optional { finish(skippedFinalPath.path); @@ -2235,7 +2254,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() [&](const PerhapsNeedToRegister & r) -> std::optional { return r.refs; }, - }, outputReferencesIfUnregistered.at(outputName)); + }, *orifu); if (!referencesOpt) continue; @@ -2268,25 +2287,29 @@ DrvOutputs LocalDerivationGoal::registerOutputs() for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; - if (r == scratchPath) + if (r == *scratchPath) { res.first = true; - else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); - else { - std::string newRef = outputRewrites.at(origHash); + } else if (auto outputRewrite = get(outputRewrites, origHash)) { + std::string newRef = *outputRewrite; newRef += '-'; newRef += name; res.second.insert(StorePath { newRef }); + } else { + res.second.insert(r); } } return res; }; auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo { - auto & st = outputStats.at(outputName); + auto st = get(outputStats, outputName); + if (!st) + throw BuildError( + "output path %1% without valid stats info", + actualPath); if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ - if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) + if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) throw BuildError( "output path '%1%' should be a non-executable regular file " "since recursive hashing is not enabled (outputHashMode=flat)", @@ -2294,7 +2317,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } rewriteOutput(); /* FIXME optimize and deduplicate with addToStore */ - std::string oldHashPart { scratchPath.hashPart() }; + std::string oldHashPart { scratchPath->hashPart() }; HashModuloSink caSink { outputHash.hashType, oldHashPart }; switch (outputHash.method) { case FileIngestionMethod::Recursive: @@ -2313,7 +2336,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() outputPathName(drv->name, outputName), refs.second, refs.first); - if (scratchPath != finalPath) { + if (*scratchPath != finalPath) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { StringSink sink; @@ -2354,9 +2377,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto requiredFinalPath = output.path; /* Preemptively add rewrite rule for final hash, as that is what the NAR hash will use rather than normalized-self references */ - if (scratchPath != requiredFinalPath) + if (*scratchPath != requiredFinalPath) outputRewrites.insert_or_assign( - std::string { scratchPath.hashPart() }, + std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); rewriteOutput(); auto narHashAndSize = hashPath(htSHA256, actualPath); @@ -2409,7 +2432,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() }); }, - }, output.raw()); + }, output->raw()); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ @@ -2425,7 +2448,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() derivations. */ PathLocks dynamicOutputLock; dynamicOutputLock.setDeletion(true); - auto optFixedPath = output.path(worker.store, drv->name, outputName); + auto optFixedPath = output->path(worker.store, drv->name, outputName); if (!optFixedPath || worker.store.printStorePath(*optFixedPath) != finalDestPath) { @@ -2491,11 +2514,10 @@ DrvOutputs LocalDerivationGoal::registerOutputs() /* For debugging, print out the referenced and unreferenced paths. */ for (auto & i : inputPaths) { - auto j = references.find(i); - if (j == references.end()) - debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); - else + if (references.count(i)) debug("referenced input: '%1%'", worker.store.printStorePath(i)); + else + debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); } if (curRound == nrRounds) { @@ -2612,9 +2634,11 @@ DrvOutputs LocalDerivationGoal::registerOutputs() DrvOutputs builtOutputs; for (auto & [outputName, newInfo] : infos) { + auto oldinfo = get(initialOutputs, outputName); + assert(oldinfo); auto thisRealisation = Realisation { .id = DrvOutput { - initialOutputs.at(outputName).outputHash, + oldinfo->outputHash, outputName }, .outPath = newInfo.path @@ -2710,9 +2734,10 @@ void LocalDerivationGoal::checkOutputs(const std::mappath); + else + throw BuildError("derivation contains an illegal reference specifier '%s'", i); } auto used = recursive @@ -2751,24 +2776,18 @@ void LocalDerivationGoal::checkOutputs(const std::mapgetStructuredAttrs()) { - auto outputChecks = structuredAttrs->find("outputChecks"); - if (outputChecks != structuredAttrs->end()) { - auto output = outputChecks->find(outputName); - - if (output != outputChecks->end()) { + if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { + if (auto output = get(*outputChecks, outputName)) { Checks checks; - auto maxSize = output->find("maxSize"); - if (maxSize != output->end()) + if (auto maxSize = get(*output, "maxSize")) checks.maxSize = maxSize->get(); - auto maxClosureSize = output->find("maxClosureSize"); - if (maxClosureSize != output->end()) + if (auto maxClosureSize = get(*output, "maxClosureSize")) checks.maxClosureSize = maxClosureSize->get(); - auto get = [&](const std::string & name) -> std::optional { - auto i = output->find(name); - if (i != output->end()) { + auto get_ = [&](const std::string & name) -> std::optional { + if (auto i = get(*output, name)) { Strings res; for (auto j = i->begin(); j != i->end(); ++j) { if (!j->is_string()) @@ -2781,10 +2800,10 @@ void LocalDerivationGoal::checkOutputs(const std::map fds2(j->fds); std::vector buffer(4096); for (auto & k : fds2) { - if (pollStatus.at(fdToPollStatus.at(k)).revents) { + const auto fdPollStatusId = get(fdToPollStatus, k); + assert(fdPollStatusId); + assert(*fdPollStatusId < pollStatus.size()); + if (pollStatus.at(*fdPollStatusId).revents) { ssize_t rd = ::read(k, buffer.data(), buffer.size()); // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index af3dfc409..7d7924d77 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -24,7 +24,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) Path storePath = getAttr("out"); auto mainUrl = getAttr("url"); - bool unpack = get(drv.env, "unpack").value_or("") == "1"; + bool unpack = getOr(drv.env, "unpack", "") == "1"; /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1c695de82..fe99c3c5e 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -661,8 +661,10 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut if (res.kind == DrvHash::Kind::Deferred) kind = DrvHash::Kind::Deferred; for (auto & outputName : inputOutputs) { - const auto h = res.hashes.at(outputName); - inputs2[h.to_string(Base16, false)].insert(outputName); + const auto h = get(res.hashes, outputName); + if (!h) + throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); + inputs2[h->to_string(Base16, false)].insert(outputName); } } @@ -836,8 +838,11 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { if (std::holds_alternative(output.raw())) { - auto & h = hashModulo.hashes.at(outputName); - auto outPath = store.makeOutputPath(outputName, h, drv.name); + auto h = get(hashModulo.hashes, outputName); + if (!h) + throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", + drv.name, outputName); + auto outPath = store.makeOutputPath(outputName, *h, drv.name); drv.env[outputName] = store.printStorePath(outPath); output = DerivationOutput::InputAddressed { .path = std::move(outPath), diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 319b1c790..44587ae78 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -4,6 +4,8 @@ #include +#include + namespace nix { nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { @@ -17,12 +19,12 @@ nlohmann::json DerivedPath::Built::toJSON(ref store) const { res["drvPath"] = store->printStorePath(drvPath); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); + const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); for (const auto& output : outputs) { - if (knownOutputs.at(output)) - res["outputs"][output] = store->printStorePath(knownOutputs.at(output).value()); - else - res["outputs"][output] = nullptr; + auto knownOutput = get(knownOutputs, output); + res["outputs"][output] = (knownOutput && *knownOutput) + ? store->printStorePath(**knownOutput) + : nullptr; } return res; } @@ -123,10 +125,15 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const for (auto& [outputName, outputPath] : p.outputs) { if (settings.isExperimentalFeatureEnabled( Xp::CaDerivations)) { + auto drvOutput = get(drvHashes, outputName); + if (!drvOutput) + throw Error( + "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", + store.printStorePath(p.drvPath), outputName); auto thisRealisation = store.queryRealisation( - DrvOutput{drvHashes.at(outputName), outputName}); - assert(thisRealisation); // We’ve built it, so we must h - // ve the realisation + DrvOutput{*drvOutput, outputName}); + assert(thisRealisation); // We’ve built it, so we must + // have the realisation res.insert(*thisRealisation); } else { res.insert(outputPath); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 529a41891..8454ad7d2 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -692,10 +692,10 @@ struct curlFileTransfer : public FileTransfer #if ENABLE_S3 auto [bucketName, key, params] = parseS3Uri(request.uri); - std::string profile = get(params, "profile").value_or(""); - std::string region = get(params, "region").value_or(Aws::Region::US_EAST_1); - std::string scheme = get(params, "scheme").value_or(""); - std::string endpoint = get(params, "endpoint").value_or(""); + std::string profile = getOr(params, "profile", ""); + std::string region = getOr(params, "region", Aws::Region::US_EAST_1); + std::string scheme = getOr(params, "scheme", ""); + std::string endpoint = getOr(params, "endpoint", ""); S3Helper s3Helper(profile, region, scheme, endpoint); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5cc5c91cc..eba3b0fa5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -718,7 +718,11 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat // somewhat expensive so we do lazily hashesModulo = hashDerivationModulo(*this, drv, true); } - StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName); + auto currentOutputHash = get(hashesModulo->hashes, i.first); + if (!currentOutputHash) + throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", + printStorePath(drvPath), printStorePath(doia.path), i.first); + StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName); if (doia.path != recomputed) throw Error("derivation '%s' has incorrect output '%s', should be '%s'", printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 2bbd7aa70..fb985c97b 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -278,11 +278,16 @@ std::map drvOutputReferences( std::set inputRealisations; for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { - auto outputHashes = + const auto outputHashes = staticOutputHashes(store, store.readDerivation(inputDrv)); for (const auto & outputName : outputNames) { + auto outputHash = get(outputHashes, outputName); + if (!outputHash) + throw Error( + "output '%s' of derivation '%s' isn't realised", outputName, + store.printStorePath(inputDrv)); auto thisRealisation = store.queryRealisation( - DrvOutput{outputHashes.at(outputName), outputName}); + DrvOutput{*outputHash, outputName}); if (!thisRealisation) throw Error( "output '%s' of derivation '%s' isn't built", outputName, diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 347e32094..14aeba75c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -853,15 +853,15 @@ std::vector RemoteStore::buildPathsWithResults( OutputPathMap outputs; auto drv = evalStore->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - auto drvOutputs = drv.outputsAndOptPaths(*this); + const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive + const auto drvOutputs = drv.outputsAndOptPaths(*this); for (auto & output : bfd.outputs) { - if (!outputHashes.count(output)) + auto outputHash = get(outputHashes, output); + if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", printStorePath(bfd.drvPath), output); - auto outputId = - DrvOutput{outputHashes.at(output), output}; + auto outputId = DrvOutput{ *outputHash, output }; if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); @@ -874,13 +874,14 @@ std::vector RemoteStore::buildPathsWithResults( } else { // If ca-derivations isn't enabled, assume that // the output path is statically known. - assert(drvOutputs.count(output)); - assert(drvOutputs.at(output).second); + const auto drvOutput = get(drvOutputs, output); + assert(drvOutput); + assert(drvOutput->second); res.builtOutputs.emplace( outputId, Realisation { .id = outputId, - .outPath = *drvOutputs.at(output).second + .outPath = *drvOutput->second, }); } } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 59937be4d..8861274a2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1314,7 +1314,7 @@ static bool isNonUriPath(const std::string & spec) { std::shared_ptr openFromNonUri(const std::string & uri, const Store::Params & params) { if (uri == "" || uri == "auto") { - auto stateDir = get(params, "state").value_or(settings.nixStateDir); + auto stateDir = getOr(params, "state", settings.nixStateDir); if (access(stateDir.c_str(), R_OK | W_OK) == 0) return std::make_shared(params); else if (pathExists(settings.nixDaemonSocketFile)) -- cgit v1.2.3 From 3e87c8e62b3eacecde4e9f467b5463fbbdaa6a3f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 11:22:06 +0200 Subject: Move json stuff out of util.cc --- src/libstore/build/local-derivation-goal.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7f44276bd..3ac9c20f9 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -14,6 +14,7 @@ #include "worker-protocol.hh" #include "topo-sort.hh" #include "callback.hh" +#include "json-utils.hh" #include #include @@ -56,8 +57,6 @@ #include #include -#include - namespace nix { void handleDiffHook( -- cgit v1.2.3 From 107613ad2b2b61ef92edf9ee53ec71ad664be71b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 May 2022 11:31:39 +0200 Subject: Fix compiler warning --- src/libstore/build/worker.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index e7c01a737..b192fbc77 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -350,7 +350,7 @@ void Worker::waitForInput() become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ std::vector pollStatus; - std::map fdToPollStatus; + std::map fdToPollStatus; for (auto & i : children) { for (auto & j : i.fds) { pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); -- cgit v1.2.3 From aefc6c4f41bfac0c76807c234fd0a786dd40f140 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Wed, 11 May 2022 12:15:08 +0200 Subject: Add priority for nix profile install --- src/libstore/builtins/buildenv.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 6f6ad57cb..c17c76e71 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -93,8 +93,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) throw Error( - "packages '%1%' and '%2%' have the same priority %3%; " + "files '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " + "or 'nix profile --priority NUMBER INSTALLED_PKGNAME' " "to change the priority of one of the conflicting packages" " (0 being the highest priority)", srcFile, readLink(dstFile), priority); -- cgit v1.2.3 From 49ad315c0357116787ef45a1249009b6bc00301f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 12 May 2022 20:10:02 +0000 Subject: Use `^` not `!` in indexed store derivations installable syntax Match the other syntax that was recently added --- src/libstore/derived-path.cc | 11 ++++------- src/libstore/derived-path.hh | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 44587ae78..11a3f5e23 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -93,12 +93,9 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_ return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s) +DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) { - size_t n = s.find("!"); - assert(n != s.npos); - auto drvPath = store.parseStorePath(s.substr(0, n)); - auto outputsS = s.substr(n + 1); + auto drvPath = store.parseStorePath(drvS); std::set outputs; if (outputsS != "*") outputs = tokenizeString>(outputsS, ","); @@ -107,10 +104,10 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi DerivedPath DerivedPath::parse(const Store & store, std::string_view s) { - size_t n = s.find("!"); + size_t n = s.rfind("!"); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s); + : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); } RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 24a0ae773..fab1292a7 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -47,7 +47,7 @@ struct DerivedPathBuilt { std::set outputs; std::string to_string(const Store & store) const; - static DerivedPathBuilt parse(const Store & store, std::string_view); + static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); nlohmann::json toJSON(ref store) const; bool operator < (const DerivedPathBuilt & b) const -- cgit v1.2.3 From be2b19041eeec53fba24f7c2494f3f700a4ec595 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Fri, 13 May 2022 22:02:28 +0200 Subject: Integrate review changes --- src/libstore/builtins/buildenv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c17c76e71..58ef89a1c 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -95,7 +95,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw Error( "files '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "or 'nix profile --priority NUMBER INSTALLED_PKGNAME' " + "or 'nix profile install --priority NUMBER INSTALLED_PKGNAME' " "to change the priority of one of the conflicting packages" " (0 being the highest priority)", srcFile, readLink(dstFile), priority); -- cgit v1.2.3 From e53349dd6e4a710eb7abff78722853cad418e9d2 Mon Sep 17 00:00:00 2001 From: Eli Kogan-Wang Date: Mon, 16 May 2022 16:16:06 +0200 Subject: change priority conflict message --- src/libstore/builtins/buildenv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 58ef89a1c..47458a388 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -95,7 +95,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw Error( "files '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "or 'nix profile install --priority NUMBER INSTALLED_PKGNAME' " + "or type 'nix profile install --help' if using 'nix profile' to find out how" "to change the priority of one of the conflicting packages" " (0 being the highest priority)", srcFile, readLink(dstFile), priority); -- cgit v1.2.3 From b8faa837429cbcb4f950248571c761c98895e7cd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 May 2022 13:24:04 +0200 Subject: HttpBinaryCacheStore::getFile(): Don't throw an exception This violates the noexcept specification. Fixes #6445. --- src/libstore/http-binary-cache-store.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 3cb5efdbf..73bcd6e81 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -161,7 +161,12 @@ protected: void getFile(const std::string & path, Callback> callback) noexcept override { - checkEnabled(); + try { + checkEnabled(); + } catch (...) { + callback.rethrow(); + return; + } auto request(makeRequest(path)); -- cgit v1.2.3 From 6378f0bb328437de759f7ed8405fcafbb3bcac54 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 May 2022 13:27:13 +0200 Subject: RemoteStore::queryRealisationUncached(): Fix potential noexcept violation --- src/libstore/remote-store.cc | 50 +++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 26 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 14aeba75c..bc36aef5d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -718,36 +718,34 @@ void RemoteStore::registerDrvOutput(const Realisation & info) void RemoteStore::queryRealisationUncached(const DrvOutput & id, Callback> callback) noexcept { - auto conn(getConnection()); + try { + auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { - warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); - try { - callback(nullptr); - } catch (...) { return callback.rethrow(); } - } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); + return callback(nullptr); + } - conn->to << wopQueryRealisation; - conn->to << id.to_string(); - conn.processStderr(); + conn->to << wopQueryRealisation; + conn->to << id.to_string(); + conn.processStderr(); - auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { - auto outPaths = worker_proto::read( - *this, conn->from, Phantom> {}); - if (outPaths.empty()) - return nullptr; - return std::make_shared(Realisation { .id = id, .outPath = *outPaths.begin() }); - } else { - auto realisations = worker_proto::read( - *this, conn->from, Phantom> {}); - if (realisations.empty()) - return nullptr; - return std::make_shared(*realisations.begin()); - } - }(); + auto real = [&]() -> std::shared_ptr { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + auto outPaths = worker_proto::read( + *this, conn->from, Phantom> {}); + if (outPaths.empty()) + return nullptr; + return std::make_shared(Realisation { .id = id, .outPath = *outPaths.begin() }); + } else { + auto realisations = worker_proto::read( + *this, conn->from, Phantom> {}); + if (realisations.empty()) + return nullptr; + return std::make_shared(*realisations.begin()); + } + }(); - try { callback(std::shared_ptr(real)); } catch (...) { return callback.rethrow(); } } -- cgit v1.2.3 From a9358a6097e0ec0491d4eb83c556c783128a2cb0 Mon Sep 17 00:00:00 2001 From: Lorenzo Manacorda Date: Wed, 1 Jun 2022 14:58:04 +0200 Subject: schema.sql: add comment about hash being in base16 --- src/libstore/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 09c71a2b8..d65e5335e 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -1,7 +1,7 @@ create table if not exists ValidPaths ( id integer primary key autoincrement not null, path text unique not null, - hash text not null, + hash text not null, -- base16 representation registrationTime integer not null, deriver text, narSize integer, -- cgit v1.2.3 From 95f47c28fb8786f8d8d529192465bb6ec20db46b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 3 Jun 2022 17:01:16 +0200 Subject: Make nix copy parallel again FILLME --- src/libstore/store-api.cc | 207 ++++++++++++++++++++++++---------------------- src/libstore/store-api.hh | 7 ++ 2 files changed, 117 insertions(+), 97 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8861274a2..008451666 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -258,6 +258,86 @@ StorePath Store::addToStore( return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); } +void Store::addMultipleToStore( + std::vector>> & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + std::atomic nrDone{0}; + std::atomic nrFailed{0}; + std::atomic bytesExpected{0}; + std::atomic nrRunning{0}; + + using PathWithInfo = std::pair>; + + std::map infosMap; + StorePathSet storePathsToAdd; + for (auto & thingToAdd : pathsToCopy) { + infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd); + storePathsToAdd.insert(thingToAdd.first.path); + } + + auto showProgress = [&]() { + act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed); + }; + + ThreadPool pool; + + processGraph(pool, + storePathsToAdd, + + [&](const StorePath & path) { + auto & [info, source] = *infosMap.at(path); + /* auto storePathForDst = info.storePath; */ + /* if (info->ca && info->references.empty()) { */ + /* storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); */ + /* if (dstStore.storeDir == srcStore.storeDir) */ + /* assert(storePathForDst == storePath); */ + /* if (storePathForDst != storePath) */ + /* debug("replaced path '%s' to '%s' for substituter '%s'", */ + /* srcStore.printStorePath(storePath), */ + /* dstStore.printStorePath(storePathForDst), */ + /* dstStore.getUri()); */ + /* } */ + /* pathsMap.insert_or_assign(storePath, storePathForDst); */ + + if (isValidPath(info.path)) { + nrDone++; + showProgress(); + return StorePathSet(); + } + + bytesExpected += info.narSize; + act.setExpected(actCopyPath, bytesExpected); + + return info.references; + }, + + [&](const StorePath & path) { + checkInterrupt(); + + auto & [info, source] = *infosMap.at(path); + + if (!isValidPath(info.path)) { + MaintainCount mc(nrRunning); + showProgress(); + try { + addToStore(info, *source, repair, checkSigs); + } catch (Error &e) { + nrFailed++; + if (!settings.keepGoing) + throw e; + printMsg(lvlError, "could not copy %s: %s", printStorePath(path), e.what()); + showProgress(); + return; + } + } + + nrDone++; + showProgress(); + }); +} void Store::addMultipleToStore( Source & source, @@ -998,106 +1078,39 @@ std::map copyPaths( Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); - auto sorted = srcStore.topoSortPaths(missing); - std::reverse(sorted.begin(), sorted.end()); - - auto source = sinkToSource([&](Sink & sink) { - sink << sorted.size(); - for (auto & storePath : sorted) { - auto srcUri = srcStore.getUri(); - auto dstUri = dstStore.getUri(); - auto storePathS = srcStore.printStorePath(storePath); - Activity act(*logger, lvlInfo, actCopyPath, - makeCopyPathMessage(srcUri, dstUri, storePathS), - {storePathS, srcUri, dstUri}); - PushActivity pact(act.id); - - auto info = srcStore.queryPathInfo(storePath); - info->write(sink, srcStore, 16); - srcStore.narFromPath(storePath, sink); - } - }); + /* auto sorted = srcStore.topoSortPaths(missing); */ + /* std::reverse(sorted.begin(), sorted.end()); */ + + /* auto source = sinkToSource([&](Sink & sink) { */ + /* sink << sorted.size(); */ + /* for (auto & storePath : sorted) { */ + /* auto srcUri = srcStore.getUri(); */ + /* auto dstUri = dstStore.getUri(); */ + /* auto storePathS = srcStore.printStorePath(storePath); */ + /* Activity act(*logger, lvlInfo, actCopyPath, */ + /* makeCopyPathMessage(srcUri, dstUri, storePathS), */ + /* {storePathS, srcUri, dstUri}); */ + /* PushActivity pact(act.id); */ + + /* auto info = srcStore.queryPathInfo(storePath); */ + /* info->write(sink, srcStore, 16); */ + /* srcStore.narFromPath(storePath, sink); */ + /* } */ + /* }); */ + + std::vector>> pathsToCopy; + + for (auto & missingPath : missing) { + auto info = srcStore.queryPathInfo(missingPath); + auto source = sinkToSource([&](Sink & sink) { + srcStore.narFromPath(missingPath, sink); + }); + pathsToCopy.push_back(std::pair{*info, std::move(source)}); + } - dstStore.addMultipleToStore(*source, repair, checkSigs); + dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); #if 0 - std::atomic nrDone{0}; - std::atomic nrFailed{0}; - std::atomic bytesExpected{0}; - std::atomic nrRunning{0}; - - auto showProgress = [&]() { - act.progress(nrDone, missing.size(), nrRunning, nrFailed); - }; - - ThreadPool pool; - - processGraph(pool, - StorePathSet(missing.begin(), missing.end()), - - [&](const StorePath & storePath) { - auto info = srcStore.queryPathInfo(storePath); - auto storePathForDst = storePath; - if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePath), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); - - if (dstStore.isValidPath(storePath)) { - nrDone++; - showProgress(); - return StorePathSet(); - } - - bytesExpected += info->narSize; - act.setExpected(actCopyPath, bytesExpected); - - return info->references; - }, - - [&](const StorePath & storePath) { - checkInterrupt(); - - auto info = srcStore.queryPathInfo(storePath); - - auto storePathForDst = storePath; - if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) - assert(storePathForDst == storePath); - if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", - srcStore.printStorePath(storePath), - dstStore.printStorePath(storePathForDst), - dstStore.getUri()); - } - pathsMap.insert_or_assign(storePath, storePathForDst); - - if (!dstStore.isValidPath(storePathForDst)) { - MaintainCount mc(nrRunning); - showProgress(); - try { - copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); - } catch (Error &e) { - nrFailed++; - if (!settings.keepGoing) - throw e; - printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what()); - showProgress(); - return; - } - } - - nrDone++; - showProgress(); - }); #endif return pathsMap; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0c8a4db56..d934979cf 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,6 @@ #pragma once +#include "nar-info.hh" #include "realisation.hh" #include "path.hh" #include "derived-path.hh" @@ -364,6 +365,12 @@ public: Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + virtual void addMultipleToStore( + std::vector>> & pathsToCopy, + Activity & act, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs + ); /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. -- cgit v1.2.3 From a7d25d339d94993fc8731de658f18a06e0e2a07e Mon Sep 17 00:00:00 2001 From: Jonpez2 Date: Wed, 8 Jun 2022 09:32:14 +0100 Subject: Add security.csm to the default ignore list --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index feb6899cd..0ee27ecb6 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -802,7 +802,7 @@ public: )"}; Setting ignoredAcls{ - this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls", + this, {"security.selinux", "system.nfs4_acl", "security.csm"}, "ignored-acls", R"( A list of ACLs that should be ignored, normally Nix attempts to remove all ACLs from files and directories in the Nix store, but -- cgit v1.2.3 From cb0553ecd0122f693653c1ac82beb26d127ce3cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 14:03:46 +0200 Subject: Restore the "low-latency" ssh copying --- src/libstore/remote-store.cc | 17 +++++++++++++++ src/libstore/remote-store.hh | 8 ++++++++ src/libstore/store-api.cc | 49 +++++++++++++++++++------------------------- src/libstore/store-api.hh | 6 +++++- 4 files changed, 51 insertions(+), 29 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bc36aef5d..ad2e5c18a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -673,6 +673,23 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } +void RemoteStore::addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + auto source = sinkToSource([&](Sink & sink) { + sink << pathsToCopy.size(); + for (auto & [pathInfo, pathSource] : pathsToCopy) { + pathInfo.write(sink, *this, 16); + pathSource->drainInto(sink); + } + }); + + addMultipleToStore(*source, repair, checkSigs); +} + void RemoteStore::addMultipleToStore( Source & source, RepairFlag repair, diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 8493be6fc..5a599997e 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -88,6 +88,14 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; + void addMultipleToStore( + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) override; + + + StorePath addTextToStore( std::string_view name, std::string_view s, diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 008451666..eeec6c6f7 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -259,7 +259,7 @@ StorePath Store::addToStore( } void Store::addMultipleToStore( - std::vector>> & pathsToCopy, + PathsSource & pathsToCopy, Activity & act, RepairFlag repair, CheckSigsFlag checkSigs) @@ -1072,37 +1072,33 @@ std::map copyPaths( for (auto & path : storePaths) if (!valid.count(path)) missing.insert(path); + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + + // In the general case, `addMultipleToStore` requires a sorted list of + // store paths to add, so sort them right now + auto sortedMissing = srcStore.topoSortPaths(missing); + std::reverse(sortedMissing.begin(), sortedMissing.end()); + std::map pathsMap; for (auto & path : storePaths) pathsMap.insert_or_assign(path, path); - Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + Store::PathsSource pathsToCopy; - /* auto sorted = srcStore.topoSortPaths(missing); */ - /* std::reverse(sorted.begin(), sorted.end()); */ - - /* auto source = sinkToSource([&](Sink & sink) { */ - /* sink << sorted.size(); */ - /* for (auto & storePath : sorted) { */ - /* auto srcUri = srcStore.getUri(); */ - /* auto dstUri = dstStore.getUri(); */ - /* auto storePathS = srcStore.printStorePath(storePath); */ - /* Activity act(*logger, lvlInfo, actCopyPath, */ - /* makeCopyPathMessage(srcUri, dstUri, storePathS), */ - /* {storePathS, srcUri, dstUri}); */ - /* PushActivity pact(act.id); */ - - /* auto info = srcStore.queryPathInfo(storePath); */ - /* info->write(sink, srcStore, 16); */ - /* srcStore.narFromPath(storePath, sink); */ - /* } */ - /* }); */ - - std::vector>> pathsToCopy; - - for (auto & missingPath : missing) { + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); auto source = sinkToSource([&](Sink & sink) { + + // We can reasonably assume that the copy will happen whenever we + // read the path, so log something about that at that point + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(missingPath); + Activity act(*logger, lvlInfo, actCopyPath, + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); + PushActivity pact(act.id); + srcStore.narFromPath(missingPath, sink); }); pathsToCopy.push_back(std::pair{*info, std::move(source)}); @@ -1110,9 +1106,6 @@ std::map copyPaths( dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); - #if 0 - #endif - return pathsMap; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d934979cf..c0a61115b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -360,13 +360,17 @@ public: virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; + // A list of paths infos along with a source providing the content of the + // associated store path + using PathsSource = std::vector>>; + /* Import multiple paths into the store. */ virtual void addMultipleToStore( Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); virtual void addMultipleToStore( - std::vector>> & pathsToCopy, + PathsSource & pathsToCopy, Activity & act, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs -- cgit v1.2.3 From 480c2b6699e6afc6a746ba8a72319f197c3556ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 8 Jun 2022 15:13:11 +0200 Subject: Rewrite the CA paths when moving them between store Bring back the possibility to copy CA paths with no reference (like the outputs of FO derivations or stuff imported at eval time) between stores that have a different prefix. --- src/libstore/store-api.cc | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index eeec6c6f7..61a12e84a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -289,18 +289,6 @@ void Store::addMultipleToStore( [&](const StorePath & path) { auto & [info, source] = *infosMap.at(path); - /* auto storePathForDst = info.storePath; */ - /* if (info->ca && info->references.empty()) { */ - /* storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); */ - /* if (dstStore.storeDir == srcStore.storeDir) */ - /* assert(storePathForDst == storePath); */ - /* if (storePathForDst != storePath) */ - /* debug("replaced path '%s' to '%s' for substituter '%s'", */ - /* srcStore.printStorePath(storePath), */ - /* dstStore.printStorePath(storePathForDst), */ - /* dstStore.getUri()); */ - /* } */ - /* pathsMap.insert_or_assign(storePath, storePathForDst); */ if (isValidPath(info.path)) { nrDone++; @@ -1085,10 +1073,32 @@ std::map copyPaths( Store::PathsSource pathsToCopy; + auto computeStorePathForDst = [&](const ValidPathInfo & currentPathInfo) -> StorePath { + auto storePathForSrc = currentPathInfo.path; + auto storePathForDst = storePathForSrc; + if (currentPathInfo.ca && currentPathInfo.references.empty()) { + storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca); + if (dstStore.storeDir == srcStore.storeDir) + assert(storePathForDst == storePathForSrc); + if (storePathForDst != storePathForSrc) + debug("replaced path '%s' to '%s' for substituter '%s'", + srcStore.printStorePath(storePathForSrc), + dstStore.printStorePath(storePathForDst), + dstStore.getUri()); + } + return storePathForDst; + }; + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); - auto source = sinkToSource([&](Sink & sink) { + auto storePathForDst = computeStorePathForDst(*info); + pathsMap.insert_or_assign(missingPath, storePathForDst); + + ValidPathInfo infoForDst = *info; + infoForDst.path = storePathForDst; + + auto source = sinkToSource([&](Sink & sink) { // We can reasonably assume that the copy will happen whenever we // read the path, so log something about that at that point auto srcUri = srcStore.getUri(); @@ -1101,7 +1111,7 @@ std::map copyPaths( srcStore.narFromPath(missingPath, sink); }); - pathsToCopy.push_back(std::pair{*info, std::move(source)}); + pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); } dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs); -- cgit v1.2.3 From 931930feb139e6db0d7c01097003f8e45862f68f Mon Sep 17 00:00:00 2001 From: Bernardo Meurer Date: Wed, 8 Jun 2022 13:45:39 -0400 Subject: fix(libstore/lock): support users that belong to more than 10 groups The manpage for `getgrouplist` says: > If the number of groups of which user is a member is less than or > equal to *ngroups, then the value *ngroups is returned. > > If the user is a member of more than *ngroups groups, then > getgrouplist() returns -1. In this case, the value returned in > *ngroups can be used to resize the buffer passed to a further > call getgrouplist(). In our original code, however, we allocated a list of size `10` and, if `getgrouplist` returned `-1` threw an exception. In practice, this caused the code to fail for any user belonging to more than 10 groups. While unusual for single-user systems, large companies commonly have a huge number of POSIX groups users belong to, causing this issue to crop up and make multi-user Nix unusable in such settings. The fix is relatively simple, when `getgrouplist` fails, it stores the real number of GIDs in `ngroups`, so we must resize our list and retry. Only then, if it errors once more, we can raise an exception. This should be backported to, at least, 2.9.x. --- src/libstore/lock.cc | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index f1356fdca..fa718f55d 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -67,13 +67,26 @@ bool UserLock::findFreeUser() { #if __linux__ /* Get the list of supplementary groups of this build user. This is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); + int ngroups = 32; // arbitrary initial guess + supplementaryGIDs.resize(ngroups); + + int err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(), + &ngroups); + + // Our initial size of 32 wasn't sufficient, the correct size has + // been stored in ngroups, so we try again. + if (err == -1) { + supplementaryGIDs.resize(ngroups); + err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(), + &ngroups); + } + + // If it failed once more, then something must be broken. if (err == -1) - throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); + throw Error("failed to get list of supplementary groups for '%1%'", + pw->pw_name); + // Finally, trim back the GID list to its real size supplementaryGIDs.resize(ngroups); #endif -- cgit v1.2.3 From 754cd53faf12a9e900c7ef6cefa4a798fccea573 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Fri, 10 Jun 2022 10:49:38 -0700 Subject: Add missing rethrows in conditional exception handlers Signed-off-by: Anders Kaseorg --- src/libstore/gc.cc | 2 ++ src/libstore/local-binary-cache-store.cc | 1 + 2 files changed, 3 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f65fb1b2e..d58ed78b1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -135,6 +135,7 @@ void LocalStore::addTempRoot(const StorePath & path) state->fdRootsSocket.close(); goto restart; } + throw; } } @@ -153,6 +154,7 @@ void LocalStore::addTempRoot(const StorePath & path) state->fdRootsSocket.close(); goto restart; } + throw; } catch (EndOfFile & e) { debug("GC socket disconnected"); state->fdRootsSocket.close(); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index a3c3e4806..ba4416f6d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -69,6 +69,7 @@ protected: } catch (SysError & e) { if (e.errNo == ENOENT) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); + throw; } } -- cgit v1.2.3 From 475249db8aa3c998de594b94b38e08b04b117a6c Mon Sep 17 00:00:00 2001 From: Lorenzo Manacorda Date: Wed, 15 Jun 2022 17:32:59 +0200 Subject: libstore: improve warning message on missing sig Clarifies that the substitute will be ignored/skipped. --- src/libstore/build/substitution-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index ca5218627..3a5da13fb 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -154,7 +154,7 @@ void PathSubstitutionGoal::tryNext() only after we've downloaded the path. */ if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) { - warn("the substitute for '%s' from '%s' is not signed by any of the keys in 'trusted-public-keys'", + warn("igoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); tryNext(); return; -- cgit v1.2.3 From d533a885465846e7512ff976d3599685c90316eb Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Wed, 22 Jun 2022 10:49:18 -0400 Subject: nar-info-disk-cache: refresh nix-cache-info weekly This allows changes to nix-cache-info to be picked up by existing clients. Previously, the only way for this to happen would be for clients to delete binary-cache-v6.sqlite, which is quite awkward for users. On the other hand, updates to nix-cache-info should be pretty rare, hence the choice of a fairly long TTL. Configurability is probably not useful enough to warrant implementing it. --- src/libstore/nar-info-disk-cache.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 9dd81ddfb..00325fcb8 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -62,6 +62,9 @@ public: /* How often to purge expired entries from the cache. */ const int purgeInterval = 24 * 3600; + /* How long to cache binary cache info (i.e. /nix-cache-info) */ + const int cacheInfoTtl = 7 * 24 * 3600; + struct Cache { int id; @@ -98,7 +101,7 @@ public: "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); state->queryCache.create(state->db, - "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); + "select id, timestamp, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); state->insertNAR.create(state->db, "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " @@ -186,8 +189,11 @@ public: auto queryCache(state->queryCache.use()(uri)); if (!queryCache.next()) return std::nullopt; + if (queryCache.getInt(1) + cacheInfoTtl < time(0)) + return std::nullopt; + state->caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); + Cache{(int) queryCache.getInt(0), queryCache.getStr(2), queryCache.getInt(3) != 0, (int) queryCache.getInt(4)}); } auto & cache(getCache(*state, uri)); -- cgit v1.2.3 From d3176ce076407ef3e63667c0436bccf8be317ae4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jun 2022 22:43:53 +0200 Subject: Fix build-remote in nix-static 'build-remote' is now executed via /proc/self/exe so it always works. --- src/libstore/build/hook-instance.cc | 25 ++++++++++++++++++------- src/libstore/globals.cc | 5 +++-- src/libstore/globals.hh | 2 +- 3 files changed, 22 insertions(+), 10 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 0f6f580be..1f19ddccc 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -7,6 +7,22 @@ HookInstance::HookInstance() { debug("starting build hook '%s'", settings.buildHook); + auto buildHookArgs = tokenizeString>(settings.buildHook.get()); + + if (buildHookArgs.empty()) + throw Error("'build-hook' setting is empty"); + + auto buildHook = buildHookArgs.front(); + buildHookArgs.pop_front(); + + Strings args; + + for (auto & arg : buildHookArgs) + args.push_back(arg); + + args.push_back(std::string(baseNameOf(settings.buildHook.get()))); + args.push_back(std::to_string(verbosity)); + /* Create a pipe to get the output of the child. */ fromHook.create(); @@ -36,14 +52,9 @@ HookInstance::HookInstance() if (dup2(builderOut.readSide.get(), 5) == -1) throw SysError("dupping builder's stdout/stderr"); - Strings args = { - std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), - }; - - execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); + execv(buildHook.c_str(), stringsToCharPtrs(args).data()); - throw SysError("executing '%s'", settings.buildHook); + throw SysError("executing '%s'", buildHook); }); pid.setSeparatePG(true); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index cc009a026..1d7f65135 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -67,12 +67,13 @@ Settings::Settings() sandboxPaths = tokenizeString("/bin/sh=" SANDBOX_SHELL); #endif - -/* chroot-like behavior from Apple's sandbox */ + /* chroot-like behavior from Apple's sandbox */ #if __APPLE__ sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif + + buildHook = getSelfExe().value_or("nix") + " __build-remote"; } void loadConfFile() diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 0ee27ecb6..9df1c999c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -195,7 +195,7 @@ public: )", {"build-timeout"}}; - PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", + PathSetting buildHook{this, true, "", "build-hook", "The path of the helper program that executes builds to remote machines."}; Setting builders{ -- cgit v1.2.3 From 184f4e40de0960deccad2147099ea232e5e036c3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jun 2022 23:45:36 +0200 Subject: Remove NIX_LIBEXEC_DIR --- src/libstore/globals.cc | 1 - src/libstore/globals.hh | 3 --- src/libstore/local.mk | 1 - 3 files changed, 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1d7f65135..0f2ca4b15 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -36,7 +36,6 @@ Settings::Settings() , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9df1c999c..d7f351166 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -79,9 +79,6 @@ public: /* A list of user configuration files to load. */ std::vector nixUserConfFiles; - /* The directory where internal helper programs are stored. */ - Path nixLibexecDir; - /* The directory where the main programs are stored. */ Path nixBinDir; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index b992bcbc0..0f94d3917 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -39,7 +39,6 @@ libstore_CXXFLAGS += \ -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ - -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ -DNIX_BIN_DIR=\"$(bindir)\" \ -DNIX_MAN_DIR=\"$(mandir)\" \ -DLSOF=\"$(lsof)\" -- cgit v1.2.3 From 925b97522497e9c0f7a385c904410e560796208f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Jun 2022 18:21:37 +0200 Subject: Embed the sandbox shell into the statically linked 'nix' binary With this, Nix will write a copy of the sandbox shell to /bin/sh in the sandbox rather than bind-mounting it from the host filesystem. This makes /bin/sh work out of the box with nix-static, i.e. you no longer get /nix/store/qa36xhc5gpf42l3z1a8m1lysi40l9p7s-bootstrap-stage4-stdenv-linux/setup: ./configure: /bin/sh: bad interpreter: No such file or directory --- src/libstore/build/local-derivation-goal.cc | 14 +++++++++++++- src/libstore/local.mk | 10 ++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3ac9c20f9..d1ec91ed5 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1717,7 +1717,19 @@ void LocalDerivationGoal::runChild() for (auto & i : dirsInChroot) { if (i.second.source == "/proc") continue; // backwards compatibility - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + + #if HAVE_EMBEDDED_SANDBOX_SHELL + if (i.second.source == "__embedded_sandbox_shell__") { + static unsigned char sh[] = { + #include "embedded-sandbox-shell.gen.hh" + }; + auto dst = chrootRootDir + i.first; + createDirs(dirOf(dst)); + writeFile(dst, std::string_view((const char *) sh, sizeof(sh))); + chmod_(dst, 0555); + } else + #endif + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); } /* Bind a new instance of procfs on /proc. */ diff --git a/src/libstore/local.mk b/src/libstore/local.mk index b992bcbc0..6f05c0d44 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -44,9 +44,19 @@ libstore_CXXFLAGS += \ -DNIX_MAN_DIR=\"$(mandir)\" \ -DLSOF=\"$(lsof)\" +ifeq ($(embedded_sandbox_shell),yes) +libstore_CXXFLAGS += -DSANDBOX_SHELL=\"__embedded_sandbox_shell__\" + +$(d)/build/local-derivation-goal.cc: $(d)/embedded-sandbox-shell.gen.hh + +$(d)/embedded-sandbox-shell.gen.hh: $(sandbox_shell) + $(trace-gen) hexdump -v -e '1/1 "0x%x," "\n"' < $< > $@.tmp + @mv $@.tmp $@ +else ifneq ($(sandbox_shell),) libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" endif +endif $(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh -- cgit v1.2.3 From 0b2ea0023c81e8631df549dce996833ab5213a4a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jun 2022 14:22:11 +0200 Subject: Fix typo --- src/libstore/build/substitution-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 3a5da13fb..2af105b4d 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -154,7 +154,7 @@ void PathSubstitutionGoal::tryNext() only after we've downloaded the path. */ if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) { - warn("igoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", + warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); tryNext(); return; -- cgit v1.2.3 From 2a9fddc0b16d9b4771d11fc10d8b2a9cba55ff64 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jun 2022 16:29:50 +0200 Subject: Automatically use a chroot store if /nix doesn't exist Specifically, if we're not root and the daemon socket does not exist, then we use ~/.local/share/nix/root as a chroot store. This enables non-root users to download nix-static and have it work out of the box, e.g. ubuntu@ip-10-13-1-146:~$ ~/nix run nixpkgs#hello warning: '/nix' does not exists, so Nix will use '/home/ubuntu/.local/share/nix/root' as a chroot store Hello, world! --- src/libstore/store-api.cc | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8861274a2..b46b3066b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1302,7 +1302,8 @@ std::pair splitUriAndParams(const std::string & uri_ return {uri, params}; } -static bool isNonUriPath(const std::string & spec) { +static bool isNonUriPath(const std::string & spec) +{ return // is not a URL spec.find("://") == std::string::npos @@ -1319,7 +1320,19 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para return std::make_shared(params); else if (pathExists(settings.nixDaemonSocketFile)) return std::make_shared(params); - else + else if (!pathExists(stateDir) && params.empty() && getuid() != 0) { + /* If /nix doesn't exist, there is no daemon socket, and + we're not root, then automatically set up a chroot + store in ~/.local/share/nix/root. */ + auto chrootStore = getDataDir() + "/nix/root"; + if (!pathExists(chrootStore)) + warn("'/nix' does not exists, so Nix will use '%s' as a chroot store", chrootStore); + else + debug("'/nix' does not exists, so Nix will use '%s' as a chroot store", chrootStore); + Store::Params params2; + params2["root"] = chrootStore; + return std::make_shared(params2); + } else return std::make_shared(params); } else if (uri == "daemon") { return std::make_shared(params); -- cgit v1.2.3 From 1cb376d60e3a7d0742d92fa2ea1ebebba0a513e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jun 2022 17:18:22 +0200 Subject: Fix typo Co-authored-by: Cole Helbling --- src/libstore/store-api.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b46b3066b..91080a2af 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1326,9 +1326,9 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para store in ~/.local/share/nix/root. */ auto chrootStore = getDataDir() + "/nix/root"; if (!pathExists(chrootStore)) - warn("'/nix' does not exists, so Nix will use '%s' as a chroot store", chrootStore); + warn("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore); else - debug("'/nix' does not exists, so Nix will use '%s' as a chroot store", chrootStore); + debug("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore); Store::Params params2; params2["root"] = chrootStore; return std::make_shared(params2); -- cgit v1.2.3 From 561a258f1d9fd11a5e111e14c492ee166a7551c1 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 23 Jun 2022 14:24:23 -0400 Subject: libstore/nar-info: drop unused system field This was unused everywhere (and even the official NixOS binary cache did not produce .narinfo files containing a "System:" field). --- src/libstore/nar-info.cc | 5 ----- src/libstore/nar-info.hh | 1 - 2 files changed, 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 2d75e7a82..071d8355e 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -69,8 +69,6 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (value != "unknown-deriver") deriver = StorePath(value); } - else if (name == "System") - system = value; else if (name == "Sig") sigs.insert(value); else if (name == "CA") { @@ -106,9 +104,6 @@ std::string NarInfo::to_string(const Store & store) const if (deriver) res += "Deriver: " + std::string(deriver->to_string()) + "\n"; - if (!system.empty()) - res += "System: " + system + "\n"; - for (auto sig : sigs) res += "Sig: " + sig + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 39ced76e5..01683ec73 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -14,7 +14,6 @@ struct NarInfo : ValidPathInfo std::string compression; std::optional fileHash; uint64_t fileSize = 0; - std::string system; NarInfo() = delete; NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } -- cgit v1.2.3 From 8cf26385cd8c0e33e36f8d95b9224160424c1c60 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 23 Jun 2022 14:52:16 -0400 Subject: [fixup] handle cache expiration in sqlite query --- src/libstore/nar-info-disk-cache.cc | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 00325fcb8..f4ea739b0 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -101,7 +101,7 @@ public: "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); state->queryCache.create(state->db, - "select id, timestamp, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); + "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?"); state->insertNAR.create(state->db, "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " @@ -186,14 +186,11 @@ public: auto i = state->caches.find(uri); if (i == state->caches.end()) { - auto queryCache(state->queryCache.use()(uri)); + auto queryCache(state->queryCache.use()(uri)(time(0) - cacheInfoTtl)); if (!queryCache.next()) return std::nullopt; - if (queryCache.getInt(1) + cacheInfoTtl < time(0)) - return std::nullopt; - state->caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(2), queryCache.getInt(3) != 0, (int) queryCache.getInt(4)}); + Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); } auto & cache(getCache(*state, uri)); -- cgit v1.2.3 From 30d4aa5dd651813578b67d70ffbcd0446f6f0fe7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 24 Jun 2022 23:35:21 +0200 Subject: Only do the auto chroot store on Linux --- src/libstore/store-api.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 91080a2af..53b1a8777 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1320,6 +1320,7 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para return std::make_shared(params); else if (pathExists(settings.nixDaemonSocketFile)) return std::make_shared(params); + #if __linux__ else if (!pathExists(stateDir) && params.empty() && getuid() != 0) { /* If /nix doesn't exist, there is no daemon socket, and we're not root, then automatically set up a chroot @@ -1332,7 +1333,9 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para Store::Params params2; params2["root"] = chrootStore; return std::make_shared(params2); - } else + } + #endif + else return std::make_shared(params); } else if (uri == "daemon") { return std::make_shared(params); -- cgit v1.2.3 From 6cab5284614991ea3622492eacdceb3caf52ccff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jun 2022 12:16:51 +0200 Subject: Don't fail if we can't create ~/.local/share/nix/root https://hydra.nixos.org/build/182135943 --- src/libstore/store-api.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 53b1a8777..05353bce2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1326,9 +1326,14 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para we're not root, then automatically set up a chroot store in ~/.local/share/nix/root. */ auto chrootStore = getDataDir() + "/nix/root"; - if (!pathExists(chrootStore)) + if (!pathExists(chrootStore)) { + try { + createDirs(chrootStore); + } catch (Error & e) { + return std::make_shared(params); + } warn("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore); - else + } else debug("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore); Store::Params params2; params2["root"] = chrootStore; -- cgit v1.2.3 From ff49c75502845d9938f3a479f1696ee30d3b45b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Jul 2022 17:46:26 +0200 Subject: Disable auto-chroot if $NIX_STORE_DIR is set Fixes #6732. --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 05353bce2..45c53f23e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1321,7 +1321,7 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para else if (pathExists(settings.nixDaemonSocketFile)) return std::make_shared(params); #if __linux__ - else if (!pathExists(stateDir) && params.empty() && getuid() != 0) { + else if (!pathExists(stateDir) && params.empty() && getuid() != 0 && !getEnv("NIX_STORE_DIR").has_value()) { /* If /nix doesn't exist, there is no daemon socket, and we're not root, then automatically set up a chroot store in ~/.local/share/nix/root. */ -- cgit v1.2.3 From 8735f55decab03ecf3571f756a22abc3b3dc6304 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 14 Jul 2022 20:22:46 -0400 Subject: Fix bug, test more, document more --- src/libstore/derived-path.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 11a3f5e23..f6a0c01df 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -97,8 +97,12 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi { auto drvPath = store.parseStorePath(drvS); std::set outputs; - if (outputsS != "*") + if (outputsS != "*") { outputs = tokenizeString>(outputsS, ","); + if (outputs.empty()) + throw Error( + "Explicit list of wanted outputs '%s' must not be empty. Consider using '*' as a wildcard meaning all outputs if no output in particular is wanted.", outputsS); + } return {drvPath, outputs}; } -- cgit v1.2.3 From 3bcd7a5474f065a6e94a60b3042a0d42ed0184ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 15 Jul 2022 12:32:29 +0200 Subject: Disable auto-chroot if $NIX_STATE_DIR is set Issue #6732. --- src/libstore/store-api.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 45c53f23e..91dbd991e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1321,7 +1321,12 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para else if (pathExists(settings.nixDaemonSocketFile)) return std::make_shared(params); #if __linux__ - else if (!pathExists(stateDir) && params.empty() && getuid() != 0 && !getEnv("NIX_STORE_DIR").has_value()) { + else if (!pathExists(stateDir) + && params.empty() + && getuid() != 0 + && !getEnv("NIX_STORE_DIR").has_value() + && !getEnv("NIX_STATE_DIR").has_value()) + { /* If /nix doesn't exist, there is no daemon socket, and we're not root, then automatically set up a chroot store in ~/.local/share/nix/root. */ -- cgit v1.2.3 From b88fb50e218cd3099cbceace48f7cfdf50a8f11f Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Wed, 6 Jul 2022 11:28:23 -0400 Subject: fix(libstore): allow Nix to access all Rosetta 2 paths on MacOS Fixes: #5884 --- src/libstore/sandbox-defaults.sb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 56b35c3fe..d9d710559 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -98,7 +98,9 @@ (allow file* (literal "/private/var/select/sh")) -; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin. +; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin (and vice versa). (allow file-read* (subpath "/Library/Apple/usr/libexec/oah") - (subpath "/System/Library/Apple/usr/libexec/oah")) + (subpath "/System/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist") + (subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) -- cgit v1.2.3 From 8ea3a911aa81d41efdff231f4b42b11d8861a000 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 14:39:22 -0700 Subject: local-derivation-goal.cc: improve error messages when sandboxing fails The failure modes for nix's sandboxing setup are pretty complicated. When nix is unable to set up the sandbox, let's provide more detail about what went wrong. Specifically: * Make sure the error message includes the word "sandbox" so the user knows that the failure was related to sandboxing. * If `--option sandbox-fallback false` was provided, and removing it would have allowed further attempts to make progress, let the user know. --- src/libstore/build/local-derivation-goal.cc | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d1ec91ed5..86a59e427 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -850,13 +850,23 @@ void LocalDerivationGoal::startBuilder() flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) - _exit(1); - if (child == -1) throw SysError("cloning builder process"); - + if (child == -1) + switch(errno) { + case EPERM: + case EINVAL: { + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (settings.sandboxFallback) + _exit(1); + /* Mention sandbox-fallback in the error message so the user + knows that having it disabled contributed to the + unrecoverability of this failure */ + throw SysError("creating sandboxed builder process using clone(), without sandbox-fallback"); + } + default: + throw SysError("creating sandboxed builder process using clone()"); + } writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); -- cgit v1.2.3 From 90830b1074cd09b58adde859fb1741a33390412f Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 19:28:13 -0700 Subject: local-derivation-goal.cc: warn if failing due to max_user_namespaces==0 This commit uses `warn()` to notify the user if sandbox setup fails with errno==EPERM and /proc/sys/user/max_user_namespaces is missing or zero, since that is at least part of the reason why sandbox setup failed. Note that `echo -n 0 > /proc/sys/user/max_user_namespaces` or equivalent at boot time has been the recommended mitigation for several Linux LPE vulnerabilities over the past few years. Many users have applied this mitigation and then forgotten that they have done so. --- src/libstore/build/local-derivation-goal.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 86a59e427..674b2eaa3 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -859,6 +859,8 @@ void LocalDerivationGoal::startBuilder() to true (the default). */ if (settings.sandboxFallback) _exit(1); + if (!userNamespacesEnabled && errno==EPERM) + warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ -- cgit v1.2.3 From 8d35f387dcfa61c59f898de88ef45f3f97817267 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sat, 16 Jul 2022 19:37:27 -0700 Subject: local-derivation-goal.cc: warn if failing and /proc/self/ns/user missing This commit causes nix to `warn()` if sandbox setup has failed and `/proc/self/ns/user` does not exist. This is usually a sign that the kernel was compiled without `CONFIG_USER_NS=y`, which is required for sandboxing. --- src/libstore/build/local-derivation-goal.cc | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 674b2eaa3..3aa85e264 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -861,6 +861,9 @@ void LocalDerivationGoal::startBuilder() _exit(1); if (!userNamespacesEnabled && errno==EPERM) warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + Path procSelfNsUser = "/proc/self/ns/user"; + if (!pathExists(procSelfNsUser)) + warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ -- cgit v1.2.3 From 6fc56318bf32f715de8634c199c0fb812f813a8c Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sun, 17 Jul 2022 01:23:27 -0700 Subject: local-derivation-goal.cc: add comment re: CLONE_NEWUSER local-derivation-goal.cc contains a comment stating that "Some distros patch Linux to not allow unprivileged user namespaces." Let's give a pointer to a common version of this patch for those who want more details about this failure mode. --- src/libstore/build/local-derivation-goal.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3aa85e264..1c7618045 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -845,6 +845,7 @@ void LocalDerivationGoal::startBuilder() /* Some distros patch Linux to not allow unprivileged * user namespaces. If we get EPERM or EINVAL, try * without CLONE_NEWUSER and see if that works. + * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 */ usingUserNamespace = false; flags &= ~CLONE_NEWUSER; -- cgit v1.2.3 From c8c6203c2c6912a52c5f08881c453a04e7fc3f58 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sun, 17 Jul 2022 01:27:22 -0700 Subject: local-derivation-goal.cc: detect unprivileged_userns_clone failure mode The workaround for "Some distros patch Linux" mentioned in local-derivation-goal.cc will not help in the `--option sandbox-fallback false` case. To provide the user more helpful guidance on how to get the sandbox working, let's check to see if the `/proc` node created by the aforementioned patch is present and configured in a way that will cause us problems. If so, give the user a suggestion for how to troubleshoot the problem. --- src/libstore/build/local-derivation-goal.cc | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 1c7618045..047c5c8ea 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -862,6 +862,13 @@ void LocalDerivationGoal::startBuilder() _exit(1); if (!userNamespacesEnabled && errno==EPERM) warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + if (userNamespacesEnabled) { + Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; + if (pathExists(procSysKernelUnprivilegedUsernsClone) + && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { + warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); + } + } Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); -- cgit v1.2.3 From 5f51539f88227285866843f1383fd47d80fd5918 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:30:52 -0700 Subject: change warn() to notice() --- src/libstore/build/local-derivation-goal.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 047c5c8ea..595149f0a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -861,17 +861,17 @@ void LocalDerivationGoal::startBuilder() if (settings.sandboxFallback) _exit(1); if (!userNamespacesEnabled && errno==EPERM) - warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); + notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; if (pathExists(procSysKernelUnprivilegedUsernsClone) && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - warn("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); + notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); } } Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) - warn("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ -- cgit v1.2.3 From 99fcc91f67ece5a9646065665395f496d6a0cb84 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:33:12 -0700 Subject: as requested by @thufschmitt https://github.com/NixOS/nix/pull/6814#discussion_r924275777 --- src/libstore/build/local-derivation-goal.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 595149f0a..43df41e34 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -855,11 +855,6 @@ void LocalDerivationGoal::startBuilder() switch(errno) { case EPERM: case EINVAL: { - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); if (!userNamespacesEnabled && errno==EPERM) notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { @@ -872,6 +867,11 @@ void LocalDerivationGoal::startBuilder() Path procSelfNsUser = "/proc/self/ns/user"; if (!pathExists(procSelfNsUser)) notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (settings.sandboxFallback) + _exit(1); /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ -- cgit v1.2.3 From 36e1383b6b32abd414c8402dd66f7ef78f93f4e1 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Tue, 19 Jul 2022 03:49:52 -0700 Subject: local-derivation-goal.cc: save global errno to the stack before performing tests which might clobber it --- src/libstore/build/local-derivation-goal.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 43df41e34..79a241ae0 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -851,10 +851,11 @@ void LocalDerivationGoal::startBuilder() flags &= ~CLONE_NEWUSER; child = clone(childEntry, stack + stackSize, flags, this); } - if (child == -1) + if (child == -1) { switch(errno) { case EPERM: case EINVAL: { + int errno_ = errno; if (!userNamespacesEnabled && errno==EPERM) notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); if (userNamespacesEnabled) { @@ -875,11 +876,12 @@ void LocalDerivationGoal::startBuilder() /* Mention sandbox-fallback in the error message so the user knows that having it disabled contributed to the unrecoverability of this failure */ - throw SysError("creating sandboxed builder process using clone(), without sandbox-fallback"); + throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); } default: throw SysError("creating sandboxed builder process using clone()"); } + } writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); -- cgit v1.2.3 From 56f6f3725f4fbeeb7900ae95bd71d559695b3dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 19 Jul 2022 19:46:00 +0200 Subject: Don't ultimately trust the signed paths Like the old implem did (and like you'd want it to be anyways) --- src/libstore/store-api.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 61a12e84a..06d03bff2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -305,7 +305,9 @@ void Store::addMultipleToStore( [&](const StorePath & path) { checkInterrupt(); - auto & [info, source] = *infosMap.at(path); + auto & [info_, source] = *infosMap.at(path); + auto info = info_; + info.ultimate = false; if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); -- cgit v1.2.3 From 1af5d798a4d10a07e995d420a759f2fe752b583a Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Fri, 15 Jul 2022 00:14:29 -0400 Subject: libstore/globals.cc: Automatically set cores based on cgroup CPU limit By default, Nix sets the "cores" setting to the number of CPUs which are physically present on the machine. If cgroups are used to limit the CPU and memory consumption of a large Nix build, the OOM killer may be invoked. For example, consider a GitLab CI pipeline which builds a large software package. The GitLab runner spawns a container whose CPU is limited to 4 cores and whose memory is limited to 16 GiB. If the underlying machine has 64 cores, Nix will invoke the build with -j64. In many cases, that level of parallelism will invoke the OOM killer and the build will completely fail. This change sets the default value of "cores" to be ceil(cpu_quota / cpu_period), with a fallback to std::thread::hardware_concurrency() if cgroups v2 is not detected. --- src/libstore/globals.cc | 50 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 0f2ca4b15..48df839fa 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -11,6 +11,11 @@ #include #include +#if __linux__ +#include +#include +#endif + #include @@ -114,7 +119,50 @@ std::vector getUserConfigFiles() unsigned int Settings::getDefaultCores() { - return std::max(1U, std::thread::hardware_concurrency()); + unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + + #if __linux__ + FILE *fp = fopen("/proc/mounts", "r"); + if (!fp) + return concurrency; + + Strings cgPathParts; + + struct mntent *ent; + while ((ent = getmntent(fp))) { + std::string mountType, mountPath; + + mountType = ent->mnt_type; + mountPath = ent->mnt_dir; + + if (mountType == "cgroup2") { + cgPathParts.push_back(mountPath); + break; + } + } + + fclose(fp); + + if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { + std::string currentCgroup = readFile("/proc/self/cgroup"); + Strings cgValues = tokenizeString(currentCgroup, ":"); + cgPathParts.push_back(trim(cgValues.back(), "\n")); + cgPathParts.push_back("cpu.max"); + std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); + + if (pathExists(fullCgPath)) { + std::string cpuMax = readFile(fullCgPath); + std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); + std::string quota = cpuMaxParts[0]; + std::string period = trim(cpuMaxParts[1], "\n"); + + if (quota != "max") + concurrency = std::ceil(std::stoi(quota) / std::stof(period)); + } + } + #endif + + return concurrency; } StringSet Settings::getDefaultSystemFeatures() -- cgit v1.2.3 From 722de8ddcc875c7e8e9a228f9d88454bae31fd40 Mon Sep 17 00:00:00 2001 From: Alex Wied Date: Tue, 19 Jul 2022 02:09:46 -0400 Subject: libstore/globals.cc: Move cgroup detection to libutil --- src/libstore/globals.cc | 54 ++++++------------------------------------------- 1 file changed, 6 insertions(+), 48 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 48df839fa..d724897bb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -11,11 +11,6 @@ #include #include -#if __linux__ -#include -#include -#endif - #include @@ -119,50 +114,13 @@ std::vector getUserConfigFiles() unsigned int Settings::getDefaultCores() { - unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); - - #if __linux__ - FILE *fp = fopen("/proc/mounts", "r"); - if (!fp) - return concurrency; - - Strings cgPathParts; - - struct mntent *ent; - while ((ent = getmntent(fp))) { - std::string mountType, mountPath; - - mountType = ent->mnt_type; - mountPath = ent->mnt_dir; - - if (mountType == "cgroup2") { - cgPathParts.push_back(mountPath); - break; - } - } - - fclose(fp); - - if (cgPathParts.size() > 0 && pathExists("/proc/self/cgroup")) { - std::string currentCgroup = readFile("/proc/self/cgroup"); - Strings cgValues = tokenizeString(currentCgroup, ":"); - cgPathParts.push_back(trim(cgValues.back(), "\n")); - cgPathParts.push_back("cpu.max"); - std::string fullCgPath = canonPath(concatStringsSep("/", cgPathParts)); - - if (pathExists(fullCgPath)) { - std::string cpuMax = readFile(fullCgPath); - std::vector cpuMaxParts = tokenizeString>(cpuMax, " "); - std::string quota = cpuMaxParts[0]; - std::string period = trim(cpuMaxParts[1], "\n"); - - if (quota != "max") - concurrency = std::ceil(std::stoi(quota) / std::stof(period)); - } - } - #endif + const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); + const unsigned int maxCPU = getMaxCPU(); - return concurrency; + if (maxCPU > 0) + return maxCPU; + else + return concurrency; } StringSet Settings::getDefaultSystemFeatures() -- cgit v1.2.3 From 1467a98d4c2014e512825c87e9056de79408421a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= Date: Mon, 1 Aug 2022 11:37:21 +0200 Subject: derivation-goal.cc: remove bmCheck custom return branch on buildDone Once a derivation goal has been completed, we check whether or not this goal was meant to be repeated to check its output. An early return branch was preventing the worker to reach that repeat code branch, hence breaking the --check command (#2619). It seems like this early return branch is an artifact of a passed refactoring. As far as I can tell, buildDone's main branch also cleanup the tmp directory before returning. --- src/libstore/build/derivation-goal.cc | 6 ------ 1 file changed, 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3fff2385f..3c0b14029 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -914,12 +914,6 @@ void DerivationGoal::buildDone() outputPaths ); - if (buildMode == bmCheck) { - cleanupPostOutputsRegisteredModeCheck(); - done(BuildResult::Built, std::move(builtOutputs)); - return; - } - cleanupPostOutputsRegisteredModeNonCheck(); /* Repeat the build if necessary. */ -- cgit v1.2.3 From c2de0a232c1cfddb1f1385ffd23dd43a2099458e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 17 Mar 2022 15:28:46 +0100 Subject: =?UTF-8?q?Create=20a=20wrapper=20around=20stdlib=E2=80=99s=20`ren?= =?UTF-8?q?ame`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Directly takes some c++ strings, and gently throws an exception on error (rather than having to inline this logic everywhere) --- src/libstore/build/derivation-goal.cc | 3 +-- src/libstore/build/local-derivation-goal.cc | 8 +++----- src/libstore/builtins/unpack-channel.cc | 3 +-- src/libstore/gc.cc | 4 +--- src/libstore/local-binary-cache-store.cc | 3 +-- src/libstore/local-store.cc | 6 ++---- src/libstore/optimise-store.cc | 6 ++++-- 7 files changed, 13 insertions(+), 20 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3fff2385f..93f923f18 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -705,8 +705,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + moveFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 79a241ae0..2435377e2 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -223,8 +223,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - if (rename(src.c_str(), dst.c_str())) - throw SysError("renaming '%1%' to '%2%'", src, dst); + moveFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -311,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (buildMode != bmCheck && status.known->isValid()) continue; auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) - rename((chrootRootDir + p).c_str(), p.c_str()); + moveFile((chrootRootDir + p), p); } return diskFull; @@ -2625,8 +2624,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; - if (rename(path.c_str(), dst.c_str())) - throw SysError("renaming '%s' to '%s'", path, dst); + moveFile(path, dst); } } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index 426d58a53..a8417d7ff 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -22,8 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) auto entries = readDirectory(out); if (entries.size() != 1) throw Error("channel tarball '%s' contains more than one file", src); - if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1) - throw SysError("renaming channel directory"); + moveFile((out + "/" + entries[0].name), (out + "/" + channelName)); } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d58ed78b1..3682f81cc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -39,9 +39,7 @@ static void makeSymlink(const Path & link, const Path & target) createSymlink(target, tempLink); /* Atomically replace the old one. */ - if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError("cannot rename '%1%' to '%2%'", - tempLink , link); + moveFile(tempLink, link); } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index ba4416f6d..ff7403f9d 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -57,8 +57,7 @@ protected: AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - if (rename(tmp.c_str(), path2.c_str())) - throw SysError("renaming '%1%' to '%2%'", tmp, path2); + moveFile(tmp, path2); del.cancel(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eba3b0fa5..7f708b243 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,8 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - if (rename(tempPath.c_str(), realPath.c_str())) - throw Error("renaming '%s' to '%s'", tempPath, realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1942,8 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) writeFile(tmpFile, compress("bzip2", log)); - if (rename(tmpFile.c_str(), logPath.c_str()) != 0) - throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath); + moveFile(tmpFile, logPath); } std::optional LocalStore::getVersion() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8af9b1dde..20b9c7611 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -229,7 +229,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } /* Atomically replace the old file with the new hard link. */ - if (rename(tempLink.c_str(), path.c_str()) == -1) { + try { + moveFile(tempLink, path); + } catch (SysError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { @@ -240,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); + throw; } stats.filesLinked++; -- cgit v1.2.3 From d71d9e9fbfc972514b0b940181ab7f9e766f41f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:10:36 +0200 Subject: moveFile -> renameFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `move` tends to have this `mv` connotation of “I will copy it for you if needs be” --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 6 +++--- src/libstore/builtins/unpack-channel.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/local-store.cc | 4 ++-- src/libstore/optimise-store.cc | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 93f923f18..459fdae79 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -705,7 +705,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - moveFile(src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2435377e2..6843173a7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -223,7 +223,7 @@ static void movePath(const Path & src, const Path & dst) if (changePerm) chmod_(src, st.st_mode | S_IWUSR); - moveFile(src, dst); + renameFile(src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -310,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() if (buildMode != bmCheck && status.known->isValid()) continue; auto p = worker.store.printStorePath(status.known->path); if (pathExists(chrootRootDir + p)) - moveFile((chrootRootDir + p), p); + renameFile((chrootRootDir + p), p); } return diskFull; @@ -2624,7 +2624,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() Path prev = path + checkSuffix; deletePath(prev); Path dst = path + checkSuffix; - moveFile(path, dst); + renameFile(path, dst); } } diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index a8417d7ff..ba04bb16c 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -22,7 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) auto entries = readDirectory(out); if (entries.size() != 1) throw Error("channel tarball '%s' contains more than one file", src); - moveFile((out + "/" + entries[0].name), (out + "/" + channelName)); + renameFile((out + "/" + entries[0].name), (out + "/" + channelName)); } } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 3682f81cc..4c1a82279 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -39,7 +39,7 @@ static void makeSymlink(const Path & link, const Path & target) createSymlink(target, tempLink); /* Atomically replace the old one. */ - moveFile(tempLink, link); + renameFile(tempLink, link); } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index ff7403f9d..f20b1fa02 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -57,7 +57,7 @@ protected: AutoDelete del(tmp, false); StreamToSourceAdapter source(istream); writeFile(tmp, source); - moveFile(tmp, path2); + renameFile(tmp, path2); del.cancel(); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7f708b243..2d076f404 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,7 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - moveFile(tempPath, realPath); + renameFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1941,7 +1941,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) writeFile(tmpFile, compress("bzip2", log)); - moveFile(tmpFile, logPath); + renameFile(tmpFile, logPath); } std::optional LocalStore::getVersion() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 20b9c7611..4d2781180 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -230,7 +230,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ try { - moveFile(tempLink, path); + renameFile(tempLink, path); } catch (SysError & e) { if (unlink(tempLink.c_str()) == -1) printError("unable to unlink '%1%'", tempLink); -- cgit v1.2.3 From 90f968073338d6ba276994bc281fa5efa5306e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 13 Apr 2022 14:19:42 +0200 Subject: Only use `renameFile` where needed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In most places the fallback to copying isn’t needed and can actually be bad, so we’d rather not transparently fallback --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2d076f404..a272e4301 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1430,7 +1430,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name writeFile(realPath, dumpSource); } else { /* Move the temporary path we restored above. */ - renameFile(tempPath, realPath); + moveFile(tempPath, realPath); } /* For computing the nar hash. In recursive SHA-256 mode, this -- cgit v1.2.3 From 53e7b7e8ac44704199a868c0f6850ac53a0b8ad1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 12 Aug 2022 12:28:02 +0200 Subject: Remove warnLargeDump() This message was unhelpful (#1184) and probably misleading since memory is O(1) in most cases now. --- src/libstore/remote-store.cc | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index bc36aef5d..eaaf9669f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -580,7 +580,6 @@ ref RemoteStore::addCAToStore( try { conn->to.written = 0; - conn->to.warn = true; connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); @@ -591,7 +590,6 @@ ref RemoteStore::addCAToStore( dumpString(contents, conn->to); } } - conn->to.warn = false; conn.processStderr(); } catch (SysError & e) { /* Daemon closed while we were sending the path. Probably OOM -- cgit v1.2.3 From 7d934f7880d460cba1ab908fd35b3d43e97a4749 Mon Sep 17 00:00:00 2001 From: pennae Date: Fri, 19 Aug 2022 11:26:26 +0200 Subject: don't read outputs into memory for output rewriting RewritingSink can handle being fed input where a reference crosses a chunk boundary. we don't need to load the whole source into memory, and in fact *not* loading the whole source lets nix build FODs that do not fit into memory (eg fetchurl'ing data files larger than system memory). --- src/libstore/build/local-derivation-goal.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6843173a7..18b682e13 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2374,10 +2374,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() if (*scratchPath != finalPath) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { - StringSink sink; - dumpPath(actualPath, sink); RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); - rsink2(sink.s); + dumpPath(actualPath, rsink2); rsink2.flush(); }); Path tmpPath = actualPath + ".tmp"; -- cgit v1.2.3 From 0d2bf7acf994ba331d6f72c746721b354931be76 Mon Sep 17 00:00:00 2001 From: Solene Rapenne Date: Fri, 19 Aug 2022 12:40:22 +0200 Subject: add a nix.conf option to set a download speed limit --- src/libstore/filetransfer.cc | 3 +++ src/libstore/globals.hh | 7 +++++++ 2 files changed, 10 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8454ad7d2..252403cb5 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -308,6 +308,9 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + if (settings.downloadSpeed.get() > 0) + curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024)); + if (request.head) curl_easy_setopt(req, CURLOPT_NOBODY, 1); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d7f351166..ca8fc6d5f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -746,6 +746,13 @@ public: /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`. )"}; + Setting downloadSpeed { + this, 0, "download-speed", + R"( + Specify the maxium transfer rate in kilobytes per second you want + nix to use for download. + )"}; + Setting netrcFile{ this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", R"( -- cgit v1.2.3 From caad87e6dbfbc62fafb4a055b45d7f2eb3d11efa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sol=C3=A8ne=20Rapenne?= Date: Sat, 20 Aug 2022 18:21:36 +0200 Subject: Better documentation wording Co-authored-by: Anderson Torres --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ca8fc6d5f..1ff7d2d0a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -749,7 +749,7 @@ public: Setting downloadSpeed { this, 0, "download-speed", R"( - Specify the maxium transfer rate in kilobytes per second you want + Specify the maximum transfer rate in kilobytes per second you want nix to use for download. )"}; -- cgit v1.2.3 From c21b1a7e67cc28b6e95a563daa786f385bc716b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 14:14:14 +0200 Subject: Spelling --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1ff7d2d0a..e9d721e59 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -750,7 +750,7 @@ public: this, 0, "download-speed", R"( Specify the maximum transfer rate in kilobytes per second you want - nix to use for download. + Nix to use for downloads. )"}; Setting netrcFile{ -- cgit v1.2.3 From f865048332a26f69a007881877203b9428783357 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Aug 2022 15:30:38 +0200 Subject: Indentation --- src/libstore/remote-store.hh | 10 ++++------ src/libstore/store-api.hh | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 5a599997e..11d089cd2 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -89,12 +89,10 @@ public: CheckSigsFlag checkSigs) override; void addMultipleToStore( - PathsSource & pathsToCopy, - Activity & act, - RepairFlag repair, - CheckSigsFlag checkSigs) override; - - + PathsSource & pathsToCopy, + Activity & act, + RepairFlag repair, + CheckSigsFlag checkSigs) override; StorePath addTextToStore( std::string_view name, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c0a61115b..c8a667c6d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -369,12 +369,12 @@ public: Source & source, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + virtual void addMultipleToStore( PathsSource & pathsToCopy, Activity & act, RepairFlag repair = NoRepair, - CheckSigsFlag checkSigs = CheckSigs - ); + CheckSigsFlag checkSigs = CheckSigs); /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. -- cgit v1.2.3 From f0358ed4650e4608a383bd9f59ee23545f86c4ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Aug 2022 14:14:47 +0200 Subject: Fix a hang in nix-copy-ssh.sh This hang for some reason didn't trigger in the Nix build, but did running 'make installcheck' interactively. What happened: * Store::addMultipleToStore() calls a SinkToSource object to copy a path, which in turn calls LegacySSHStore::narFromPath(), which acquires a connection. * The SinkToSource object is not destroyed after the last bytes has been read, so the coroutine's stack is still alive and its destructors are not run. So the connection is not released. * Then when the next path is copied, because max-connections = 1, LegacySSHStore::narFromPath() hangs forever waiting for a connection to be released. The fix is to make sure that the source object is destroyed when we're done with it. --- src/libstore/store-api.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 1406bf657..9c3a0b3d6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,6 +11,7 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +#include "finally.hh" #include @@ -271,7 +272,7 @@ void Store::addMultipleToStore( using PathWithInfo = std::pair>; - std::map infosMap; + std::map infosMap; StorePathSet storePathsToAdd; for (auto & thingToAdd : pathsToCopy) { infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd); @@ -288,7 +289,8 @@ void Store::addMultipleToStore( storePathsToAdd, [&](const StorePath & path) { - auto & [info, source] = *infosMap.at(path); + + auto & [info, _] = *infosMap.at(path); if (isValidPath(info.path)) { nrDone++; @@ -309,12 +311,21 @@ void Store::addMultipleToStore( auto info = info_; info.ultimate = false; + /* Make sure that the Source object is destroyed when + we're done. In particular, a SinkToSource object must + be destroyed to ensure that the destructors on its + stack frame are run; this includes + LegacySSHStore::narFromPath()'s connection lock. */ + Finally cleanupSource{[&]() { + source.reset(); + }}; + if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); showProgress(); try { addToStore(info, *source, repair, checkSigs); - } catch (Error &e) { + } catch (Error & e) { nrFailed++; if (!settings.keepGoing) throw e; -- cgit v1.2.3 From 8d906b1f3bd4343b6b309ddfca824d5dd00a09b1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 14:11:03 +0200 Subject: Fix macOS build --- src/libstore/store-api.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9c3a0b3d6..2cd6c15ec 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -307,8 +307,9 @@ void Store::addMultipleToStore( [&](const StorePath & path) { checkInterrupt(); - auto & [info_, source] = *infosMap.at(path); + auto & [info_, source_] = *infosMap.at(path); auto info = info_; + auto source = std::move(source_); info.ultimate = false; /* Make sure that the Source object is destroyed when -- cgit v1.2.3 From 56d97d4b4df02e3464a2f003a90b7f6abae16722 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Aug 2022 14:49:58 +0200 Subject: Remove redundant Finally --- src/libstore/store-api.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2cd6c15ec..86b12257a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -11,7 +11,6 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" -#include "finally.hh" #include @@ -309,7 +308,6 @@ void Store::addMultipleToStore( auto & [info_, source_] = *infosMap.at(path); auto info = info_; - auto source = std::move(source_); info.ultimate = false; /* Make sure that the Source object is destroyed when @@ -317,9 +315,7 @@ void Store::addMultipleToStore( be destroyed to ensure that the destructors on its stack frame are run; this includes LegacySSHStore::narFromPath()'s connection lock. */ - Finally cleanupSource{[&]() { - source.reset(); - }}; + auto source = std::move(source_); if (!isValidPath(info.path)) { MaintainCount mc(nrRunning); -- cgit v1.2.3 From 8e5659423eb9c64cefa3c85af8bed78c4e82eb08 Mon Sep 17 00:00:00 2001 From: Winter Date: Wed, 24 Aug 2022 13:09:44 -0400 Subject: fix(libstore): allow access to trustd on macOS --- src/libstore/sandbox-network.sb | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/sandbox-network.sb b/src/libstore/sandbox-network.sb index 56beec761..19e9eea9a 100644 --- a/src/libstore/sandbox-network.sb +++ b/src/libstore/sandbox-network.sb @@ -14,3 +14,7 @@ ; Allow DNS lookups. (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))) + +; Allow access to trustd. +(allow mach-lookup (global-name "com.apple.trustd")) +(allow mach-lookup (global-name "com.apple.trustd.agent")) -- cgit v1.2.3 From 59dc8346ca53f49ccdbbd6709b12a479376d1464 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 17:51:56 -0700 Subject: move substituter signature-checking conditions to configuration file documentation --- src/libstore/globals.hh | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d7f351166..a659036e2 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -613,6 +613,14 @@ public: are tried based on their Priority value, which each substituter can set independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. + + Nix will copy a store path from a remote store only if one + of the following is true: + + - the store object is signed by one of the `trusted-public-keys` + - the substituter is in the `trusted-substituters` list + - the `no-require-sigs` option has been set to disable signature checking + - the store object is [output-addressed](#gloss-output-addressed-store-object) )", {"binary-caches"}}; -- cgit v1.2.3 From 1ab913467ef8e9ff946e64bd31841775d743b2d6 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 1 Sep 2022 17:54:23 -0700 Subject: linkify mention of other options --- src/libstore/globals.hh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a659036e2..a4db3bf08 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -617,10 +617,10 @@ public: Nix will copy a store path from a remote store only if one of the following is true: - - the store object is signed by one of the `trusted-public-keys` - - the substituter is in the `trusted-substituters` list - - the `no-require-sigs` option has been set to disable signature checking - - the store object is [output-addressed](#gloss-output-addressed-store-object) + - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) + - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list + - the [`require-sigs`](#conf-require-sigs) option has been set to `false` + - the store object is [output-addressed](glossary.md#gloss-output-addressed-store-object) )", {"binary-caches"}}; -- cgit v1.2.3 From 4894e567fb27c02866abd75f59da740efa6c33a5 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 2 Sep 2022 11:46:34 -0500 Subject: =?UTF-8?q?Don=E2=80=99t=20readDerivation=20if=20impure=20derivati?= =?UTF-8?q?ons=20feature=20is=20disabled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readDerivation is pretty slow, and while it may not be significant for some use cases, on things like ghc-nix where we have thousands of derivations is really slows things down. So, this just doesn’t do the impure derivation check if the impure derivation experimental feature is disabled. Perhaps we could cache the result of isPure() and keep the check, but this is a quick fix to for the slowdown introduced with impure derivations features in 2.8.0. --- src/libstore/build/derivation-goal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 83da657f0..41d2e2a1c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -344,7 +344,7 @@ void DerivationGoal::gaveUpOnSubstitution() for (auto & i : dynamic_cast(drv.get())->inputDrvs) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ - if (drv->type().isPure() && !drv->type().isFixed()) { + if (settings.isExperimentalFeatureEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { auto inputDrv = worker.evalStore.readDerivation(i.first); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", -- cgit v1.2.3 From 1f041ac54f43093e4f4df1caa630d491ff51c3f8 Mon Sep 17 00:00:00 2001 From: Andrew Brooks Date: Fri, 2 Sep 2022 18:32:35 -0500 Subject: Prevent tempdir from being GC-ed before addToStoreFromDump has renamed it This fixes issue 6823 by placing the tempdir used in LocalStore::addToStoreFromDump outside the Nix store, where automatic GC is no longer a concern. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a272e4301..6abd52683 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1388,7 +1388,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name StringSource dumpSource { dump }; ChainSource bothSource { dumpSource, source }; - auto tempDir = createTempDir(realStoreDir, "add"); + auto tempDir = createTempDir("", "add"); delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; -- cgit v1.2.3 From 84fe75a12a085c6b4b8d4ac65a048f569de1252b Mon Sep 17 00:00:00 2001 From: Andrew Brooks Date: Tue, 6 Sep 2022 17:48:00 -0500 Subject: Keep created temp dirs inside store, but protect from GC Implements the approach suggested by feedback on PR #6994, where tempdir paths are created in the store (now with an exclusive lock). As part of this work, the currently-broken and unused `createTempDirInStore` function is updated to create an exclusive lock on the temp directory in the store. The GC now makes a non-blocking attempt to lock any store directories that "look like" the temp directories created by this function, and if it can't acquire one, ignores the directory. --- src/libstore/gc.cc | 12 ++++++++++++ src/libstore/local-store.cc | 29 +++++++++++++++++++---------- src/libstore/local-store.hh | 2 +- 3 files changed, 32 insertions(+), 11 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 4c1a82279..6cd7efbc9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -619,6 +619,18 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) Path path = storeDir + "/" + std::string(baseName); Path realPath = realStoreDir + "/" + std::string(baseName); + /* There may be temp directories in the store that are still in use + by another process. We need to be sure that we can acquire an + exclusive lock before deleting them. */ + AutoCloseFD tmpDirFd; + if (baseName.rfind("add-", 0) == 0) { + tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); + if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) { + debug("skipping locked tempdir '%s'", realPath); + return; + } + } + printInfo("deleting '%1%'", path); results.paths.insert(path); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6abd52683..5ee451da3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1382,13 +1382,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name std::unique_ptr delTempDir; Path tempPath; + Path tempDir; + AutoCloseFD tempDirFd; if (!inMemory) { /* Drain what we pulled so far, and then keep on pulling */ StringSource dumpSource { dump }; ChainSource bothSource { dumpSource, source }; - auto tempDir = createTempDir("", "add"); + std::tie(tempDir, tempDirFd) = createTempDirInStore(); delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; @@ -1431,6 +1433,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name } else { /* Move the temporary path we restored above. */ moveFile(tempPath, realPath); + tempDirFd.close(); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1507,18 +1510,24 @@ StorePath LocalStore::addTextToStore( /* Create a temporary directory in the store that won't be - garbage-collected. */ -Path LocalStore::createTempDirInStore() + garbage-collected until the returned FD is closed. */ +std::pair LocalStore::createTempDirInStore() { - Path tmpDir; + Path tmpDirFn; + AutoCloseFD tmpDirFd; + bool lockedByUs = false; do { /* There is a slight possibility that `tmpDir' gets deleted by - the GC between createTempDir() and addTempRoot(), so repeat - until `tmpDir' exists. */ - tmpDir = createTempDir(realStoreDir); - addTempRoot(parseStorePath(tmpDir)); - } while (!pathExists(tmpDir)); - return tmpDir; + the GC between createTempDir() and when we acquire a lock on it. + We'll repeat until 'tmpDir' exists and we've locked it. */ + tmpDirFn = createTempDir(realStoreDir, "add"); + tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY); + if (tmpDirFd.get() < 0) { + continue; + } + lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true); + } while (!pathExists(tmpDirFn) || !lockedByUs); + return {tmpDirFn, std::move(tmpDirFd)}; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 70d225be3..bd0ce1fe6 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -256,7 +256,7 @@ private: void findRuntimeRoots(Roots & roots, bool censor); - Path createTempDirInStore(); + std::pair createTempDirInStore(); void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); -- cgit v1.2.3 From 565d888e0f6a2c66ee7b10f6fe6a97f79fa51732 Mon Sep 17 00:00:00 2001 From: Andrew Brooks Date: Mon, 12 Sep 2022 11:33:23 -0500 Subject: Address PR feedback on #6694 --- src/libstore/gc.cc | 5 ++--- src/libstore/local-store.cc | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 6cd7efbc9..9ef8972f3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -622,9 +622,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* There may be temp directories in the store that are still in use by another process. We need to be sure that we can acquire an exclusive lock before deleting them. */ - AutoCloseFD tmpDirFd; - if (baseName.rfind("add-", 0) == 0) { - tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); + if (baseName.find("tmp-", 0) == 0) { + AutoCloseFD tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY); if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) { debug("skipping locked tempdir '%s'", realPath); return; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 5ee451da3..0b07cde34 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1433,7 +1433,6 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name } else { /* Move the temporary path we restored above. */ moveFile(tempPath, realPath); - tempDirFd.close(); } /* For computing the nar hash. In recursive SHA-256 mode, this @@ -1520,7 +1519,7 @@ std::pair LocalStore::createTempDirInStore() /* There is a slight possibility that `tmpDir' gets deleted by the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ - tmpDirFn = createTempDir(realStoreDir, "add"); + tmpDirFn = createTempDir(realStoreDir, "tmp"); tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY); if (tmpDirFd.get() < 0) { continue; -- cgit v1.2.3 From 0f977bf91e29192d7f0c0f9cad16351bad7cd137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Mon, 19 Sep 2022 08:42:43 +0200 Subject: Remove a useless debug message in filetransfer.cc Remove the `verify TLS: Nix CA file = 'blah'` message that Nix used to print when fetching anything as it's both useless (`libcurl` prints the same info in its logs) and misleading (gives the impression that a new TLS connection is being established which might not be the case because of multiplexing. See #7011 ) --- src/libstore/filetransfer.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 252403cb5..5746c32a3 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -322,7 +322,6 @@ struct curlFileTransfer : public FileTransfer } if (request.verifyTLS) { - debug("verify TLS: Nix CA file = '%s'", settings.caFile); if (settings.caFile != "") curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); } else { -- cgit v1.2.3 From 1b595026e18afb050de3f62ded8f7180bc8b2b0e Mon Sep 17 00:00:00 2001 From: squalus Date: Mon, 19 Sep 2022 11:15:31 -0700 Subject: Improve durability of schema version file writes - call close explicitly in writeFile to prevent the close exception from being ignored - fsync after writing schema file to flush data to disk - fsync schema file parent to flush metadata to disk https://github.com/NixOS/nix/issues/7064 --- src/libstore/local-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0b07cde34..37302d3a8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -158,7 +158,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) txn.commit(); } - writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); + writeFile(schemaPath, fmt("%d", nixCASchemaVersion), 0666, true); lockFile(lockFd.get(), ltRead, true); } } @@ -281,7 +281,7 @@ LocalStore::LocalStore(const Params & params) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; openDB(*state, true); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true); } else if (curSchema < nixSchemaVersion) { @@ -329,7 +329,7 @@ LocalStore::LocalStore(const Params & params) txn.commit(); } - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true); lockFile(globalLock.get(), ltRead, true); } -- cgit v1.2.3 From 752f967c0fe2489fe13d8c2c65c3ecba72064adc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 22 Sep 2022 10:43:48 -0400 Subject: "valid signature" -> "trustworthy signature" I just had a colleague get confused by the previous phrase for good reason. "valid" sounds like an *objective* criterion, e.g. and *invalid signature* would be one that would be trusted by no one, e.g. because it misformatted or something. What is actually going is that there might be a signature which is perfectly valid to *someone else*, but not to the user, because they don't trust the corresponding public key. This is a *subjective* criterion, because it depends on the arbitrary and personal choice of which public keys to trust. I therefore think "trustworthy" is a better adjective to use. Whether something is worthy of trust is clearly subjective, and then "trust" within that word nicely evokes `trusted-public-keys` and friends. --- src/libstore/globals.hh | 2 +- src/libstore/local-store.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index e9d721e59..fb8f810c2 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -560,7 +560,7 @@ public: R"( If set to `true` (the default), any non-content-addressed path added or copied to the Nix store (e.g. when substituting from a binary - cache) must have a valid signature, that is, be signed using one of + cache) must have a trustworthy signature, that is, be signed using one of the keys listed in `trusted-public-keys` or `secret-key-files`. Set to `false` to disable signature checking. )"}; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 37302d3a8..b64ae6080 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -751,7 +751,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) registerDrvOutput(info); else - throw Error("cannot register realisation '%s' because it lacks a valid signature", info.outPath.to_string()); + throw Error("cannot register realisation '%s' because it lacks a trustworthy signature", info.outPath.to_string()); } void LocalStore::registerDrvOutput(const Realisation & info) @@ -1266,7 +1266,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { if (checkSigs && pathInfoIsUntrusted(info)) - throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); + throw Error("cannot add path '%s' because it lacks a trustworthy signature", printStorePath(info.path)); addTempRoot(info.path); -- cgit v1.2.3 From a2a8cb10ac17e03691b9f73ae14e5b6edbe66f4e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 22 Sep 2022 14:36:26 -0400 Subject: Dodge "trusted" vs "trustworthy" by being explicit Hopefully this is best! --- src/libstore/globals.hh | 12 +++++++++--- src/libstore/local-store.cc | 4 ++-- 2 files changed, 11 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index fb8f810c2..e2bb0ffc9 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -560,9 +560,15 @@ public: R"( If set to `true` (the default), any non-content-addressed path added or copied to the Nix store (e.g. when substituting from a binary - cache) must have a trustworthy signature, that is, be signed using one of - the keys listed in `trusted-public-keys` or `secret-key-files`. Set - to `false` to disable signature checking. + cache) must have a signature by a key we trust. A trusted key is one + listed in `trusted-public-keys`, or a public key counterpart to a + private key stored in a file listed in `secret-key-files`. + + Set to `false` to disable signature checking and trust all + non-content-addressed paths unconditionally. + + (Content-addressed paths are inherently trustworthy and thus + unaffected by this configuration option.) )"}; Setting extraPlatforms{ diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b64ae6080..d374d4558 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -751,7 +751,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) registerDrvOutput(info); else - throw Error("cannot register realisation '%s' because it lacks a trustworthy signature", info.outPath.to_string()); + throw Error("cannot register realisation '%s' because it lacks a signature by a trusted key", info.outPath.to_string()); } void LocalStore::registerDrvOutput(const Realisation & info) @@ -1266,7 +1266,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { if (checkSigs && pathInfoIsUntrusted(info)) - throw Error("cannot add path '%s' because it lacks a trustworthy signature", printStorePath(info.path)); + throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path)); addTempRoot(info.path); -- cgit v1.2.3 From 6e049ae607b53eba3c9c6bed260a0b39a3f73a70 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 22 Sep 2022 13:59:16 -0500 Subject: Allow pass max-silent-time and build-poll-interval to daemon untrusted These settings seem harmless, they control the same polling functionality that timeout does, but with different behavior. Should be safe for untrusted users to pass in. --- src/libstore/daemon.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index de69b50ee..48dd5c247 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -239,6 +239,8 @@ struct ClientSettings else if (trusted || name == settings.buildTimeout.name || name == settings.buildRepeat.name + || name == settings.maxSilentTime.name + || name == settings.pollInterval.name || name == "connect-timeout" || (name == "builders" && value == "")) settings.set(name, value); -- cgit v1.2.3 From 223f8dace091a0549f86b966b1d935c42bb01efd Mon Sep 17 00:00:00 2001 From: squalus Date: Thu, 22 Sep 2022 12:47:43 -0700 Subject: archive: check close errors when extracting nars --- src/libstore/nar-accessor.cc | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 72d41cc94..398147fc3 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -75,6 +75,9 @@ struct NarAccessor : public FSAccessor createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); } + void closeRegularFile() override + { } + void isExecutable() override { parents.top()->isExecutable = true; -- cgit v1.2.3 From 60e23c8baeb0e28ec163676b4fd4a24c40d89fe9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 23 Sep 2022 13:57:57 -0400 Subject: Apply suggestions from code review Co-authored-by: Valentin Gagarin Co-authored-by: Rune K. Svendsen --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index e2bb0ffc9..75927d395 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -560,7 +560,7 @@ public: R"( If set to `true` (the default), any non-content-addressed path added or copied to the Nix store (e.g. when substituting from a binary - cache) must have a signature by a key we trust. A trusted key is one + cache) must have a signature by a trusted key. A trusted key is one listed in `trusted-public-keys`, or a public key counterpart to a private key stored in a file listed in `secret-key-files`. -- cgit v1.2.3 From a86916eb72f3a774d1d3ca859381b3ea8be5e0af Mon Sep 17 00:00:00 2001 From: Steam Deck User Date: Thu, 6 Oct 2022 11:01:55 -0700 Subject: Make warning about chroot store location more accurate While trying to use an alternate directory for my Nix installation, I noticed that nix's output didn't reflect the updated state directory. This patch corrects that and now prints the warning before attempting to create the directory (if the directory creation fails, it wouldn't have been obvious why nix was attempting to create the directory in the first place). With this patch, I now get the following warning: warning: '/home/deck/.var/app/org.nixos.nix/var/nix' does not exist, so Nix will use '/home/deck/.local/share/nix/root' as a chroot store --- src/libstore/store-api.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 86b12257a..06a9758fc 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1363,9 +1363,9 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para } catch (Error & e) { return std::make_shared(params); } - warn("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore); + warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); } else - debug("'/nix' does not exist, so Nix will use '%s' as a chroot store", chrootStore); + debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); Store::Params params2; params2["root"] = chrootStore; return std::make_shared(params2); -- cgit v1.2.3 From 96eb5ef156641ffc4c5ff01befe73a3419b346bc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 13 Oct 2022 11:45:30 -0700 Subject: Improve Rosetta detection Turns out that one of those *.plist files can exist even if Rosetta is not installed. So let's just try to run an x86_64-darwin binary directly. --- src/libstore/globals.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d724897bb..c3d5f9b8c 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -154,12 +154,12 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist") || - pathExists("/System/Library/LaunchDaemons/com.apple.oahd.plist")) { - if (std::string{SYSTEM} == "x86_64-darwin") - extraPlatforms.insert("aarch64-darwin"); - else if (std::string{SYSTEM} == "aarch64-darwin") + if (std::string{SYSTEM} == "aarch64-darwin") { + if (runProgram(RunOptions {.program = "arch", .args = {"-arch", "x86_64", "/bin/pwd"}, .mergeStderrToStdout = true}).first == 0) { + debug("Rosetta detected"); extraPlatforms.insert("x86_64-darwin"); + } else + debug("Rosetta not detected"); } #endif -- cgit v1.2.3 From 0359d6d12314e46e45f16cccca7e0b38046d2e1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 13 Oct 2022 21:35:16 +0200 Subject: Fix error display if execve() in the builder fails After we've send "\2\n" to the parent, we can't send a serialized exception anymore. It will show up garbled like $ nix-build --store /tmp/nix --expr 'derivation { name = "foo"; system = "x86_64-linux"; builder = "/foo/bar"; }' this derivation will be built: /nix/store/xmdip0z5x1zqpp6gnxld3vqng7zbpapp-foo.drv building '/nix/store/xmdip0z5x1zqpp6gnxld3vqng7zbpapp-foo.drv'... ErrorErrorEexecuting '/foo/bar': No such file or directory error: builder for '/nix/store/xmdip0z5x1zqpp6gnxld3vqng7zbpapp-foo.drv' failed with exit code 1 --- src/libstore/build/local-derivation-goal.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 18b682e13..5cea3b590 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1594,6 +1594,8 @@ void LocalDerivationGoal::runChild() /* Warning: in the child we should absolutely not make any SQLite calls! */ + bool sendException = true; + try { /* child */ commonChildInit(builderOut); @@ -2050,6 +2052,8 @@ void LocalDerivationGoal::runChild() /* Indicate that we managed to set up the build environment. */ writeFull(STDERR_FILENO, std::string("\2\n")); + sendException = false; + /* Execute the program. This should not return. */ if (drv->isBuiltin()) { try { @@ -2103,10 +2107,13 @@ void LocalDerivationGoal::runChild() throw SysError("executing '%1%'", drv->builder); } catch (Error & e) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); + if (sendException) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); + } else + std::cerr << e.msg(); _exit(1); } } -- cgit v1.2.3 From ddd550395070eaee40e758ab630525f7e1162b85 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 14 Oct 2022 00:34:31 -0700 Subject: Use /usr/bin/true --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index c3d5f9b8c..903621da0 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -155,7 +155,7 @@ StringSet Settings::getDefaultExtraPlatforms() // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. if (std::string{SYSTEM} == "aarch64-darwin") { - if (runProgram(RunOptions {.program = "arch", .args = {"-arch", "x86_64", "/bin/pwd"}, .mergeStderrToStdout = true}).first == 0) { + if (runProgram(RunOptions {.program = "arch", .args = {"-arch", "x86_64", "/usr/bin/true"}, .mergeStderrToStdout = true}).first == 0) { debug("Rosetta detected"); extraPlatforms.insert("x86_64-darwin"); } else -- cgit v1.2.3 From 285277a61af8d7ad49f2155166690601aa1a59a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 14 Oct 2022 00:35:33 -0700 Subject: Remove useless debug statements We haven't parsed the '-v' command line flags yet when this code executes, so we can't actually get debug output here. --- src/libstore/globals.cc | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 903621da0..ff658c428 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -154,13 +154,9 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - if (std::string{SYSTEM} == "aarch64-darwin") { - if (runProgram(RunOptions {.program = "arch", .args = {"-arch", "x86_64", "/usr/bin/true"}, .mergeStderrToStdout = true}).first == 0) { - debug("Rosetta detected"); - extraPlatforms.insert("x86_64-darwin"); - } else - debug("Rosetta not detected"); - } + if (std::string{SYSTEM} == "aarch64-darwin" && + runProgram(RunOptions {.program = "arch", .args = {"-arch", "x86_64", "/usr/bin/true"}, .mergeStderrToStdout = true}).first == 0) + extraPlatforms.insert("x86_64-darwin"); #endif return extraPlatforms; -- cgit v1.2.3 From e136d57f26155a9f54dfb0ca00212b2016932104 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Oct 2022 17:48:09 +0200 Subject: Implement BinaryCacheStore::queryPathFromHashPart() --- src/libstore/binary-cache-store.cc | 11 +++++++++++ src/libstore/binary-cache-store.hh | 3 +-- 2 files changed, 12 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9226c4e19..a26770c79 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -331,6 +331,17 @@ bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) return fileExists(narInfoFileFor(storePath)); } +std::optional BinaryCacheStore::queryPathFromHashPart(const std::string & hashPart) +{ + auto pseudoPath = StorePath(hashPart + "-" + MissingName); + try { + auto info = queryPathInfo(pseudoPath); + return info->path; + } catch (InvalidPath &) { + return std::nullopt; + } +} + void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast(); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index ca538b3cb..8c82e2387 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -95,8 +95,7 @@ public: void queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept override; - std::optional queryPathFromHashPart(const std::string & hashPart) override - { unsupported("queryPathFromHashPart"); } + std::optional queryPathFromHashPart(const std::string & hashPart) override; void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) override; -- cgit v1.2.3 From 8e7804273cec1bb3fa05ce09a37f0732b71a72ec Mon Sep 17 00:00:00 2001 From: Austin Kiekintveld Date: Sat, 22 Oct 2022 19:51:22 -0500 Subject: Defer to SSH config files for ForwardAgent option Currently, Nix passes `-a` when it runs commands on a remote machine via SSH, which disables agent forwarding. This causes issues when the `ForwardAgent` option is set in SSH config files, as the command line operation always overrides those. In particular, this causes issues if the command being run is `sudo` and the remote machine is configured with the equivalent of NixOS's `security.pam.enableSSHAgentAuth` option. Not allowing SSH agent forwarding can cause authentication to fail unexpectedly. This can currently be worked around by setting `NIX_SSHOPTS="-A"`, but we should defer to the options in the SSH config files to be least surprising for users. --- src/libstore/ssh.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 1bbad71f2..69bfe3418 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -67,7 +67,7 @@ std::unique_ptr SSHMaster::startCommand(const std::string if (fakeSSH) { args = { "bash", "-c" }; } else { - args = { "ssh", host.c_str(), "-x", "-a" }; + args = { "ssh", host.c_str(), "-x" }; addCommonSSHOpts(args); if (socketPath != "") args.insert(args.end(), {"-S", socketPath}); -- cgit v1.2.3 From f8d01933838f719b2511a9a73a5fa710cdd59496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 27 Oct 2022 11:53:04 +0200 Subject: Pass the right argv when calling the build hook Call it as `['nix', '__build-remote', ... ]` rather than the previous `["__build-remote", "nix __build-remote", ... ]` which seemed to have been most likely unintended --- src/libstore/build/hook-instance.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 1f19ddccc..cb58a1f02 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -16,11 +16,11 @@ HookInstance::HookInstance() buildHookArgs.pop_front(); Strings args; + args.push_back(std::string(baseNameOf(buildHook))); for (auto & arg : buildHookArgs) args.push_back(arg); - args.push_back(std::string(baseNameOf(settings.buildHook.get()))); args.push_back(std::to_string(verbosity)); /* Create a pipe to get the output of the child. */ -- cgit v1.2.3 From 2fde7e0108d70bcba64ebecc5e5c7ee2863e3446 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Nov 2022 16:03:42 +0100 Subject: Split auto UID allocation from cgroups Cgroups are now only used for derivations that require the uid-range range feature. This allows auto UID allocation even on systems that don't have cgroups (like macOS). Also, make things work on modern systems that use cgroups v2 (where there is a single hierarchy and no "systemd" controller). --- src/libstore/build/local-derivation-goal.cc | 19 ++-- src/libstore/build/local-derivation-goal.hh | 7 +- src/libstore/cgroup.cc | 1 + src/libstore/globals.cc | 4 + src/libstore/globals.hh | 6 +- src/libstore/lock.cc | 137 +++++++++++++++------------- src/libstore/lock.hh | 24 ++--- src/libstore/parsed-derivations.cc | 6 ++ src/libstore/parsed-derivations.hh | 2 + 9 files changed, 116 insertions(+), 90 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 64540d262..09da87476 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -160,7 +160,7 @@ void LocalDerivationGoal::tryLocalBuild() { if (useBuildUsers()) { if (!buildUser) - buildUser = acquireUserLock(); + buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1); if (!buildUser) { if (!actLock) @@ -495,8 +495,8 @@ void LocalDerivationGoal::startBuilder() } } - useUidRange = parsedDrv->getRequiredSystemFeatures().count("uid-range"); useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("Systemd-cgroup"); + assert(!useSystemdCgroup); if (useChroot) { @@ -576,7 +576,8 @@ void LocalDerivationGoal::startBuilder() printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); - if (mkdir(chrootRootDir.c_str(), useUidRange ? 0755 : 0750) == -1) + // FIXME: make this 0700 + if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); // FIXME: only make root writable for user namespace builds. @@ -596,8 +597,8 @@ void LocalDerivationGoal::startBuilder() createDirs(chrootRootDir + "/etc"); chownToBuilder(chrootRootDir + "/etc"); - if (useUidRange && (!buildUser || buildUser->getUIDCount() < 65536)) - throw Error("feature 'uid-range' requires '%s' to be enabled", settings.autoAllocateUids.name); + if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536)) + throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ @@ -670,7 +671,7 @@ void LocalDerivationGoal::startBuilder() #endif #endif } else { - if (useUidRange) + if (parsedDrv->useUidRange()) throw Error("feature 'uid-range' is only supported in sandboxed builds"); if (useSystemdCgroup) throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds"); @@ -934,12 +935,12 @@ void LocalDerivationGoal::startBuilder() the calling user (if build users are disabled). */ uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); - uint32_t nrIds = buildUser && useUidRange ? buildUser->getUIDCount() : 1; + uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1; writeFile("/proc/" + std::to_string(pid) + "/uid_map", fmt("%d %d %d", sandboxUid(), hostUid, nrIds)); - if (!useUidRange) + if (!buildUser || buildUser->getUIDCount() == 1) writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); writeFile("/proc/" + std::to_string(pid) + "/gid_map", @@ -1793,7 +1794,7 @@ void LocalDerivationGoal::runChild() throw SysError("mounting /proc"); /* Mount sysfs on /sys. */ - if (useUidRange) { + if (buildUser && buildUser->getUIDCount() != 1) { createDirs(chrootRootDir + "/sys"); if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1) throw SysError("mounting /sys"); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index e6700a383..61b0f9145 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -41,9 +41,6 @@ struct LocalDerivationGoal : public DerivationGoal Path chrootRootDir; - /* Whether to give the build more than 1 UID. */ - bool useUidRange = false; - /* Whether to make the 'systemd' cgroup controller available to the build. */ bool useSystemdCgroup = false; @@ -99,8 +96,8 @@ struct LocalDerivationGoal : public DerivationGoal result. */ std::map prevInfos; - uid_t sandboxUid() { return usingUserNamespace ? (useUidRange ? 0 : 1000) : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? (useUidRange ? 0 : 100) : buildUser->getGID(); } + uid_t sandboxUid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } const static Path homeDir; diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 5d31609da..56e980be3 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -13,6 +13,7 @@ namespace nix { +// FIXME: obsolete, check for cgroup2 std::map getCgroups(const Path & cgroupFile) { std::map cgroups; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ff658c428..b7f55cae7 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -130,6 +130,10 @@ StringSet Settings::getDefaultSystemFeatures() actually require anything special on the machines. */ StringSet features{"nixos-test", "benchmark", "big-parallel"}; + #if __linux__ + features.insert("uid-range"); + #endif + #if __linux__ if (access("/dev/kvm", R_OK | W_OK) == 0) features.insert("kvm"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d3e86cc55..be741a830 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -46,6 +46,8 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; +const uint32_t maxIdsPerBuild = 1 << 16; + class Settings : public Config { unsigned int getDefaultCores(); @@ -279,12 +281,10 @@ public: Setting autoAllocateUids{this, false, "auto-allocate-uids", "Whether to allocate UIDs for builders automatically."}; - const uint32_t idsPerBuild = 1 << 16; - Setting startId{this, 872415232, "start-id", "The first UID and GID to use for dynamic ID allocation."}; - Setting uidCount{this, idsPerBuild * 128, "id-count", + Setting uidCount{this, maxIdsPerBuild * 128, "id-count", "The number of UIDs/GIDs to use for dynamic ID allocation."}; #endif diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index cc3977496..ecc51cebe 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -20,12 +20,8 @@ struct SimpleUserLock : UserLock killUser(uid); } - std::pair getUIDRange() override - { - assert(uid); - return {uid, uid}; - } - + uid_t getUID() override { assert(uid); return uid; } + uid_t getUIDCount() override { return 1; } gid_t getGID() override { assert(gid); return gid; } std::vector getSupplementaryGIDs() override { return supplementaryGIDs; } @@ -115,48 +111,65 @@ struct SimpleUserLock : UserLock } }; -#if __linux__ -struct CgroupUserLock : UserLock +struct AutoUserLock : UserLock { AutoCloseFD fdUserLock; - uid_t uid; + uid_t firstUid = 0; + uid_t nrIds = 1; + #if __linux__ + std::optional cgroup; + #endif + + ~AutoUserLock() + { + // Get rid of our cgroup, ignoring errors. + if (cgroup) rmdir(cgroup->c_str()); + } void kill() override { + #if __linux__ if (cgroup) { + printError("KILL CGROUP %s", *cgroup); destroyCgroup(*cgroup); - cgroup.reset(); + if (mkdir(cgroup->c_str(), 0755) == -1) + throw SysError("creating cgroup '%s'", *cgroup); + } else + #endif + { + assert(firstUid); + printError("KILL USER %d", firstUid); + killUser(firstUid); } } - std::pair getUIDRange() override - { - assert(uid); - return {uid, uid + settings.idsPerBuild - 1}; - } + uid_t getUID() override { assert(firstUid); return firstUid; } + + gid_t getUIDCount() override { return nrIds; } gid_t getGID() override { // We use the same GID ranges as for the UIDs. - assert(uid); - return uid; + assert(firstUid); + return firstUid; } std::vector getSupplementaryGIDs() override { return {}; } - static std::unique_ptr acquire() + static std::unique_ptr acquire(uid_t nrIds) { settings.requireExperimentalFeature(Xp::AutoAllocateUids); assert(settings.startId > 0); - assert(settings.startId % settings.idsPerBuild == 0); - assert(settings.uidCount % settings.idsPerBuild == 0); + assert(settings.startId % maxIdsPerBuild == 0); + assert(settings.uidCount % maxIdsPerBuild == 0); assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); + assert(nrIds <= maxIdsPerBuild); // FIXME: check whether the id range overlaps any known users createDirs(settings.nixStateDir + "/userpool2"); - size_t nrSlots = settings.uidCount / settings.idsPerBuild; + size_t nrSlots = settings.uidCount / maxIdsPerBuild; for (size_t i = 0; i < nrSlots; i++) { debug("trying user slot '%d'", i); @@ -170,62 +183,64 @@ struct CgroupUserLock : UserLock throw SysError("opening user lock '%s'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { - auto lock = std::make_unique(); - lock->fdUserLock = std::move(fd); - lock->uid = settings.startId + i * settings.idsPerBuild; - auto s = drainFD(lock->fdUserLock.get()); - if (s != "") lock->cgroup = s; - return lock; - } - } + auto s = drainFD(fd.get()); - return nullptr; - } + #if __linux__ + if (s != "") { + /* Kill the old cgroup, to ensure there are no + processes left over from an interrupted build. */ + destroyCgroup(s); + } + #endif - std::optional cgroup; + if (ftruncate(fd.get(), 0) == -1) + throw Error("truncating user lock"); - std::optional getCgroup() override - { - if (!cgroup) { - /* Create a systemd cgroup since that's the minimum - required by systemd-nspawn. */ - auto ourCgroups = getCgroups("/proc/self/cgroup"); - auto systemdCgroup = ourCgroups["systemd"]; - if (systemdCgroup == "") - throw Error("'systemd' cgroup does not exist"); + auto lock = std::make_unique(); + lock->fdUserLock = std::move(fd); + lock->firstUid = settings.startId + i * maxIdsPerBuild; + lock->nrIds = nrIds; - auto hostCgroup = canonPath("/sys/fs/cgroup/systemd/" + systemdCgroup); + if (nrIds > 1) { + auto ourCgroups = getCgroups("/proc/self/cgroup"); + auto ourCgroup = ourCgroups[""]; + if (ourCgroup == "") + throw Error("cannot determine cgroup name from /proc/self/cgroup"); - if (!pathExists(hostCgroup)) - throw Error("expected cgroup directory '%s'", hostCgroup); + auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup); - cgroup = fmt("%s/nix-%d", hostCgroup, uid); + printError("PARENT CGROUP = %s", ourCgroupPath); - destroyCgroup(*cgroup); + if (!pathExists(ourCgroupPath)) + throw Error("expected cgroup directory '%s'", ourCgroupPath); - if (mkdir(cgroup->c_str(), 0755) == -1) - throw SysError("creating cgroup '%s'", *cgroup); + lock->cgroup = fmt("%s/nix-build-%d", ourCgroupPath, lock->firstUid); - /* Record the cgroup in the lock file. This ensures that - if we subsequently get executed under a different parent - cgroup, we kill the previous cgroup first. */ - if (ftruncate(fdUserLock.get(), 0) == -1) - throw Error("truncating user lock"); - writeFull(fdUserLock.get(), *cgroup); + printError("CHILD CGROUP = %s", *lock->cgroup); + + /* Record the cgroup in the lock file. This ensures that + if we subsequently get executed under a different parent + cgroup, we kill the previous cgroup first. */ + writeFull(lock->fdUserLock.get(), *lock->cgroup); + } + + return lock; + } } - return cgroup; - }; + return nullptr; + } + + #if __linux__ + std::optional getCgroup() override { return cgroup; } + #endif }; -#endif -std::unique_ptr acquireUserLock() +std::unique_ptr acquireUserLock(uid_t nrIds) { - #if __linux__ if (settings.autoAllocateUids) - return CgroupUserLock::acquire(); + return AutoUserLock::acquire(nrIds); else - #endif return SimpleUserLock::acquire(); } diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 4b6d34069..62676a523 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -11,18 +11,16 @@ struct UserLock virtual ~UserLock() { } /* Get the first and last UID. */ - virtual std::pair getUIDRange() = 0; - - /* Get the first UID. */ - uid_t getUID() + std::pair getUIDRange() { - return getUIDRange().first; + auto first = getUID(); + return {first, first + getUIDCount() - 1}; } - uid_t getUIDCount() - { - return getUIDRange().second - getUIDRange().first + 1; - } + /* Get the first UID. */ + virtual uid_t getUID() = 0; + + virtual uid_t getUIDCount() = 0; virtual gid_t getGID() = 0; @@ -31,12 +29,14 @@ struct UserLock /* Kill any processes currently executing as this user. */ virtual void kill() = 0; + #if __linux__ virtual std::optional getCgroup() { return {}; }; + #endif }; -/* Acquire a user lock. Note that this may return nullptr if no user - is available. */ -std::unique_ptr acquireUserLock(); +/* Acquire a user lock for a UID range of size `nrIds`. Note that this + may return nullptr if no user is available. */ +std::unique_ptr acquireUserLock(uid_t nrIds); bool useBuildUsers(); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index f2288a04e..487dbcfbb 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -90,6 +90,7 @@ std::optional ParsedDerivation::getStringsAttr(const std::string & name StringSet ParsedDerivation::getRequiredSystemFeatures() const { + // FIXME: cache this? StringSet res; for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) res.insert(i); @@ -125,6 +126,11 @@ bool ParsedDerivation::substitutesAllowed() const return getBoolAttr("allowSubstitutes", true); } +bool ParsedDerivation::useUidRange() const +{ + return getRequiredSystemFeatures().count("uid-range"); +} + static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 95bec21e8..bfb3857c0 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -38,6 +38,8 @@ public: bool substitutesAllowed() const; + bool useUidRange() const; + std::optional prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths); }; -- cgit v1.2.3 From 05d258667d12b2decda87024a59250c43343b509 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Nov 2022 08:00:29 -0800 Subject: Fix build on macOS --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/globals.hh | 26 +++++++++++++++++++++----- src/libstore/lock.cc | 10 ++++------ src/libstore/lock.hh | 2 ++ 4 files changed, 28 insertions(+), 12 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 09da87476..45ea9968f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -659,7 +659,7 @@ void LocalDerivationGoal::startBuilder() } #else - if (useUidRange) + if (parsedDrv->useUidRange()) throw Error("feature 'uid-range' is not supported on this platform"); if (useSystemdCgroup) throw Error("feature 'systemd-cgroup' is not supported on this platform"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index be741a830..88fe72202 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -46,7 +46,13 @@ struct PluginFilesSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; -const uint32_t maxIdsPerBuild = 1 << 16; +const uint32_t maxIdsPerBuild = + #if __linux__ + 1 << 16 + #else + 1 + #endif + ; class Settings : public Config { @@ -277,16 +283,26 @@ public: multi-user settings with untrusted users. )"}; - #if __linux__ Setting autoAllocateUids{this, false, "auto-allocate-uids", "Whether to allocate UIDs for builders automatically."}; - Setting startId{this, 872415232, "start-id", + Setting startId{this, + #if __linux__ + 872415232, + #else + 56930, + #endif + "start-id", "The first UID and GID to use for dynamic ID allocation."}; - Setting uidCount{this, maxIdsPerBuild * 128, "id-count", + Setting uidCount{this, + #if __linux__ + maxIdsPerBuild * 128, + #else + 128, + #endif + "id-count", "The number of UIDs/GIDs to use for dynamic ID allocation."}; - #endif Setting impersonateLinux26{this, false, "impersonate-linux-26", "Whether to impersonate a Linux 2.6 machine on newer kernels.", diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index ecc51cebe..f9892bb91 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -122,15 +122,16 @@ struct AutoUserLock : UserLock ~AutoUserLock() { + #if __linux__ // Get rid of our cgroup, ignoring errors. if (cgroup) rmdir(cgroup->c_str()); + #endif } void kill() override { #if __linux__ if (cgroup) { - printError("KILL CGROUP %s", *cgroup); destroyCgroup(*cgroup); if (mkdir(cgroup->c_str(), 0755) == -1) throw SysError("creating cgroup '%s'", *cgroup); @@ -138,7 +139,6 @@ struct AutoUserLock : UserLock #endif { assert(firstUid); - printError("KILL USER %d", firstUid); killUser(firstUid); } } @@ -201,6 +201,7 @@ struct AutoUserLock : UserLock lock->firstUid = settings.startId + i * maxIdsPerBuild; lock->nrIds = nrIds; + #if __linux__ if (nrIds > 1) { auto ourCgroups = getCgroups("/proc/self/cgroup"); auto ourCgroup = ourCgroups[""]; @@ -209,20 +210,17 @@ struct AutoUserLock : UserLock auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup); - printError("PARENT CGROUP = %s", ourCgroupPath); - if (!pathExists(ourCgroupPath)) throw Error("expected cgroup directory '%s'", ourCgroupPath); lock->cgroup = fmt("%s/nix-build-%d", ourCgroupPath, lock->firstUid); - printError("CHILD CGROUP = %s", *lock->cgroup); - /* Record the cgroup in the lock file. This ensures that if we subsequently get executed under a different parent cgroup, we kill the previous cgroup first. */ writeFull(lock->fdUserLock.get(), *lock->cgroup); } + #endif return lock; } diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 62676a523..b5536408c 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -4,6 +4,8 @@ #include +#include + namespace nix { struct UserLock -- cgit v1.2.3 From 6c6eff8ac40e2f5d7b6ff8e772feebb1aa484039 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Nov 2022 17:24:12 +0100 Subject: Remove the SystemdCgroup feature --- src/libstore/build/local-derivation-goal.cc | 23 ++++++++--------------- src/libstore/build/local-derivation-goal.hh | 4 ---- 2 files changed, 8 insertions(+), 19 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 45ea9968f..e652c425c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -495,9 +495,6 @@ void LocalDerivationGoal::startBuilder() } } - useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("Systemd-cgroup"); - assert(!useSystemdCgroup); - if (useChroot) { /* Allow a user-configurable set of directories from the @@ -649,20 +646,18 @@ void LocalDerivationGoal::startBuilder() dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); } - if (useSystemdCgroup) { - settings.requireExperimentalFeature(Xp::SystemdCgroup); - std::optional cgroup; - if (!buildUser || !(cgroup = buildUser->getCgroup())) - throw Error("feature 'systemd-cgroup' requires 'auto-allocate-uids = true' in nix.conf"); - chownToBuilder(*cgroup); - chownToBuilder(*cgroup + "/cgroup.procs"); + if (buildUser) { + if (auto cgroup = buildUser->getCgroup()) { + chownToBuilder(*cgroup); + chownToBuilder(*cgroup + "/cgroup.procs"); + chownToBuilder(*cgroup + "/cgroup.threads"); + //chownToBuilder(*cgroup + "/cgroup.subtree_control"); + } } #else if (parsedDrv->useUidRange()) throw Error("feature 'uid-range' is not supported on this platform"); - if (useSystemdCgroup) - throw Error("feature 'systemd-cgroup' is not supported on this platform"); #if __APPLE__ /* We don't really have any parent prep work to do (yet?) All work happens in the child, instead. */ @@ -673,8 +668,6 @@ void LocalDerivationGoal::startBuilder() } else { if (parsedDrv->useUidRange()) throw Error("feature 'uid-range' is only supported in sandboxed builds"); - if (useSystemdCgroup) - throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds"); } if (needsHashRewrite() && pathExists(homeDir)) @@ -1845,7 +1838,7 @@ void LocalDerivationGoal::runChild() /* Unshare the cgroup namespace. This means /proc/self/cgroup will show the child's cgroup as '/' rather than whatever it is in the parent. */ - if (useSystemdCgroup && unshare(CLONE_NEWCGROUP) == -1) + if (buildUser && buildUser->getUIDCount() != 1 && unshare(CLONE_NEWCGROUP) == -1) throw SysError("unsharing cgroup namespace"); /* Do the chroot(). */ diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 61b0f9145..070ae53f3 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -41,10 +41,6 @@ struct LocalDerivationGoal : public DerivationGoal Path chrootRootDir; - /* Whether to make the 'systemd' cgroup controller available to - the build. */ - bool useSystemdCgroup = false; - /* RAII object to delete the chroot directory. */ std::shared_ptr autoDelChroot; -- cgit v1.2.3 From e7ed9ae0c711c4efd83756b16379549ecff52355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 4 Nov 2022 14:19:31 +0100 Subject: Restrict `readFile` context to references that appear in the string When calling `builtins.readFile` on a store path, the references of that path are currently added to the resulting string's context. This change makes those references the *possible* context of the string, but filters them to keep only the references whose hash actually appears in the string, similarly to what is done for determining the runtime references of a path. --- src/libstore/references.cc | 53 ++++++++++++++++++++++++++++++---------------- src/libstore/references.hh | 13 ++++++++++++ 2 files changed, 48 insertions(+), 18 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 34dce092c..3bb297fc8 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -67,20 +67,12 @@ void RefScanSink::operator () (std::string_view data) } -std::pair scanForReferences( - const std::string & path, - const StorePathSet & refs) -{ - HashSink hashSink { htSHA256 }; - auto found = scanForReferences(hashSink, path, refs); - auto hash = hashSink.finish(); - return std::pair(found, hash); -} +PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map && backMap) + : RefScanSink(std::move(hashes)) + , backMap(std::move(backMap)) +{ } -StorePathSet scanForReferences( - Sink & toTee, - const Path & path, - const StorePathSet & refs) +PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) { StringSet hashes; std::map backMap; @@ -92,14 +84,14 @@ StorePathSet scanForReferences( hashes.insert(hashPart); } - /* Look for the hashes in the NAR dump of the path. */ - RefScanSink refsSink(std::move(hashes)); - TeeSink sink { refsSink, toTee }; - dumpPath(path, sink); + return PathRefScanSink(std::move(hashes), std::move(backMap)); +} +StorePathSet PathRefScanSink::getResultPaths() +{ /* Map the hashes found back to their store paths. */ StorePathSet found; - for (auto & i : refsSink.getResult()) { + for (auto & i : getResult()) { auto j = backMap.find(i); assert(j != backMap.end()); found.insert(j->second); @@ -109,6 +101,31 @@ StorePathSet scanForReferences( } +std::pair scanForReferences( + const std::string & path, + const StorePathSet & refs) +{ + HashSink hashSink { htSHA256 }; + auto found = scanForReferences(hashSink, path, refs); + auto hash = hashSink.finish(); + return std::pair(found, hash); +} + +StorePathSet scanForReferences( + Sink & toTee, + const Path & path, + const StorePathSet & refs) +{ + PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); + TeeSink sink { refsSink, toTee }; + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + return refsSink.getResultPaths(); +} + + RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) : from(from), to(to), nextSink(nextSink) { diff --git a/src/libstore/references.hh b/src/libstore/references.hh index a6119c861..6f381f96c 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -27,6 +27,19 @@ public: void operator () (std::string_view data) override; }; +class PathRefScanSink : public RefScanSink +{ + std::map backMap; + + PathRefScanSink(StringSet && hashes, std::map && backMap); + +public: + + static PathRefScanSink fromPaths(const StorePathSet & refs); + + StorePathSet getResultPaths(); +}; + struct RewritingSink : Sink { std::string from, to, prev; -- cgit v1.2.3 From 07f2cb1e8f03784041475c27c2ba0aac7be6c0b7 Mon Sep 17 00:00:00 2001 From: Tobias Mayer Date: Thu, 10 Nov 2022 08:59:23 +0100 Subject: libstore: link to aws-crt-cpp This change is needed to support aws-sdk-cpp 1.10 and newer. I opted not to make this dependent on the sdk version because the crt dependency has been in the interface of the older sdk as well, and it was only coincidence that libstore didn't make use of any privately defined symbols directly. --- src/libstore/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 1d26ac918..8f28bec6c 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -20,7 +20,7 @@ endif $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) ifeq ($(ENABLE_S3), 1) - libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core + libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp endif ifdef HOST_SOLARIS -- cgit v1.2.3 From 7e162c69fe6cbfb929b5356a7df9de5c25c22565 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 14 Nov 2022 16:26:20 +0100 Subject: derivation-goal: Fix `requires non-existing output` error It occurred when a output of the dependency was already available, so it didn't need rebuilding and didn't get added to the inputDrvOutputs. This process-related info wasn't suitable for the purpose of finding the actual input paths for the builder. It is better to do this in absolute terms by querying the store. --- src/libstore/build/derivation-goal.cc | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 41d2e2a1c..00e375fe9 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -528,13 +528,31 @@ void DerivationGoal::inputsRealised() /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - for (auto & j : wantedDepOutputs) - if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) + for (auto & j : wantedDepOutputs) { + /* TODO (impure derivations-induced tech debt): + Tracking input derivation outputs statefully through the + goals is error prone and has led to bugs. + For a robust nix, we need to move towards the `else` branch, + which does not rely on goal state to match up with the + reality of the store, which is our real source of truth. + However, the impure derivations feature still relies on this + fragile way of doing things, because its builds do not have + a representation in the store, which is a usability problem + in itself */ + if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { worker.store.computeFSClosure(*outPath, inputPaths); - else - throw Error( - "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + } + else { + auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); + auto outMapPath = outMap.find(j); + if (outMapPath == outMap.end()) { + throw Error( + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + } + worker.store.computeFSClosure(outMapPath->second, inputPaths); + } + } } } -- cgit v1.2.3 From bcd298d39bffbb1a79ae5ce2c4eec8c45fa9f2a0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 15 Nov 2022 17:57:40 +0100 Subject: libstore/derivation-goal: Elaborate a TODO for performance concern --- src/libstore/build/derivation-goal.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 00e375fe9..2f22dbcec 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -538,7 +538,8 @@ void DerivationGoal::inputsRealised() However, the impure derivations feature still relies on this fragile way of doing things, because its builds do not have a representation in the store, which is a usability problem - in itself */ + in itself. When implementing this logic entirely with lookups + make sure that they're cached. */ if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { worker.store.computeFSClosure(*outPath, inputPaths); } -- cgit v1.2.3 From 09f00dd4d01aa1b6866978d162022133e521614f Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Wed, 16 Nov 2022 16:49:49 +0100 Subject: Replace src/libutil/json.cc with nlohmann json generation --- src/libstore/binary-cache-store.cc | 18 +++-------- src/libstore/build/derivation-goal.cc | 1 - src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/nar-accessor.cc | 30 +++++++++--------- src/libstore/nar-accessor.hh | 6 ++-- src/libstore/parsed-derivations.cc | 16 +++------- src/libstore/remote-fs-accessor.cc | 8 ++--- src/libstore/store-api.cc | 48 ++++++++++++++--------------- src/libstore/store-api.hh | 4 +-- 9 files changed, 57 insertions(+), 76 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a26770c79..12d0c32fb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -9,7 +9,6 @@ #include "remote-fs-accessor.hh" #include "nar-info-disk-cache.hh" #include "nar-accessor.hh" -#include "json.hh" #include "thread-pool.hh" #include "callback.hh" @@ -194,19 +193,12 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { - std::ostringstream jsonOut; - - { - JSONObject jsonRoot(jsonOut); - jsonRoot.attr("version", 1); - - { - auto res = jsonRoot.placeholder("root"); - listNar(res, ref(narAccessor), "", true); - } - } + nlohmann::json j = { + {"version", 1}, + {"root", listNar(ref(narAccessor), "", true)}, + }; - upsertFile(std::string(info.path.hashPart()) + ".ls", jsonOut.str(), "application/json"); + upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); } /* Optionally maintain an index of DWARF debug info files diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 00e375fe9..1938f4bcb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -7,7 +7,6 @@ #include "finally.hh" #include "util.hh" #include "archive.hh" -#include "json.hh" #include "compression.hh" #include "worker-protocol.hh" #include "topo-sort.hh" diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 5cea3b590..c786e8613 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -8,7 +8,6 @@ #include "finally.hh" #include "util.hh" #include "archive.hh" -#include "json.hh" #include "compression.hh" #include "daemon.hh" #include "worker-protocol.hh" @@ -56,6 +55,7 @@ #include #include +#include namespace nix { diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 398147fc3..9a0003588 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -1,6 +1,5 @@ #include "nar-accessor.hh" #include "archive.hh" -#include "json.hh" #include #include @@ -243,42 +242,43 @@ ref makeLazyNarAccessor(const std::string & listing, return make_ref(listing, getNarBytes); } -void listNar(JSONPlaceholder & res, ref accessor, - const Path & path, bool recurse) +using nlohmann::json; +json listNar(ref accessor, const Path & path, bool recurse) { auto st = accessor->stat(path); - auto obj = res.object(); + json obj = json::object(); switch (st.type) { case FSAccessor::Type::tRegular: - obj.attr("type", "regular"); - obj.attr("size", st.fileSize); + obj["type"] = "regular"; + obj["size"] = st.fileSize; if (st.isExecutable) - obj.attr("executable", true); + obj["executable"] = true; if (st.narOffset) - obj.attr("narOffset", st.narOffset); + obj["narOffset"] = st.narOffset; break; case FSAccessor::Type::tDirectory: - obj.attr("type", "directory"); + obj["type"] = "directory"; { - auto res2 = obj.object("entries"); + obj["entries"] = json::object(); + json &res2 = obj["entries"]; for (auto & name : accessor->readDirectory(path)) { if (recurse) { - auto res3 = res2.placeholder(name); - listNar(res3, accessor, path + "/" + name, true); + res2[name] = listNar(accessor, path + "/" + name, true); } else - res2.object(name); + res2[name] = json::object(); } } break; case FSAccessor::Type::tSymlink: - obj.attr("type", "symlink"); - obj.attr("target", accessor->readLink(path)); + obj["type"] = "symlink"; + obj["target"] = accessor->readLink(path); break; default: throw Error("path '%s' does not exist in NAR", path); } + return obj; } } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index c2241a04c..7d998ae0b 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -2,6 +2,7 @@ #include +#include #include "fs-accessor.hh" namespace nix { @@ -24,11 +25,8 @@ ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); -class JSONPlaceholder; - /* Write a JSON representation of the contents of a NAR (except file contents). */ -void listNar(JSONPlaceholder & res, ref accessor, - const Path & path, bool recurse); +nlohmann::json listNar(ref accessor, const Path & path, bool recurse); } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index f2288a04e..59a30db10 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -2,7 +2,6 @@ #include #include -#include "json.hh" namespace nix { @@ -144,16 +143,11 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s auto e = json.find("exportReferencesGraph"); if (e != json.end() && e->is_object()) { for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(store.parseStorePath(p.get())); - store.pathInfoToJSON(jsonRoot, - store.exportReferences(storePaths, inputPaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh + StorePathSet storePaths; + for (auto & p : *i) + storePaths.insert(store.parseStorePath(p.get())); + json[i.key()] = store.pathInfoToJSON( + store.exportReferences(storePaths, inputPaths), false, true); } } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 0ce335646..fcfb527f5 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -1,6 +1,6 @@ +#include #include "remote-fs-accessor.hh" #include "nar-accessor.hh" -#include "json.hh" #include #include @@ -38,10 +38,8 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str if (cacheDir != "") { try { - std::ostringstream str; - JSONPlaceholder jsonRoot(str); - listNar(jsonRoot, narAccessor, "", true); - writeFile(makeCacheFile(hashPart, "ls"), str.str()); + nlohmann::json j = listNar(narAccessor, "", true); + writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreException(); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 06a9758fc..8811ab578 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,14 +6,16 @@ #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" -#include "json.hh" #include "url.hh" #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +#include #include +using json = nlohmann::json; + namespace nix { @@ -838,56 +840,53 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor return paths; } - -void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, +json Store::pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase, AllowInvalidFlag allowInvalid) { - auto jsonList = jsonOut.list(); + json::array_t jsonList = json::array(); for (auto & storePath : storePaths) { - auto jsonPath = jsonList.object(); + auto& jsonPath = jsonList.emplace_back(json::object()); try { auto info = queryPathInfo(storePath); - jsonPath.attr("path", printStorePath(info->path)); - jsonPath - .attr("narHash", info->narHash.to_string(hashBase, true)) - .attr("narSize", info->narSize); + jsonPath["path"] = printStorePath(info->path); + jsonPath["narHash"] = info->narHash.to_string(hashBase, true); + jsonPath["narSize"] = info->narSize; { - auto jsonRefs = jsonPath.list("references"); + auto& jsonRefs = (jsonPath["references"] = json::array()); for (auto & ref : info->references) - jsonRefs.elem(printStorePath(ref)); + jsonRefs.emplace_back(printStorePath(ref)); } if (info->ca) - jsonPath.attr("ca", renderContentAddress(info->ca)); + jsonPath["ca"] = renderContentAddress(info->ca); std::pair closureSizes; if (showClosureSize) { closureSizes = getClosureSize(info->path); - jsonPath.attr("closureSize", closureSizes.first); + jsonPath["closureSize"] = closureSizes.first; } if (includeImpureInfo) { if (info->deriver) - jsonPath.attr("deriver", printStorePath(*info->deriver)); + jsonPath["deriver"] = printStorePath(*info->deriver); if (info->registrationTime) - jsonPath.attr("registrationTime", info->registrationTime); + jsonPath["registrationTime"] = info->registrationTime; if (info->ultimate) - jsonPath.attr("ultimate", info->ultimate); + jsonPath["ultimate"] = info->ultimate; if (!info->sigs.empty()) { - auto jsonSigs = jsonPath.list("signatures"); for (auto & sig : info->sigs) - jsonSigs.elem(sig); + jsonPath["signatures"].push_back(sig); } auto narInfo = std::dynamic_pointer_cast( @@ -895,21 +894,22 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (narInfo) { if (!narInfo->url.empty()) - jsonPath.attr("url", narInfo->url); + jsonPath["url"] = narInfo->url; if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true)); + jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true); if (narInfo->fileSize) - jsonPath.attr("downloadSize", narInfo->fileSize); + jsonPath["downloadSize"] = narInfo->fileSize; if (showClosureSize) - jsonPath.attr("closureDownloadSize", closureSizes.second); + jsonPath["closureDownloadSize"] = closureSizes.second; } } } catch (InvalidPath &) { - jsonPath.attr("path", printStorePath(storePath)); - jsonPath.attr("valid", false); + jsonPath["path"] = printStorePath(storePath); + jsonPath["valid"] = false; } } + return jsonList; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index c8a667c6d..151ec10d6 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -14,6 +14,7 @@ #include "path-info.hh" #include "repair-flag.hh" +#include #include #include #include @@ -68,7 +69,6 @@ struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; -class JSONPlaceholder; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; @@ -512,7 +512,7 @@ public: variable elements such as the registration time are included. If ‘showClosureSize’ is true, the closure size of each path is included. */ - void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, + nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); -- cgit v1.2.3 From f423d4425f6573206045e9626812002906d9493d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 17 Nov 2022 11:56:45 +0100 Subject: Fix segfault in unprivileged mode --- src/libstore/build/local-derivation-goal.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 070ae53f3..f92280aa1 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -92,8 +92,8 @@ struct LocalDerivationGoal : public DerivationGoal result. */ std::map prevInfos; - uid_t sandboxUid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } - gid_t sandboxGid() { return usingUserNamespace ? (buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } + uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } + gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } const static Path homeDir; -- cgit v1.2.3 From 128910ba23f586ba1765a137ecff23cfd22cff89 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Nov 2022 10:39:28 +0100 Subject: Separate cgroup support from auto-uid-allocation The new experimental feature 'cgroups' enables the use of cgroups for all builds. This allows better containment and enables setting resource limits and getting some build stats. --- src/libstore/build/local-derivation-goal.cc | 114 ++++++++++++++++++++-------- src/libstore/build/local-derivation-goal.hh | 7 ++ src/libstore/lock.cc | 71 ----------------- src/libstore/lock.hh | 7 -- 4 files changed, 91 insertions(+), 108 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index e652c425c..2d1e093ca 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -15,6 +15,7 @@ #include "topo-sort.hh" #include "callback.hh" #include "json-utils.hh" +#include "cgroup.hh" #include #include @@ -129,26 +130,36 @@ void LocalDerivationGoal::killChild() if (pid != -1) { worker.childTerminated(this); - if (buildUser) { - /* If we're using a build user, then there is a tricky - race condition: if we kill the build user before the - child has done its setuid() to the build user uid, then - it won't be killed, and we'll potentially lock up in - pid.wait(). So also send a conventional kill to the - child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ - buildUser->kill(); - pid.wait(); - } else - pid.kill(); + /* If we're using a build user, then there is a tricky race + condition: if we kill the build user before the child has + done its setuid() to the build user uid, then it won't be + killed, and we'll potentially lock up in pid.wait(). So + also send a conventional kill to the child. */ + ::kill(-pid, SIGKILL); /* ignore the result */ + + killSandbox(); - assert(pid == -1); + pid.wait(); } DerivationGoal::killChild(); } +void LocalDerivationGoal::killSandbox() +{ + if (cgroup) { + destroyCgroup(*cgroup); + } + + else if (buildUser) { + auto uid = buildUser->getUID(); + assert(uid != 0); + killUser(uid); + } +} + + void LocalDerivationGoal::tryLocalBuild() { unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { @@ -169,10 +180,6 @@ void LocalDerivationGoal::tryLocalBuild() { worker.waitForAWhile(shared_from_this()); return; } - - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); } actLock.reset(); @@ -263,7 +270,7 @@ void LocalDerivationGoal::cleanupPostChildKill() malicious user from leaving behind a process that keeps files open and modifies them after they have been chown'ed to root. */ - if (buildUser) buildUser->kill(); + killSandbox(); /* Terminate the recursive Nix daemon. */ stopDaemon(); @@ -356,6 +363,55 @@ static void linkOrCopy(const Path & from, const Path & to) void LocalDerivationGoal::startBuilder() { + if ((buildUser && buildUser->getUIDCount() != 1) + || settings.isExperimentalFeatureEnabled(Xp::Cgroups)) + { + #if __linux__ + auto ourCgroups = getCgroups("/proc/self/cgroup"); + auto ourCgroup = ourCgroups[""]; + if (ourCgroup == "") + throw Error("cannot determine cgroup name from /proc/self/cgroup"); + + auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup); + + if (!pathExists(ourCgroupPath)) + throw Error("expected cgroup directory '%s'", ourCgroupPath); + + static std::atomic counter{0}; + + cgroup = buildUser + ? fmt("%s/nix-build-uid-%d", ourCgroupPath, buildUser->getUID()) + : fmt("%s/nix-build-pid-%d-%d", ourCgroupPath, getpid(), counter++); + + debug("using cgroup '%s'", *cgroup); + + /* When using a build user, record the cgroup we used for that + user so that if we got interrupted previously, we can kill + any left-over cgroup first. */ + if (buildUser) { + auto cgroupsDir = settings.nixStateDir + "/cgroups"; + createDirs(cgroupsDir); + + auto cgroupFile = fmt("%s/%d", cgroupsDir, buildUser->getUID()); + + if (pathExists(cgroupFile)) { + auto prevCgroup = readFile(cgroupFile); + destroyCgroup(prevCgroup); + } + + writeFile(cgroupFile, *cgroup); + } + + #else + throw Error("cgroups are not supported on this platform"); + #endif + } + + /* Make sure that no other processes are executing under the + sandbox uids. This must be done before any chownToBuilder() + calls. */ + killSandbox(); + /* Right platform? */ if (!parsedDrv->canBuildLocally(worker.store)) throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", @@ -646,13 +702,13 @@ void LocalDerivationGoal::startBuilder() dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); } - if (buildUser) { - if (auto cgroup = buildUser->getCgroup()) { - chownToBuilder(*cgroup); - chownToBuilder(*cgroup + "/cgroup.procs"); - chownToBuilder(*cgroup + "/cgroup.threads"); - //chownToBuilder(*cgroup + "/cgroup.subtree_control"); - } + if (cgroup) { + if (mkdir(cgroup->c_str(), 0755) != 0) + throw SysError("creating cgroup '%s'", *cgroup); + chownToBuilder(*cgroup); + chownToBuilder(*cgroup + "/cgroup.procs"); + chownToBuilder(*cgroup + "/cgroup.threads"); + //chownToBuilder(*cgroup + "/cgroup.subtree_control"); } #else @@ -965,10 +1021,8 @@ void LocalDerivationGoal::startBuilder() } /* Move the child into its own cgroup. */ - if (buildUser) { - if (auto cgroup = buildUser->getCgroup()) - writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); - } + if (cgroup) + writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); @@ -1838,7 +1892,7 @@ void LocalDerivationGoal::runChild() /* Unshare the cgroup namespace. This means /proc/self/cgroup will show the child's cgroup as '/' rather than whatever it is in the parent. */ - if (buildUser && buildUser->getUIDCount() != 1 && unshare(CLONE_NEWCGROUP) == -1) + if (cgroup && unshare(CLONE_NEWCGROUP) == -1) throw SysError("unsharing cgroup namespace"); /* Do the chroot(). */ diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index f92280aa1..1ec6b3649 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -15,6 +15,9 @@ struct LocalDerivationGoal : public DerivationGoal /* The process ID of the builder. */ Pid pid; + /* The cgroup of the builder, if any. */ + std::optional cgroup; + /* The temporary directory. */ Path tmpDir; @@ -197,6 +200,10 @@ struct LocalDerivationGoal : public DerivationGoal /* Forcibly kill the child process, if any. */ void killChild() override; + /* Kill any processes running under the build user UID or in the + cgroup of the build. */ + void killSandbox(); + /* Create alternative path calculated from but distinct from the input, so we can avoid overwriting outputs (or other store paths) that already exist. */ diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index f9892bb91..4fad3bfd2 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,7 +1,6 @@ #include "lock.hh" #include "globals.hh" #include "pathlocks.hh" -#include "cgroup.hh" #include #include @@ -15,11 +14,6 @@ struct SimpleUserLock : UserLock gid_t gid; std::vector supplementaryGIDs; - void kill() override - { - killUser(uid); - } - uid_t getUID() override { assert(uid); return uid; } uid_t getUIDCount() override { return 1; } gid_t getGID() override { assert(gid); return gid; } @@ -116,32 +110,6 @@ struct AutoUserLock : UserLock AutoCloseFD fdUserLock; uid_t firstUid = 0; uid_t nrIds = 1; - #if __linux__ - std::optional cgroup; - #endif - - ~AutoUserLock() - { - #if __linux__ - // Get rid of our cgroup, ignoring errors. - if (cgroup) rmdir(cgroup->c_str()); - #endif - } - - void kill() override - { - #if __linux__ - if (cgroup) { - destroyCgroup(*cgroup); - if (mkdir(cgroup->c_str(), 0755) == -1) - throw SysError("creating cgroup '%s'", *cgroup); - } else - #endif - { - assert(firstUid); - killUser(firstUid); - } - } uid_t getUID() override { assert(firstUid); return firstUid; } @@ -183,55 +151,16 @@ struct AutoUserLock : UserLock throw SysError("opening user lock '%s'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { - auto s = drainFD(fd.get()); - - #if __linux__ - if (s != "") { - /* Kill the old cgroup, to ensure there are no - processes left over from an interrupted build. */ - destroyCgroup(s); - } - #endif - - if (ftruncate(fd.get(), 0) == -1) - throw Error("truncating user lock"); - auto lock = std::make_unique(); lock->fdUserLock = std::move(fd); lock->firstUid = settings.startId + i * maxIdsPerBuild; lock->nrIds = nrIds; - - #if __linux__ - if (nrIds > 1) { - auto ourCgroups = getCgroups("/proc/self/cgroup"); - auto ourCgroup = ourCgroups[""]; - if (ourCgroup == "") - throw Error("cannot determine cgroup name from /proc/self/cgroup"); - - auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup); - - if (!pathExists(ourCgroupPath)) - throw Error("expected cgroup directory '%s'", ourCgroupPath); - - lock->cgroup = fmt("%s/nix-build-%d", ourCgroupPath, lock->firstUid); - - /* Record the cgroup in the lock file. This ensures that - if we subsequently get executed under a different parent - cgroup, we kill the previous cgroup first. */ - writeFull(lock->fdUserLock.get(), *lock->cgroup); - } - #endif - return lock; } } return nullptr; } - - #if __linux__ - std::optional getCgroup() override { return cgroup; } - #endif }; std::unique_ptr acquireUserLock(uid_t nrIds) diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index b5536408c..e7ceefab8 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -27,13 +27,6 @@ struct UserLock virtual gid_t getGID() = 0; virtual std::vector getSupplementaryGIDs() = 0; - - /* Kill any processes currently executing as this user. */ - virtual void kill() = 0; - - #if __linux__ - virtual std::optional getCgroup() { return {}; }; - #endif }; /* Acquire a user lock for a UID range of size `nrIds`. Note that this -- cgit v1.2.3 From fa68eb367e79297bb1c0451cd92ad18a06edce96 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Nov 2022 13:40:59 +0100 Subject: Get CPU stats from the cgroup --- src/libstore/build-result.hh | 5 +++- src/libstore/build/derivation-goal.cc | 8 ++++++ src/libstore/build/local-derivation-goal.cc | 14 +++++++---- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/cgroup.cc | 39 ++++++++++++++++++++++++++--- src/libstore/cgroup.hh | 14 ++++++++++- 6 files changed, 71 insertions(+), 11 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 24fb1f763..a5749cf33 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -5,7 +5,7 @@ #include #include - +#include namespace nix { @@ -78,6 +78,9 @@ struct BuildResult was repeated). */ time_t startTime = 0, stopTime = 0; + /* User and system CPU time the build took. */ + std::optional cpuUser, cpuSystem; + bool success() { return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 41d2e2a1c..9bc3dc742 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -869,6 +869,14 @@ void DerivationGoal::buildDone() cleanupPostChildKill(); + if (buildResult.cpuUser && buildResult.cpuSystem) { + debug("builder for '%s' terminated with status %d, user CPU %.3fs, system CPU %.3fs", + worker.store.printStorePath(drvPath), + status, + ((double) buildResult.cpuUser->count()) / 1000000, + ((double) buildResult.cpuSystem->count()) / 1000000); + } + bool diskFull = false; try { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2d1e093ca..f273ebe8a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -137,7 +137,7 @@ void LocalDerivationGoal::killChild() also send a conventional kill to the child. */ ::kill(-pid, SIGKILL); /* ignore the result */ - killSandbox(); + killSandbox(true); pid.wait(); } @@ -146,10 +146,14 @@ void LocalDerivationGoal::killChild() } -void LocalDerivationGoal::killSandbox() +void LocalDerivationGoal::killSandbox(bool getStats) { if (cgroup) { - destroyCgroup(*cgroup); + auto stats = destroyCgroup(*cgroup); + if (getStats) { + buildResult.cpuUser = stats.cpuUser; + buildResult.cpuSystem = stats.cpuSystem; + } } else if (buildUser) { @@ -270,7 +274,7 @@ void LocalDerivationGoal::cleanupPostChildKill() malicious user from leaving behind a process that keeps files open and modifies them after they have been chown'ed to root. */ - killSandbox(); + killSandbox(true); /* Terminate the recursive Nix daemon. */ stopDaemon(); @@ -410,7 +414,7 @@ void LocalDerivationGoal::startBuilder() /* Make sure that no other processes are executing under the sandbox uids. This must be done before any chownToBuilder() calls. */ - killSandbox(); + killSandbox(false); /* Right platform? */ if (!parsedDrv->canBuildLocally(worker.store)) diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 1ec6b3649..34c4e9187 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -202,7 +202,7 @@ struct LocalDerivationGoal : public DerivationGoal /* Kill any processes running under the build user UID or in the cgroup of the build. */ - void killSandbox(); + void killSandbox(bool getStats); /* Create alternative path calculated from but distinct from the input, so we can avoid overwriting outputs (or other store paths) diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 56e980be3..2a485f0f9 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -31,13 +31,16 @@ std::map getCgroups(const Path & cgroupFile) return cgroups; } -void destroyCgroup(const Path & cgroup) +static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) { - if (!pathExists(cgroup)) return; + if (!pathExists(cgroup)) return {}; + + if (!pathExists(cgroup + "/cgroup.procs")) + throw Error("'%s' is not a cgroup", cgroup); for (auto & entry : readDirectory(cgroup)) { if (entry.type != DT_DIR) continue; - destroyCgroup(cgroup + "/" + entry.name); + destroyCgroup(cgroup + "/" + entry.name, false); } int round = 1; @@ -79,8 +82,38 @@ void destroyCgroup(const Path & cgroup) round++; } + CgroupStats stats; + + if (returnStats) { + auto cpustatPath = cgroup + "/cpu.stat"; + + if (pathExists(cpustatPath)) { + for (auto & line : tokenizeString>(readFile(cpustatPath), "\n")) { + std::string_view userPrefix = "user_usec "; + if (hasPrefix(line, userPrefix)) { + auto n = string2Int(line.substr(userPrefix.size())); + if (n) stats.cpuUser = std::chrono::microseconds(*n); + } + + std::string_view systemPrefix = "system_usec "; + if (hasPrefix(line, systemPrefix)) { + auto n = string2Int(line.substr(systemPrefix.size())); + if (n) stats.cpuSystem = std::chrono::microseconds(*n); + } + } + } + + } + if (rmdir(cgroup.c_str()) == -1) throw SysError("deleting cgroup '%s'", cgroup); + + return stats; +} + +CgroupStats destroyCgroup(const Path & cgroup) +{ + return destroyCgroup(cgroup, true); } } diff --git a/src/libstore/cgroup.hh b/src/libstore/cgroup.hh index dc6758957..3ead4735f 100644 --- a/src/libstore/cgroup.hh +++ b/src/libstore/cgroup.hh @@ -2,13 +2,25 @@ #if __linux__ +#include +#include + #include "types.hh" namespace nix { std::map getCgroups(const Path & cgroupFile); -void destroyCgroup(const Path & cgroup); +struct CgroupStats +{ + std::optional cpuUser, cpuSystem; +}; + +/* Destroy the cgroup denoted by 'path'. The postcondition is that + 'path' does not exist, and thus any processes in the cgroup have + been killed. Also return statistics from the cgroup just before + destruction. */ +CgroupStats destroyCgroup(const Path & cgroup); } -- cgit v1.2.3 From e6b71f84a0a766429fdceaf188ea0167e36a20d9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 18 Nov 2022 16:59:36 +0100 Subject: Use cgroup.kill to quickly kill cgroups --- src/libstore/cgroup.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc index 2a485f0f9..f693d77be 100644 --- a/src/libstore/cgroup.cc +++ b/src/libstore/cgroup.cc @@ -35,9 +35,19 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) { if (!pathExists(cgroup)) return {}; - if (!pathExists(cgroup + "/cgroup.procs")) + auto procsFile = cgroup + "/cgroup.procs"; + + if (!pathExists(procsFile)) throw Error("'%s' is not a cgroup", cgroup); + /* Use the fast way to kill every process in a cgroup, if + available. */ + auto killFile = cgroup + "/cgroup.kill"; + if (pathExists(killFile)) + writeFile(killFile, "1"); + + /* Otherwise, manually kill every process in the subcgroups and + this cgroup. */ for (auto & entry : readDirectory(cgroup)) { if (entry.type != DT_DIR) continue; destroyCgroup(cgroup + "/" + entry.name, false); @@ -48,7 +58,7 @@ static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) std::unordered_set pidsShown; while (true) { - auto pids = tokenizeString>(readFile(cgroup + "/cgroup.procs")); + auto pids = tokenizeString>(readFile(procsFile)); if (pids.empty()) break; -- cgit v1.2.3 From f538ee434285304cb61cf10bf13127f13bfced1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 09:38:08 +0100 Subject: Rename derivedPathsWithHintsToJSON -> builtPathsToJSON --- src/libstore/derived-path.cc | 2 +- src/libstore/derived-path.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 44587ae78..7fe9b9648 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -64,7 +64,7 @@ nlohmann::json stuffToJSON(const std::vector & ts, ref store) { return res; } -nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store) +nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store) { return stuffToJSON(buildables, store); } nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) { return stuffToJSON(paths, store); } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 24a0ae773..b2d0956b8 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -125,7 +125,7 @@ struct BuiltPath : _BuiltPathRaw { typedef std::vector DerivedPaths; typedef std::vector BuiltPaths; -nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store); +nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store); nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref store); } -- cgit v1.2.3 From 300753d594fd7cd818d08f9c7a18a9ebc305bd95 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 10:49:01 +0100 Subject: nix build --json: Include build statistics Example: # nix build -L --extra-experimental-features cgroups --impure --expr 'with import {}; runCommand "foo" {} "dd if=/dev/urandom bs=1M count=1024 | md5sum; mkdir $out"' --json [ { "cpuSystem": 1.911431, "cpuUser": 1.214249, "drvPath": "/nix/store/xzdqz67xba18hljhycp0hwfigzrs2z69-foo.drv", "outputs": { "out": "/nix/store/rh9mc9l2gkpq8kn2sgzndr6ll7ffjh6l-foo" }, "startTime": 1669024076, "stopTime": 1669024079 } ] --- src/libstore/derived-path.cc | 23 ++++------------------- src/libstore/derived-path.hh | 3 --- 2 files changed, 4 insertions(+), 22 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 7fe9b9648..88b59f615 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -53,28 +53,13 @@ StorePathSet BuiltPath::outPaths() const ); } -template -nlohmann::json stuffToJSON(const std::vector & ts, ref store) { - auto res = nlohmann::json::array(); - for (const T & t : ts) { - std::visit([&res, store](const auto & t) { - res.push_back(t.toJSON(store)); - }, t.raw()); - } - return res; -} - -nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store) -{ return stuffToJSON(buildables, store); } -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) -{ return stuffToJSON(paths, store); } - - -std::string DerivedPath::Opaque::to_string(const Store & store) const { +std::string DerivedPath::Opaque::to_string(const Store & store) const +{ return store.printStorePath(path); } -std::string DerivedPath::Built::to_string(const Store & store) const { +std::string DerivedPath::Built::to_string(const Store & store) const +{ return store.printStorePath(drvPath) + "!" + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index b2d0956b8..878696136 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -125,7 +125,4 @@ struct BuiltPath : _BuiltPathRaw { typedef std::vector DerivedPaths; typedef std::vector BuiltPaths; -nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store); -nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref store); - } -- cgit v1.2.3 From ec45f4b82eaef8da04f4b828b2b06a77aa3f986f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 11:12:45 +0100 Subject: Fix indentation --- src/libstore/local-store.cc | 50 ++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 04223d860..b67668e52 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -897,48 +897,48 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, std::shared_ptr LocalStore::queryPathInfoInternal(State & state, const StorePath & path) { - /* Get the path info. */ + /* Get the path info. */ auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path))); - if (!useQueryPathInfo.next()) - return std::shared_ptr(); + if (!useQueryPathInfo.next()) + return std::shared_ptr(); - auto id = useQueryPathInfo.getInt(0); + auto id = useQueryPathInfo.getInt(0); - auto narHash = Hash::dummy; - try { - narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); - } catch (BadHash & e) { - throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); - } + auto narHash = Hash::dummy; + try { + narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); + } catch (BadHash & e) { + throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); + } - auto info = std::make_shared(path, narHash); + auto info = std::make_shared(path, narHash); - info->id = id; + info->id = id; - info->registrationTime = useQueryPathInfo.getInt(2); + info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); - if (s) info->deriver = parseStorePath(s); + if (s) info->deriver = parseStorePath(s); - /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); - info->ultimate = useQueryPathInfo.getInt(5) == 1; + info->ultimate = useQueryPathInfo.getInt(5) == 1; s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6); - if (s) info->sigs = tokenizeString(s, " "); + if (s) info->sigs = tokenizeString(s, " "); s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + if (s) info->ca = parseContentAddressOpt(s); - /* Get the references. */ + /* Get the references. */ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + while (useQueryReferences.next()) + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); - return info; + return info; } @@ -1041,9 +1041,9 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) auto path = path_; auto outputs = retrySQLite>>([&]() { auto state(_state.lock()); - std::map> outputs; + std::map> outputs; uint64_t drvId; - drvId = queryValidPathId(*state, path); + drvId = queryValidPathId(*state, path); auto use(state->stmts->QueryDerivationOutputs.use()(drvId)); while (use.next()) outputs.insert_or_assign( -- cgit v1.2.3 From 82d5cf2a76ec009fd94a925c22a5e099a0b7321b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 11:45:41 +0100 Subject: Fix macOS build --- src/libstore/build/local-derivation-goal.cc | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index f273ebe8a..34f8ab5f1 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -149,11 +149,15 @@ void LocalDerivationGoal::killChild() void LocalDerivationGoal::killSandbox(bool getStats) { if (cgroup) { + #if __linux__ auto stats = destroyCgroup(*cgroup); if (getStats) { buildResult.cpuUser = stats.cpuUser; buildResult.cpuSystem = stats.cpuSystem; } + #else + abort(); + #endif } else if (buildUser) { -- cgit v1.2.3 From e7a5b76844a649645e51a60dd18fd383d14d8755 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 09:38:08 +0100 Subject: Rename derivedPathsWithHintsToJSON -> builtPathsToJSON --- src/libstore/derived-path.cc | 2 +- src/libstore/derived-path.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 44587ae78..7fe9b9648 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -64,7 +64,7 @@ nlohmann::json stuffToJSON(const std::vector & ts, ref store) { return res; } -nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store) +nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store) { return stuffToJSON(buildables, store); } nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) { return stuffToJSON(paths, store); } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 24a0ae773..b2d0956b8 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -125,7 +125,7 @@ struct BuiltPath : _BuiltPathRaw { typedef std::vector DerivedPaths; typedef std::vector BuiltPaths; -nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref store); +nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store); nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref store); } -- cgit v1.2.3 From f0baa5c1283359a413ca3a254527587c86b2f097 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 10:49:01 +0100 Subject: nix build --json: Include build statistics Example: # nix build -L --extra-experimental-features cgroups --impure --expr 'with import {}; runCommand "foo" {} "dd if=/dev/urandom bs=1M count=1024 | md5sum; mkdir $out"' --json [ { "cpuSystem": 1.911431, "cpuUser": 1.214249, "drvPath": "/nix/store/xzdqz67xba18hljhycp0hwfigzrs2z69-foo.drv", "outputs": { "out": "/nix/store/rh9mc9l2gkpq8kn2sgzndr6ll7ffjh6l-foo" }, "startTime": 1669024076, "stopTime": 1669024079 } ] --- src/libstore/derived-path.cc | 23 ++++------------------- src/libstore/derived-path.hh | 3 --- 2 files changed, 4 insertions(+), 22 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 7fe9b9648..88b59f615 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -53,28 +53,13 @@ StorePathSet BuiltPath::outPaths() const ); } -template -nlohmann::json stuffToJSON(const std::vector & ts, ref store) { - auto res = nlohmann::json::array(); - for (const T & t : ts) { - std::visit([&res, store](const auto & t) { - res.push_back(t.toJSON(store)); - }, t.raw()); - } - return res; -} - -nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store) -{ return stuffToJSON(buildables, store); } -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) -{ return stuffToJSON(paths, store); } - - -std::string DerivedPath::Opaque::to_string(const Store & store) const { +std::string DerivedPath::Opaque::to_string(const Store & store) const +{ return store.printStorePath(path); } -std::string DerivedPath::Built::to_string(const Store & store) const { +std::string DerivedPath::Built::to_string(const Store & store) const +{ return store.printStorePath(drvPath) + "!" + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs)); diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index b2d0956b8..878696136 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -125,7 +125,4 @@ struct BuiltPath : _BuiltPathRaw { typedef std::vector DerivedPaths; typedef std::vector BuiltPaths; -nlohmann::json builtPathsToJSON(const BuiltPaths & buildables, ref store); -nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref store); - } -- cgit v1.2.3 From 9d17ce07e872e88057480744414e0d1ef4fd5fa8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 12:55:49 +0100 Subject: AutoUserLock: If sandboxing is disabled, use the build users group We have to use a gid that has write access to the Nix store. --- src/libstore/build/local-derivation-goal.cc | 60 ++++++++++++++--------------- src/libstore/lock.cc | 22 ++++++----- src/libstore/lock.hh | 2 +- 3 files changed, 44 insertions(+), 40 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 34f8ab5f1..b7084384a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -177,9 +177,38 @@ void LocalDerivationGoal::tryLocalBuild() { return; } + /* Are we doing a chroot build? */ + { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); + if (settings.sandboxMode == smEnabled) { + if (noChroot) + throw Error("derivation '%s' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error("derivation '%s' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); +#endif + useChroot = true; + } + else if (settings.sandboxMode == smDisabled) + useChroot = false; + else if (settings.sandboxMode == smRelaxed) + useChroot = derivationType.isSandboxed() && !noChroot; + } + + auto & localStore = getLocalStore(); + if (localStore.storeDir != localStore.realStoreDir.get()) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } + if (useBuildUsers()) { if (!buildUser) - buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1); + buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot); if (!buildUser) { if (!actLock) @@ -433,35 +462,6 @@ void LocalDerivationGoal::startBuilder() additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif - /* Are we doing a chroot build? */ - { - auto noChroot = parsedDrv->getBoolAttr("__noChroot"); - if (settings.sandboxMode == smEnabled) { - if (noChroot) - throw Error("derivation '%s' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); -#if __APPLE__ - if (additionalSandboxProfile != "") - throw Error("derivation '%s' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); -#endif - useChroot = true; - } - else if (settings.sandboxMode == smDisabled) - useChroot = false; - else if (settings.sandboxMode == smRelaxed) - useChroot = derivationType.isSandboxed() && !noChroot; - } - - auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir.get()) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif - } - /* Create a temporary directory where the build will take place. */ tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 4fad3bfd2..3b93979a8 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -109,22 +109,18 @@ struct AutoUserLock : UserLock { AutoCloseFD fdUserLock; uid_t firstUid = 0; + gid_t firstGid = 0; uid_t nrIds = 1; uid_t getUID() override { assert(firstUid); return firstUid; } gid_t getUIDCount() override { return nrIds; } - gid_t getGID() override - { - // We use the same GID ranges as for the UIDs. - assert(firstUid); - return firstUid; - } + gid_t getGID() override { assert(firstGid); return firstGid; } std::vector getSupplementaryGIDs() override { return {}; } - static std::unique_ptr acquire(uid_t nrIds) + static std::unique_ptr acquire(uid_t nrIds, bool useChroot) { settings.requireExperimentalFeature(Xp::AutoAllocateUids); assert(settings.startId > 0); @@ -154,6 +150,14 @@ struct AutoUserLock : UserLock auto lock = std::make_unique(); lock->fdUserLock = std::move(fd); lock->firstUid = settings.startId + i * maxIdsPerBuild; + if (useChroot) + lock->firstGid = lock->firstUid; + else { + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (!gr) + throw Error("the group '%s' specified in 'build-users-group' does not exist", settings.buildUsersGroup); + lock->firstGid = gr->gr_gid; + } lock->nrIds = nrIds; return lock; } @@ -163,10 +167,10 @@ struct AutoUserLock : UserLock } }; -std::unique_ptr acquireUserLock(uid_t nrIds) +std::unique_ptr acquireUserLock(uid_t nrIds, bool useChroot) { if (settings.autoAllocateUids) - return AutoUserLock::acquire(nrIds); + return AutoUserLock::acquire(nrIds, useChroot); else return SimpleUserLock::acquire(); } diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index e7ceefab8..49ad86de7 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -31,7 +31,7 @@ struct UserLock /* Acquire a user lock for a UID range of size `nrIds`. Note that this may return nullptr if no user is available. */ -std::unique_ptr acquireUserLock(uid_t nrIds); +std::unique_ptr acquireUserLock(uid_t nrIds, bool useChroot); bool useBuildUsers(); -- cgit v1.2.3 From c776dfbb35e961ac3f011ab8665dfc85ab067ef8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Nov 2022 18:46:55 +0100 Subject: Use hex for startId Co-authored-by: Linus Heckemann --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 88fe72202..653d108aa 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -288,7 +288,7 @@ public: Setting startId{this, #if __linux__ - 872415232, + 0x34000000, #else 56930, #endif -- cgit v1.2.3 From b37c2d84b67635fc928ed174166f04d6f4d30c6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Nov 2022 09:02:17 +0100 Subject: Always call setgroups() We shouldn't skip this if the supplementary group list is empty, because then the sandbox won't drop the supplementary groups of the parent (like "root"). --- src/libstore/build/local-derivation-goal.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b7084384a..232440f74 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1988,9 +1988,8 @@ void LocalDerivationGoal::runChild() if (setUser && buildUser) { /* Preserve supplementary groups of the build user, to allow admins to specify groups such as "kvm". */ - if (!buildUser->getSupplementaryGIDs().empty() && - setgroups(buildUser->getSupplementaryGIDs().size(), - buildUser->getSupplementaryGIDs().data()) == -1) + auto gids = buildUser->getSupplementaryGIDs(); + if (setgroups(gids.size(), gids.data()) == -1) throw SysError("cannot set supplementary groups of build user"); if (setgid(buildUser->getGID()) == -1 || -- cgit v1.2.3 From 3d23b9d0324ff415af9e5f35568aca98c04a90cc Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Nov 2022 09:03:30 +0100 Subject: SimpleUserLock::getSupplementaryGIDs(): Filter out main gid This avoids having the user's gid in the supplementary group list as well. --- src/libstore/lock.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 3b93979a8..7459d837d 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -71,21 +71,22 @@ struct SimpleUserLock : UserLock user. This is usually either empty or contains a group such as "kvm". */ int ngroups = 32; // arbitrary initial guess - lock->supplementaryGIDs.resize(ngroups); + std::vector gids; + gids.resize(ngroups); int err = getgrouplist( pw->pw_name, pw->pw_gid, - lock->supplementaryGIDs.data(), + gids.data(), &ngroups); /* Our initial size of 32 wasn't sufficient, the correct size has been stored in ngroups, so we try again. */ if (err == -1) { - lock->supplementaryGIDs.resize(ngroups); + gids.resize(ngroups); err = getgrouplist( pw->pw_name, pw->pw_gid, - lock->supplementaryGIDs.data(), + gids.data(), &ngroups); } @@ -94,7 +95,9 @@ struct SimpleUserLock : UserLock throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); // Finally, trim back the GID list to its real size. - lock->supplementaryGIDs.resize(ngroups); + for (auto i = 0; i < ngroups; i++) + if (gids[i] != lock->gid) + lock->supplementaryGIDs.push_back(gids[i]); #endif return lock; -- cgit v1.2.3 From 5b798f6caeab64854394e1250e30aab91241cf26 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 27 Nov 2022 12:57:18 +0100 Subject: Fix random client failures during GC server shutdown We need to close the GC server socket before shutting down the active GC client connections, otherwise a client may (re)connect and get ECONNRESET. But also handle ECONNRESET for resilience. Fixes random failures like GC socket disconnected connecting to '/tmp/nix-shell.y07M0H/nix-test/default/var/nix/gc-socket/socket' sending GC root '/tmp/nix-shell.y07M0H/nix-test/default/store/kb5yzija0f1x5xkqkgclrdzldxj6nnc6-non-blocking' reading GC root from client: error: unexpected EOF reading a line 1 store paths deleted, 0.00 MiB freed error: reading from file: Connection reset by peer in gc-non-blocking.sh. --- src/libstore/gc.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 9ef8972f3..5d91829f1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -147,7 +147,7 @@ void LocalStore::addTempRoot(const StorePath & path) } catch (SysError & e) { /* The garbage collector may have exited, so we need to restart. */ - if (e.errNo == EPIPE) { + if (e.errNo == EPIPE || e.errNo == ECONNRESET) { debug("GC socket disconnected"); state->fdRootsSocket.close(); goto restart; @@ -506,6 +506,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) Finally cleanup([&]() { debug("GC roots server shutting down"); + fdServer.close(); while (true) { auto item = remove_begin(*connections.lock()); if (!item) break; -- cgit v1.2.3 From ff12d1c1a1bb0dcea5a9ac6b8a5036d7e5dc11ca Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 28 Nov 2022 20:49:17 +0100 Subject: Check that auto-allocated UIDs don't clash with existing accounts --- src/libstore/lock.cc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 7459d837d..2858137d6 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -127,13 +127,10 @@ struct AutoUserLock : UserLock { settings.requireExperimentalFeature(Xp::AutoAllocateUids); assert(settings.startId > 0); - assert(settings.startId % maxIdsPerBuild == 0); assert(settings.uidCount % maxIdsPerBuild == 0); assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); assert(nrIds <= maxIdsPerBuild); - // FIXME: check whether the id range overlaps any known users - createDirs(settings.nixStateDir + "/userpool2"); size_t nrSlots = settings.uidCount / maxIdsPerBuild; @@ -150,11 +147,18 @@ struct AutoUserLock : UserLock throw SysError("opening user lock '%s'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { + + auto firstUid = settings.startId + i * maxIdsPerBuild; + + auto pw = getpwuid(firstUid); + if (pw) + throw Error("auto-allocated UID %d clashes with existing user account '%s'", firstUid, pw->pw_name); + auto lock = std::make_unique(); lock->fdUserLock = std::move(fd); - lock->firstUid = settings.startId + i * maxIdsPerBuild; + lock->firstUid = firstUid; if (useChroot) - lock->firstGid = lock->firstUid; + lock->firstGid = firstUid; else { struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) -- cgit v1.2.3 From 67bcb99700a0da1395fa063d7c6586740b304598 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 28 Nov 2022 21:54:02 +0100 Subject: Add a setting for enabling cgroups --- src/libstore/build/local-derivation-goal.cc | 7 ++++++- src/libstore/globals.hh | 23 +++++++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d44694890..69a7df411 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -401,9 +401,14 @@ static void linkOrCopy(const Path & from, const Path & to) void LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) - || settings.isExperimentalFeatureEnabled(Xp::Cgroups)) + #if __linux__ + || settings.useCgroups + #endif + ) { #if __linux__ + settings.requireExperimentalFeature(Xp::Cgroups); + auto ourCgroups = getCgroups("/proc/self/cgroup"); auto ourCgroup = ourCgroups[""]; if (ourCgroup == "") diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 653d108aa..b40dcfa77 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -304,6 +304,17 @@ public: "id-count", "The number of UIDs/GIDs to use for dynamic ID allocation."}; + #if __linux__ + Setting useCgroups{ + this, false, "use-cgroups", + R"( + Whether to execute builds inside cgroups. Cgroups are + enabled automatically for derivations that require the + `uid-range` system feature. + )" + }; + #endif + Setting impersonateLinux26{this, false, "impersonate-linux-26", "Whether to impersonate a Linux 2.6 machine on newer kernels.", {"build-impersonate-linux-26"}}; @@ -592,10 +603,10 @@ public: cache) must have a signature by a trusted key. A trusted key is one listed in `trusted-public-keys`, or a public key counterpart to a private key stored in a file listed in `secret-key-files`. - + Set to `false` to disable signature checking and trust all non-content-addressed paths unconditionally. - + (Content-addressed paths are inherently trustworthy and thus unaffected by this configuration option.) )"}; @@ -681,7 +692,7 @@ public: is `root`. > **Warning** - > + > > Adding a user to `trusted-users` is essentially equivalent to > giving that user root access to the system. For example, the user > can set `sandbox-paths` and thereby obtain read access to @@ -771,13 +782,13 @@ public: The program executes with no arguments. The program's environment contains the following environment variables: - - `DRV_PATH` + - `DRV_PATH` The derivation for the built paths. Example: `/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv` - - `OUT_PATHS` + - `OUT_PATHS` Output paths of the built derivation, separated by a space character. @@ -815,7 +826,7 @@ public: documentation](https://ec.haxx.se/usingcurl-netrc.html). > **Note** - > + > > This must be an absolute path, and `~` is not resolved. For > example, `~/.netrc` won't resolve to your home directory's > `.netrc`. -- cgit v1.2.3 From 4f762e2b023fd451fdbab0de8d6394dd7201640d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 29 Nov 2022 13:10:53 +0100 Subject: Restore ownership of / for non-uid-range builds --- src/libstore/build/local-derivation-goal.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 69a7df411..359966288 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -646,8 +646,7 @@ void LocalDerivationGoal::startBuilder() if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); - // FIXME: only make root writable for user namespace builds. - if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUIDCount() != 1 ? buildUser->getUID() : 0, buildUser->getGID()) == -1) throw SysError("cannot change ownership of '%1%'", chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need -- cgit v1.2.3 From 0ea62670eda39ebeaff5335fd8225707ec27e7a3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 1 Dec 2022 04:40:02 +0100 Subject: move documentation on `auto-allocate-uids` to options docs this is where it belongs and can be found together with the other options. --- src/libstore/globals.hh | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b40dcfa77..b61a34461 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -284,7 +284,21 @@ public: )"}; Setting autoAllocateUids{this, false, "auto-allocate-uids", - "Whether to allocate UIDs for builders automatically."}; + R"( + Whether to allocate UIDs for builders automatically. + + These UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. + + > **Warning** + > This is an experimental feature. + + To enable it, add the following to [`nix.conf`](#): + + ``` + extra-experimental-features = auto-allocate-uids + auto-allocate-uids = true + ``` + )"}; Setting startId{this, #if __linux__ @@ -308,11 +322,21 @@ public: Setting useCgroups{ this, false, "use-cgroups", R"( - Whether to execute builds inside cgroups. Cgroups are - enabled automatically for derivations that require the - `uid-range` system feature. - )" - }; + Whether to execute builds inside cgroups. + Only on Linux with systemd. + + cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. + + > **Warning** + > This is an experimental feature. + + To enable it, add the following to [`nix.conf`](#): + + ``` + extra-experimental-features = cgroups + use-cgroups = true + ``` + )"}; #endif Setting impersonateLinux26{this, false, "impersonate-linux-26", -- cgit v1.2.3 From 1211e59a038379026496bbee4b203bbd66833b01 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2022 12:38:03 +0100 Subject: Move cgroup.{cc,hh} to libutil --- src/libstore/cgroup.cc | 131 ------------------------------------------------- src/libstore/cgroup.hh | 27 ---------- 2 files changed, 158 deletions(-) delete mode 100644 src/libstore/cgroup.cc delete mode 100644 src/libstore/cgroup.hh (limited to 'src/libstore') diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc deleted file mode 100644 index f693d77be..000000000 --- a/src/libstore/cgroup.cc +++ /dev/null @@ -1,131 +0,0 @@ -#if __linux__ - -#include "cgroup.hh" -#include "util.hh" - -#include -#include -#include -#include -#include - -#include - -namespace nix { - -// FIXME: obsolete, check for cgroup2 -std::map getCgroups(const Path & cgroupFile) -{ - std::map cgroups; - - for (auto & line : tokenizeString>(readFile(cgroupFile), "\n")) { - static std::regex regex("([0-9]+):([^:]*):(.*)"); - std::smatch match; - if (!std::regex_match(line, match, regex)) - throw Error("invalid line '%s' in '%s'", line, cgroupFile); - - std::string name = hasPrefix(std::string(match[2]), "name=") ? std::string(match[2], 5) : match[2]; - cgroups.insert_or_assign(name, match[3]); - } - - return cgroups; -} - -static CgroupStats destroyCgroup(const Path & cgroup, bool returnStats) -{ - if (!pathExists(cgroup)) return {}; - - auto procsFile = cgroup + "/cgroup.procs"; - - if (!pathExists(procsFile)) - throw Error("'%s' is not a cgroup", cgroup); - - /* Use the fast way to kill every process in a cgroup, if - available. */ - auto killFile = cgroup + "/cgroup.kill"; - if (pathExists(killFile)) - writeFile(killFile, "1"); - - /* Otherwise, manually kill every process in the subcgroups and - this cgroup. */ - for (auto & entry : readDirectory(cgroup)) { - if (entry.type != DT_DIR) continue; - destroyCgroup(cgroup + "/" + entry.name, false); - } - - int round = 1; - - std::unordered_set pidsShown; - - while (true) { - auto pids = tokenizeString>(readFile(procsFile)); - - if (pids.empty()) break; - - if (round > 20) - throw Error("cannot kill cgroup '%s'", cgroup); - - for (auto & pid_s : pids) { - pid_t pid; - if (auto o = string2Int(pid_s)) - pid = *o; - else - throw Error("invalid pid '%s'", pid); - if (pidsShown.insert(pid).second) { - try { - auto cmdline = readFile(fmt("/proc/%d/cmdline", pid)); - using namespace std::string_literals; - warn("killing stray builder process %d (%s)...", - pid, trim(replaceStrings(cmdline, "\0"s, " "))); - } catch (SysError &) { - } - } - // FIXME: pid wraparound - if (kill(pid, SIGKILL) == -1 && errno != ESRCH) - throw SysError("killing member %d of cgroup '%s'", pid, cgroup); - } - - auto sleep = std::chrono::milliseconds((int) std::pow(2.0, std::min(round, 10))); - if (sleep.count() > 100) - printError("waiting for %d ms for cgroup '%s' to become empty", sleep.count(), cgroup); - std::this_thread::sleep_for(sleep); - round++; - } - - CgroupStats stats; - - if (returnStats) { - auto cpustatPath = cgroup + "/cpu.stat"; - - if (pathExists(cpustatPath)) { - for (auto & line : tokenizeString>(readFile(cpustatPath), "\n")) { - std::string_view userPrefix = "user_usec "; - if (hasPrefix(line, userPrefix)) { - auto n = string2Int(line.substr(userPrefix.size())); - if (n) stats.cpuUser = std::chrono::microseconds(*n); - } - - std::string_view systemPrefix = "system_usec "; - if (hasPrefix(line, systemPrefix)) { - auto n = string2Int(line.substr(systemPrefix.size())); - if (n) stats.cpuSystem = std::chrono::microseconds(*n); - } - } - } - - } - - if (rmdir(cgroup.c_str()) == -1) - throw SysError("deleting cgroup '%s'", cgroup); - - return stats; -} - -CgroupStats destroyCgroup(const Path & cgroup) -{ - return destroyCgroup(cgroup, true); -} - -} - -#endif diff --git a/src/libstore/cgroup.hh b/src/libstore/cgroup.hh deleted file mode 100644 index 3ead4735f..000000000 --- a/src/libstore/cgroup.hh +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#if __linux__ - -#include -#include - -#include "types.hh" - -namespace nix { - -std::map getCgroups(const Path & cgroupFile); - -struct CgroupStats -{ - std::optional cpuUser, cpuSystem; -}; - -/* Destroy the cgroup denoted by 'path'. The postcondition is that - 'path' does not exist, and thus any processes in the cgroup have - been killed. Also return statistics from the cgroup just before - destruction. */ -CgroupStats destroyCgroup(const Path & cgroup); - -} - -#endif -- cgit v1.2.3 From 1e6a5d1ff6e8ef5bf340502f74c4d5039cedc67a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 2 Dec 2022 12:57:41 +0100 Subject: Clean up cgroup handling in getMaxCPU() Also, don't assume in LocalDerivationGoal that cgroups are mounted on /sys/fs/cgroup. --- src/libstore/build/local-derivation-goal.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index c9b7b24f3..d2798888b 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -409,12 +409,16 @@ void LocalDerivationGoal::startBuilder() #if __linux__ settings.requireExperimentalFeature(Xp::Cgroups); + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) + throw Error("cannot determine the cgroups file system"); + auto ourCgroups = getCgroups("/proc/self/cgroup"); auto ourCgroup = ourCgroups[""]; if (ourCgroup == "") throw Error("cannot determine cgroup name from /proc/self/cgroup"); - auto ourCgroupPath = canonPath("/sys/fs/cgroup/" + ourCgroup); + auto ourCgroupPath = canonPath(*cgroupFS + "/" + ourCgroup); if (!pathExists(ourCgroupPath)) throw Error("expected cgroup directory '%s'", ourCgroupPath); -- cgit v1.2.3 From e4f9f3bf246d66c57b07b45583469b98ba0db367 Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Mon, 5 Dec 2022 11:27:47 -0500 Subject: check the store for input before failing (hopefully fix #6700) --- src/libstore/build/derivation-goal.cc | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 67cfc38af..5aed51bcd 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -501,6 +501,14 @@ void DerivationGoal::inputsRealised() now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal. */ std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs); + if (!attempt) { + /* TODO (impure derivations-induced tech debt) (see below): + The above attempt should have found it, but because we manage + inputDrvOutputs statefully, sometimes it gets out of sync with + the real source of truth (store). So we query the store + directly if there's a problem. */ + attempt = fullDrv.tryResolve(worker.store); + } assert(attempt); Derivation drvResolved { *std::move(attempt) }; -- cgit v1.2.3 From 8c7661da0963a6a578605374cd9b309177563b3d Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Mon, 5 Dec 2022 23:21:58 -0500 Subject: check the store for input before failing (hopefully fix #6383) --- src/libstore/build/derivation-goal.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5aed51bcd..7dd39051b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1016,11 +1016,20 @@ void DerivationGoal::resolvedFinished() throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", worker.store.printStorePath(drvPath), wantedOutput); - auto realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); - if (!realisation) + + const Realisation * realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); + if (!realisation) { + /* The above `get` should work. But sateful tracking of + outputs in resolvedResult, this can get out of sync with the + store, which is our actual source of truth. For now we just + check the store directly if it fails. */ + realisation = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput }).get(); + if (!realisation) { throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); + } + } if (drv->type().isPure()) { auto newRealisation = *realisation; newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput }; -- cgit v1.2.3 From 484578d3f9b15c34c0e3a42cc0d1f87f1c15d7f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Dec 2022 10:30:36 +0100 Subject: Tweak option descriptions --- src/libstore/globals.hh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index b61a34461..ca72ad31e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -285,9 +285,10 @@ public: Setting autoAllocateUids{this, false, "auto-allocate-uids", R"( - Whether to allocate UIDs for builders automatically. + Whether to select UIDs for builds automatically, instead of using the + users in `build-users-group`. - These UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. + UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. > **Warning** > This is an experimental feature. @@ -323,9 +324,10 @@ public: this, false, "use-cgroups", R"( Whether to execute builds inside cgroups. - Only on Linux with systemd. + This is only supported on Linux. - cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. + Cgroups are required and enabled automatically for derivations + that require the `uid-range` system feature. > **Warning** > This is an experimental feature. -- cgit v1.2.3 From 1c8de7d3d03d7a6ba259387b0698874fa879428c Mon Sep 17 00:00:00 2001 From: Taeer Bar-Yam Date: Tue, 6 Dec 2022 11:25:38 -0500 Subject: improve style --- src/libstore/build/derivation-goal.cc | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7dd39051b..1f9bfaae0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1017,30 +1017,33 @@ void DerivationGoal::resolvedFinished() "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,resolve)", worker.store.printStorePath(drvPath), wantedOutput); - const Realisation * realisation = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); - if (!realisation) { + auto realisation = [&]{ + auto take1 = get(resolvedResult.builtOutputs, DrvOutput { *resolvedHash, wantedOutput }); + if (take1) return *take1; + /* The above `get` should work. But sateful tracking of outputs in resolvedResult, this can get out of sync with the store, which is our actual source of truth. For now we just check the store directly if it fails. */ - realisation = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput }).get(); - if (!realisation) { - throw Error( - "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", - worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); - } - } + auto take2 = worker.evalStore.queryRealisation(DrvOutput { *resolvedHash, wantedOutput }); + if (take2) return *take2; + + throw Error( + "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolvedFinished,realisation)", + worker.store.printStorePath(resolvedDrvGoal->drvPath), wantedOutput); + }(); + if (drv->type().isPure()) { - auto newRealisation = *realisation; + auto newRealisation = realisation; newRealisation.id = DrvOutput { initialOutput->outputHash, wantedOutput }; newRealisation.signatures.clear(); if (!drv->type().isFixed()) - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } - outputPaths.insert(realisation->outPath); - builtOutputs.emplace(realisation->id, *realisation); + outputPaths.insert(realisation.outPath); + builtOutputs.emplace(realisation.id, realisation); } runPostBuildHook( -- cgit v1.2.3 From 8e0946e8df968391d1430af8377bdb51204e4666 Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Mon, 26 Sep 2022 20:55:56 +0200 Subject: Remove repeat and enforce-determinism options These only functioned if a very narrow combination of conditions held: - The result path does not yet exist (--check did not result in repeated builds), AND - The result path is not available from any configured substituters, AND - No remote builders that can build the path are available. If any of these do not hold, a derivation would be built 0 or 1 times regardless of the repeat option. Thus, remove it to avoid confusion. --- src/libstore/build/derivation-goal.cc | 17 +------- src/libstore/build/derivation-goal.hh | 5 --- src/libstore/build/local-derivation-goal.cc | 62 +---------------------------- src/libstore/daemon.cc | 1 - src/libstore/globals.hh | 22 ---------- src/libstore/legacy-ssh-store.cc | 4 +- 6 files changed, 6 insertions(+), 105 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5aed51bcd..98f9d681a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -571,10 +571,6 @@ void DerivationGoal::inputsRealised() /* What type of derivation are we building? */ derivationType = drv->type(); - /* Don't repeat fixed-output derivations since they're already - verified by their output hash.*/ - nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1; - /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ @@ -589,12 +585,11 @@ void DerivationGoal::started() auto msg = fmt( buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + "building '%s'", worker.store.printStorePath(drvPath)); fmt("building '%s'", worker.store.printStorePath(drvPath)); if (hook) msg += fmt(" on '%s'", machineName); act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1}); mcRunningBuilds = std::make_unique>(worker.runningBuilds); worker.updateProgress(); } @@ -948,14 +943,6 @@ void DerivationGoal::buildDone() cleanupPostOutputsRegisteredModeNonCheck(); - /* Repeat the build if necessary. */ - if (curRound++ < nrRounds) { - outputLocks.unlock(); - state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return; - } - /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not create new lock files with the same names as the old diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 2d8bfd592..d33e04cbc 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -115,11 +115,6 @@ struct DerivationGoal : public Goal BuildMode buildMode; - /* The current round, if we're building multiple times. */ - size_t curRound = 1; - - size_t nrRounds; - std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; std::unique_ptr act; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d2798888b..6fe3bc49c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2260,7 +2260,6 @@ DrvOutputs LocalDerivationGoal::registerOutputs() InodesSeen inodesSeen; Path checkSuffix = ".check"; - bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; std::exception_ptr delayedException; @@ -2688,10 +2687,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); } - if (curRound == nrRounds) { - localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - worker.markContentsGood(newInfo.path); - } + localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() + worker.markContentsGood(newInfo.path); newInfo.deriver = drvPath; newInfo.ultimate = true; @@ -2720,61 +2717,6 @@ DrvOutputs LocalDerivationGoal::registerOutputs() /* Apply output checks. */ checkOutputs(infos); - /* Compare the result with the previous round, and report which - path is different, if any.*/ - if (curRound > 1 && prevInfos != infos) { - assert(prevInfos.size() == infos.size()); - for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) - if (!(*i == *j)) { - buildResult.isNonDeterministic = true; - Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; - bool prevExists = keepPreviousRound && pathExists(prev); - hintformat hint = prevExists - ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : hintfmt("output '%s' of '%s' differs from previous round", - worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - prev, worker.store.printStorePath(i->second.path), - worker.store.printStorePath(drvPath), tmpDir); - - if (settings.enforceDeterminism) - throw NotDeterministic(hint); - - printError(hint); - - curRound = nrRounds; // we know enough, bail out early - } - } - - /* If this is the first round of several, then move the output out of the way. */ - if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { - for (auto & [_, outputStorePath] : finalOutputs) { - auto path = worker.store.printStorePath(outputStorePath); - Path prev = path + checkSuffix; - deletePath(prev); - Path dst = path + checkSuffix; - renameFile(path, dst); - } - } - - if (curRound < nrRounds) { - prevInfos = std::move(infos); - return {}; - } - - /* Remove the .check directories if we're done. FIXME: keep them - if the result was not determistic? */ - if (curRound == nrRounds) { - for (auto & [_, outputStorePath] : finalOutputs) { - Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix; - deletePath(prev); - } - } - /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 48dd5c247..12596ba49 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -238,7 +238,6 @@ struct ClientSettings } else if (trusted || name == settings.buildTimeout.name - || name == settings.buildRepeat.name || name == settings.maxSilentTime.name || name == settings.pollInterval.name || name == "connect-timeout" diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ca72ad31e..54a5d0fc7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -373,11 +373,6 @@ public: )", {"build-max-log-size"}}; - /* When buildRepeat > 0 and verboseBuild == true, whether to print - repeated builds (i.e. builds other than the first one) to - stderr. Hack to prevent Hydra logs from being polluted. */ - bool printRepeatedBuilds = true; - Setting pollInterval{this, 5, "build-poll-interval", "How often (in seconds) to poll for locks."}; @@ -501,19 +496,6 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting buildRepeat{ - this, 0, "repeat", - R"( - How many times to repeat builds to check whether they are - deterministic. The default value is 0. If the value is non-zero, - every build is repeated the specified number of times. If the - contents of any of the runs differs from the previous ones and - `enforce-determinism` is true, the build is rejected and the - resulting store paths are not registered as “valid” in Nix’s - database. - )", - {"build-repeat"}}; - #if __linux__ Setting sandboxShmSize{ this, "50%", "sandbox-dev-shm-size", @@ -577,10 +559,6 @@ public: configuration file, and cannot be passed at the command line. )"}; - Setting enforceDeterminism{ - this, true, "enforce-determinism", - "Whether to fail if repeated builds produce different output. See `repeat`."}; - Setting trustedPublicKeys{ this, {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index dd34b19c6..4d398b21d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -255,8 +255,8 @@ private: << settings.maxLogSize; if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) conn.to - << settings.buildRepeat - << settings.enforceDeterminism; + << 0 // buildRepeat hasn't worked for ages anyway + << 0; if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) { conn.to << ((int) settings.keepFailed); -- cgit v1.2.3 From 703d863a48f549b2626382eda407ffae779f8725 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Dec 2022 12:58:58 +0100 Subject: Trivial changes from the lazy-trees branch --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/build/derivation-goal.cc | 6 +++--- src/libstore/build/entry-points.cc | 6 +++--- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/derivations.cc | 2 +- src/libstore/derivations.hh | 2 +- src/libstore/filetransfer.cc | 6 +++--- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 8 ++++---- src/libstore/store-api.hh | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 12d0c32fb..149d414d3 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -346,7 +346,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) try { getFile(info->url, *decompressor); } catch (NoSuchBinaryCacheFile & e) { - throw SubstituteGone(e.info()); + throw SubstituteGone(std::move(e.info())); } decompressor->finish(); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5aed51bcd..2949a0a1f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -134,7 +134,7 @@ void DerivationGoal::killChild() void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, {}, ex); + done(BuildResult::TimedOut, {}, std::move(ex)); } @@ -984,7 +984,7 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, {}, e); + done(st, {}, std::move(e)); return; } } @@ -1435,7 +1435,7 @@ void DerivationGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } - amDone(buildResult.success() ? ecSuccess : ecFailed, ex); + amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index bea7363db..e1b80165e 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -30,7 +30,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod if (ex) logError(i->ex->info()); else - ex = i->ex; + ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); @@ -40,7 +40,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod if (failed.size() == 1 && ex) { ex->status = worker.exitStatus(); - throw *ex; + throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); @@ -109,7 +109,7 @@ void Store::ensurePath(const StorePath & path) if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { goal->ex->status = worker.exitStatus(); - throw *goal->ex; + throw std::move(*goal->ex); } else throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index d2798888b..dc6f8eeba 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -230,7 +230,7 @@ void LocalDerivationGoal::tryLocalBuild() { outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, {}, e); + done(BuildResult::InputRejected, {}, std::move(e)); return; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fe99c3c5e..42a53912e 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -448,7 +448,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, // FIXME: remove -bool isDerivation(const std::string & fileName) +bool isDerivation(std::string_view fileName) { return hasSuffix(fileName, drvExtension); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index af198a767..f3cd87fb1 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -224,7 +224,7 @@ StorePath writeDerivation(Store & store, Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); // FIXME: remove -bool isDerivation(const std::string & fileName); +bool isDerivation(std::string_view fileName); /* Calculate the name that will be used for the store path for this output. diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 5746c32a3..2ff411e18 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -142,9 +142,9 @@ struct curlFileTransfer : public FileTransfer } template - void fail(const T & e) + void fail(T && e) { - failEx(std::make_exception_ptr(e)); + failEx(std::make_exception_ptr(std::move(e))); } LambdaSink finalSink; @@ -472,7 +472,7 @@ struct curlFileTransfer : public FileTransfer fileTransfer.enqueueItem(shared_from_this()); } else - fail(exc); + fail(std::move(exc)); } } }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 96a29155c..48cf731a8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -447,7 +447,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, } catch (Error & e) { // Ugly backwards compatibility hack. if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.info()); + throw InvalidPath(std::move(e.info())); throw; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8811ab578..80b60ca1b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -19,21 +19,21 @@ using json = nlohmann::json; namespace nix { -bool Store::isInStore(const Path & path) const +bool Store::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair Store::toStorePath(const Path & path) const +std::pair Store::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); - Path::size_type slash = path.find('/', storeDir.size() + 1); + auto slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) return {parseStorePath(path), ""}; else - return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)}; + return {parseStorePath(path.substr(0, slash)), (Path) path.substr(slash)}; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 151ec10d6..4a88d7216 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -179,7 +179,7 @@ public: /* Return true if ‘path’ is in the Nix store (but not the Nix store itself). */ - bool isInStore(const Path & path) const; + bool isInStore(PathView path) const; /* Return true if ‘path’ is a store path, i.e. a direct child of the Nix store. */ @@ -187,7 +187,7 @@ public: /* Split a path like /nix/store/-/ into /nix/store/- and /. */ - std::pair toStorePath(const Path & path) const; + std::pair toStorePath(PathView path) const; /* Follow symlinks until we end up with a path in the Nix store. */ Path followLinksToStore(std::string_view path) const; -- cgit v1.2.3 From ae5f62a894190e0075eb60ae4537ba81ca2a0a8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Sep 2022 15:37:09 +0200 Subject: Move isUri() and resolveUri() out of filetransfer.cc These are purely related to NIX_PATH / -I command line parsing, so put them in libexpr. --- src/libstore/filetransfer.cc | 18 ------------------ src/libstore/filetransfer.hh | 5 ----- 2 files changed, 23 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 2ff411e18..756bd4423 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -33,14 +33,6 @@ FileTransferSettings fileTransferSettings; static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); -std::string resolveUri(std::string_view uri) -{ - if (uri.compare(0, 8, "channel:") == 0) - return "https://nixos.org/channels/" + std::string(uri.substr(8)) + "/nixexprs.tar.xz"; - else - return std::string(uri); -} - struct curlFileTransfer : public FileTransfer { CURLM * curlm = 0; @@ -873,14 +865,4 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::optional response, const Args & ... args); }; -bool isUri(std::string_view s); - -/* Resolve deprecated 'channel:' URLs. */ -std::string resolveUri(std::string_view uri); - } -- cgit v1.2.3 From dc075dcdd0306adec911ec8d898b723f464f7c0a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 12 Dec 2022 16:26:10 -0500 Subject: Apply suggestions from code review Co-authored-by: Eelco Dolstra --- src/libstore/derived-path.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 7fe797aa1..05c2303db 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -93,7 +93,7 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi DerivedPath DerivedPath::parse(const Store & store, std::string_view s) { - size_t n = s.rfind("!"); + size_t n = s.find("!"); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); -- cgit v1.2.3 From 1f3c0a3c1dde7e0348c085e0fbb60729cb067d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 14 Dec 2022 00:40:30 +0100 Subject: Allow disabling build users by unsetting `build-users-group` Unsetting `build-users-group` (without `auto-allocate-uids` enabled) gives the following error: ``` src/libstore/lock.cc:25: static std::unique_ptr nix::SimpleUserLock::acquire(): Assertion `settings.buildUsersGroup != ""' failed. ``` Fix the logic in `useBuildUsers` and document the default value for `build-users-group`. --- src/libstore/globals.hh | 5 ++++- src/libstore/lock.cc | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 54a5d0fc7..274a15dd7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -281,7 +281,10 @@ public: `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if `NIX_REMOTE` is `daemon`). Obviously, this should not be used in multi-user settings with untrusted users. - )"}; + + Defaults to `nixbld` when running as root, *empty* otherwise. + )", + {}, false}; Setting autoAllocateUids{this, false, "auto-allocate-uids", R"( diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 2858137d6..d02d20b4c 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -185,7 +185,7 @@ std::unique_ptr acquireUserLock(uid_t nrIds, bool useChroot) bool useBuildUsers() { #if __linux__ - static bool b = (settings.buildUsersGroup != "" || settings.startId.get() != 0) && getuid() == 0; + static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && getuid() == 0; return b; #elif __APPLE__ static bool b = settings.buildUsersGroup != "" && getuid() == 0; -- cgit v1.2.3 From 0687e16c4afd131540181cc66136418ac1cb845c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 15 Dec 2022 16:00:46 +0100 Subject: Fix a crash in DerivedPath::Built::toJSON() with impure derivations The use of 'nullptr' here didn't result in a null JSON value, but in a nullptr being cast to a string, which aborts. --- src/libstore/derived-path.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 05c2303db..3fa5ae4f7 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -20,11 +20,12 @@ nlohmann::json DerivedPath::Built::toJSON(ref store) const { // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it const auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); - for (const auto& output : outputs) { + for (const auto & output : outputs) { auto knownOutput = get(knownOutputs, output); - res["outputs"][output] = (knownOutput && *knownOutput) - ? store->printStorePath(**knownOutput) - : nullptr; + if (knownOutput && *knownOutput) + res["outputs"][output] = store->printStorePath(**knownOutput); + else + res["outputs"][output] = nullptr; } return res; } -- cgit v1.2.3 From c965f35de71cc9d88f912f6b90fd7213601e6eb8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 17 Dec 2022 14:50:43 +0100 Subject: Improve sqlite error messages They did not include the detailed error message, losing essential information for troubleshooting. Example message: warning: creating statement 'insert or rplace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)': at offset 10: SQL logic error, near "rplace": syntax error (in '/tmp/nix-shell.grQ6f7/nix-test/tests/binary-cache/test-home/.cache/nix/binary-cache-v6.sqlite') It's not the best example; more important information will be in the message for e.g. a constraint violation. I don't see why this specific error is printed as a warning, but that's for another commit. --- src/libstore/sqlite.cc | 15 ++++++++++----- src/libstore/sqlite.hh | 11 ++++++----- 2 files changed, 16 insertions(+), 10 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 2090beabd..6c350888f 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -8,12 +8,15 @@ namespace nix { -SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf) - : Error(""), path(path), errNo(errNo), extendedErrNo(extendedErrNo) +SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf) + : Error(""), path(path), errMsg(errMsg), errNo(errNo), extendedErrNo(extendedErrNo), offset(offset) { - err.msg = hintfmt("%s: %s (in '%s')", + auto offsetStr = (offset == -1) ? "" : "at offset " + std::to_string(offset) + ": "; + err.msg = hintfmt("%s: %s%s, %s (in '%s')", normaltxt(hf.str()), + offsetStr, sqlite3_errstr(extendedErrNo), + errMsg, path ? path : "(in-memory)"); } @@ -21,11 +24,13 @@ SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintfor { int err = sqlite3_errcode(db); int exterr = sqlite3_extended_errcode(db); + int offset = sqlite3_error_offset(db); auto path = sqlite3_db_filename(db, nullptr); + auto errMsg = sqlite3_errmsg(db); if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - auto exp = SQLiteBusy(path, err, exterr, std::move(hf)); + auto exp = SQLiteBusy(path, errMsg, err, exterr, offset, std::move(hf)); exp.err.msg = hintfmt( err == SQLITE_PROTOCOL ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" @@ -33,7 +38,7 @@ SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintfor path ? path : "(in-memory)"); throw exp; } else - throw SQLiteError(path, err, exterr, std::move(hf)); + throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf)); } SQLite::SQLite(const Path & path, bool create) diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 1d1c553ea..1853731a2 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -98,21 +98,22 @@ struct SQLiteTxn struct SQLiteError : Error { - const char *path; - int errNo, extendedErrNo; + std::string path; + std::string errMsg; + int errNo, extendedErrNo, offset; template [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args) { throw_(db, hintfmt(fs, args...)); } - SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf); + SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf); protected: template - SQLiteError(const char *path, int errNo, int extendedErrNo, const std::string & fs, const Args & ... args) - : SQLiteError(path, errNo, extendedErrNo, hintfmt(fs, args...)) + SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, const std::string & fs, const Args & ... args) + : SQLiteError(path, errNo, extendedErrNo, offset, hintfmt(fs, args...)) { } [[noreturn]] static void throw_(sqlite3 * db, hintformat && hf); -- cgit v1.2.3 From bc8ab21c5acbc9b94199be97dc66722cc5cf2252 Mon Sep 17 00:00:00 2001 From: mupdt <25388474+mupdt@users.noreply.github.com> Date: Wed, 21 Dec 2022 04:50:40 -0500 Subject: [PDT] TDE-3114: prevent a race-condition when creating the S3 cache --- src/libstore/nar-info-disk-cache.cc | 42 ++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 15 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index f4ea739b0..3e0689534 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -166,16 +166,37 @@ public: return i->second; } + std::optional queryCacheRaw(State & state, const std::string & uri) + { + auto i = state.caches.find(uri); + if (i == state.caches.end()) { + auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl)); + if (!queryCache.next()) + return std::nullopt; + state.caches.emplace(uri, + Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); + } + return getCache(state, uri); + } + void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override { retrySQLite([&]() { auto state(_state.lock()); + SQLiteTxn txn(state->db); + + // To avoid the race, we have to check if maybe someone hasn't yet created + // the cache for this URI in the meantime. + auto cache(queryCacheRaw(*state, uri)); - // FIXME: race + if (cache) + return; state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); assert(sqlite3_changes(state->db) == 1); state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; + + txn.commit(); }); } @@ -183,21 +204,12 @@ public: { return retrySQLite>([&]() -> std::optional { auto state(_state.lock()); - - auto i = state->caches.find(uri); - if (i == state->caches.end()) { - auto queryCache(state->queryCache.use()(uri)(time(0) - cacheInfoTtl)); - if (!queryCache.next()) - return std::nullopt; - state->caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); - } - - auto & cache(getCache(*state, uri)); - + auto cache(queryCacheRaw(*state, uri)); + if (!cache) + return std::nullopt; return CacheInfo { - .wantMassQuery = cache.wantMassQuery, - .priority = cache.priority + .wantMassQuery = cache->wantMassQuery, + .priority = cache->priority }; }); } -- cgit v1.2.3 From d034ed1891e66d63f1eae90444a516a931ebadc2 Mon Sep 17 00:00:00 2001 From: rski Date: Fri, 23 Dec 2022 02:33:32 +0200 Subject: src/libstore: Print the reason opening the DB failed Without this, the error is lost, and it makes for a hard to debug situation. Also remove some of the busyness inside the sqlite_open_v2 args. The errcode returned is not the extended one. The only way to make open return an extended code, would be to add SQLITE_OPEN_EXRESCODE to the flags. In the future it might be worth making this change, which would also simplify the existing SQLiteError code. --- src/libstore/sqlite.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 6c350888f..353dff9fa 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -47,9 +47,13 @@ SQLite::SQLite(const Path & path, bool create) // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK) - throw Error("cannot open SQLite database '%s'", path); + int flags = SQLITE_OPEN_READWRITE; + if (create) flags |= SQLITE_OPEN_CREATE; + int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs); + if (ret != SQLITE_OK) { + const char * err = sqlite3_errstr(ret); + throw Error("cannot open SQLite database '%s': %s", path, err); + } if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) SQLiteError::throw_(db, "setting timeout"); -- cgit v1.2.3 From c164d304f3cd6c5e536e33435084c030f018c2ab Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 23 Dec 2022 16:28:26 +0100 Subject: nix develop: Set personality This makes 'nix develop' set the Linux personality in the same way that the actual build does, allowing a command like 'nix develop nix#devShells.i686-linux.default' on x86_64-linux to work correctly. --- src/libstore/build/derivation-goal.cc | 1 - src/libstore/build/local-derivation-goal.cc | 31 ++------------------ src/libstore/build/personality.cc | 44 +++++++++++++++++++++++++++++ src/libstore/build/personality.hh | 11 ++++++++ 4 files changed, 57 insertions(+), 30 deletions(-) create mode 100644 src/libstore/build/personality.cc create mode 100644 src/libstore/build/personality.hh (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d3b995a4f..173058d1b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index dccd096ec..9d869d513 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -15,6 +15,7 @@ #include "callback.hh" #include "json-utils.hh" #include "cgroup.hh" +#include "personality.hh" #include #include @@ -24,7 +25,6 @@ #include #include #include -#include #include #include @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -1964,33 +1963,7 @@ void LocalDerivationGoal::runChild() /* Close all other file descriptors. */ closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); -#if __linux__ - /* Change the personality to 32-bit if we're doing an - i686-linux build on an x86_64-linux machine. */ - struct utsname utsbuf; - uname(&utsbuf); - if ((drv->platform == "i686-linux" - && (settings.thisSystem == "x86_64-linux" - || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) - || drv->platform == "armv7l-linux" - || drv->platform == "armv6l-linux") - { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set 32-bit personality"); - } - - /* Impersonate a Linux 2.6 machine to get some determinism in - builds that depend on the kernel version. */ - if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); - } - - /* Disable address space randomization for improved - determinism. */ - int cur = personality(0xffffffff); - if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); -#endif + setPersonality(drv->platform); /* Disable core dumps by default. */ struct rlimit limit = { 0, RLIM_INFINITY }; diff --git a/src/libstore/build/personality.cc b/src/libstore/build/personality.cc new file mode 100644 index 000000000..4ad477869 --- /dev/null +++ b/src/libstore/build/personality.cc @@ -0,0 +1,44 @@ +#include "personality.hh" +#include "globals.hh" + +#if __linux__ +#include +#include +#endif + +#include + +namespace nix { + +void setPersonality(std::string_view system) +{ +#if __linux__ + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if ((system == "i686-linux" + && (std::string_view(SYSTEM) == "x86_64-linux" + || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) + || system == "armv7l-linux" + || system == "armv6l-linux") + { + if (personality(PER_LINUX32) == -1) + throw SysError("cannot set 32-bit personality"); + } + + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((system == "i686-linux" || system == "x86_64-linux") && settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + } + + /* Disable address space randomization for improved + determinism. */ + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); +#endif +} + +} diff --git a/src/libstore/build/personality.hh b/src/libstore/build/personality.hh new file mode 100644 index 000000000..30e4f4062 --- /dev/null +++ b/src/libstore/build/personality.hh @@ -0,0 +1,11 @@ +#pragma once + +#include + +namespace nix { + +void setPersonality(std::string_view system); + +} + + -- cgit v1.2.3 From aba6eb348e0ed8177da1e7f1df46ba00d20eab96 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 19 Dec 2022 14:06:07 +0100 Subject: libstore: Make sure that initNix has been called Prevent bugs like https://github.com/cachix/cachix/pull/477 --- src/libstore/globals.cc | 14 ++++++++++++++ src/libstore/globals.hh | 8 ++++++++ src/libstore/store-api.cc | 1 + 3 files changed, 23 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index b7f55cae7..130c5b670 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -291,4 +291,18 @@ void initPlugins() settings.pluginFiles.pluginsLoaded = true; } +static bool initLibStoreDone = false; + +void assertLibStoreInitialized() { + if (!initLibStoreDone) { + printError("The program must call nix::initNix() before calling any libstore library functions."); + abort(); + }; +} + +void initLibStore() { + initLibStoreDone = true; +} + + } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 274a15dd7..22458efcf 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -984,4 +984,12 @@ std::vector getUserConfigFiles(); extern const std::string nixVersion; +/* NB: This is not sufficient. You need to call initNix() */ +void initLibStore(); + +/* It's important to initialize before doing _anything_, which is why we + call upon the programmer to handle this correctly. However, we only add + this in a key locations, so as not to litter the code. */ +void assertLibStoreInitialized(); + } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 80b60ca1b..426230ca5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -458,6 +458,7 @@ Store::Store(const Params & params) : StoreConfig(params) , state({(size_t) pathInfoCacheSize}) { + assertLibStoreInitialized(); } -- cgit v1.2.3 From 81c3f99b3668a1ea6a37792a5bcc3bc6f39729a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Sat, 17 Dec 2022 18:31:00 +0100 Subject: Release shared lock before acquiring exclusive lock In principle, this should avoid deadlocks where two instances of Nix are holding a shared lock on big-lock and are both waiting to get an exclusive lock. However, it seems like `flock(2)` is supposed to do this automatically, so it's not clear whether this is actually where the problem comes from. --- src/libstore/local-store.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b67668e52..3bab10af9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -91,6 +91,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) if (!lockFile(lockFd.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store for ca drvs..."); + lockFile(lockFd.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(lockFd.get(), ltWrite, true); } @@ -299,6 +300,7 @@ LocalStore::LocalStore(const Params & params) if (!lockFile(globalLock.get(), ltWrite, false)) { printInfo("waiting for exclusive access to the Nix store..."); + lockFile(globalLock.get(), ltNone, false); // We have acquired a shared lock; release it to prevent deadlocks lockFile(globalLock.get(), ltWrite, true); } -- cgit v1.2.3 From d5d2f50ebbe5ec2b8a9777d4184eb3e604a1f8c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 28 Dec 2022 17:09:20 +0100 Subject: doc: sandbox-paths computes closures --- src/libstore/globals.hh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 274a15dd7..f4d53757c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -329,7 +329,7 @@ public: Whether to execute builds inside cgroups. This is only supported on Linux. - Cgroups are required and enabled automatically for derivations + Cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. > **Warning** @@ -491,6 +491,9 @@ public: for example, `/dev/nvidiactl?` specifies that `/dev/nvidiactl` will only be mounted in the sandbox if it exists in the host filesystem. + If the source is in the Nix store, then its closure will be added to + the sandbox as well. + Depending on how Nix was built, the default value for this option may be empty or provide `/bin/sh` as a bind-mount of `bash`. )", -- cgit v1.2.3 From 84b08937259745ca743c0b305fc63ab1e1a20d83 Mon Sep 17 00:00:00 2001 From: Steven Shaw Date: Sun, 1 Jan 2023 12:37:43 +1000 Subject: Fix error message --- src/libstore/builtins/buildenv.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 47458a388..b1fbda13d 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -95,7 +95,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw Error( "files '%1%' and '%2%' have the same priority %3%; " "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "or type 'nix profile install --help' if using 'nix profile' to find out how" + "or type 'nix profile install --help' if using 'nix profile' to find out how " "to change the priority of one of the conflicting packages" " (0 being the highest priority)", srcFile, readLink(dstFile), priority); -- cgit v1.2.3 From 8cac451fce990151046996a13130bb1b91c6ba19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 2 Jan 2023 17:35:48 +0100 Subject: Fix why-depends for CA derivations (again) This has the same goal as b13fd4c58e81b2b2b0d72caa5ce80de861622610,but achieves it in a different way in order to not break `nix why-depends --derivation`. --- src/libstore/realisation.hh | 10 ++++++++++ src/libstore/remote-store.cc | 5 +---- 2 files changed, 11 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 9070a6ee2..911c61909 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -93,4 +93,14 @@ struct RealisedPath { GENERATE_CMP(RealisedPath, me->raw); }; +class MissingRealisation : public Error +{ +public: + MissingRealisation(DrvOutput & outputId) + : Error( "cannot operate on an output of the " + "unbuilt derivation '%s'", + outputId.to_string()) + {} +}; + } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 48cf731a8..ccf7d7e8b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -879,10 +879,7 @@ std::vector RemoteStore::buildPathsWithResults( auto realisation = queryRealisation(outputId); if (!realisation) - throw Error( - "cannot operate on an output of unbuilt " - "content-addressed derivation '%s'", - outputId.to_string()); + throw MissingRealisation(outputId); res.builtOutputs.emplace(realisation->id, *realisation); } else { // If ca-derivations isn't enabled, assume that -- cgit v1.2.3 From 224b56f10e1bb754d403c106eb9d1b947fc30414 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Jan 2023 14:51:23 +0100 Subject: Move creation of the temp roots file into its own function This also moves the file handle into its own Sync object so we're not holding the _state while acquiring the file lock. There was no real deadlock risk here since locking a newly created file cannot block, but it's still a bit nicer. --- src/libstore/gc.cc | 52 +++++++++++++++++++++++++-------------------- src/libstore/local-store.cc | 6 +++--- src/libstore/local-store.hh | 12 ++++++++--- 3 files changed, 41 insertions(+), 29 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 5d91829f1..f8c29593f 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -77,37 +77,43 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot } -void LocalStore::addTempRoot(const StorePath & path) +void LocalStore::createTempRootsFile() { - auto state(_state.lock()); + auto fdTempRoots(_fdTempRoots.lock()); /* Create the temporary roots file for this process. */ - if (!state->fdTempRoots) { - - while (1) { - if (pathExists(fnTempRoots)) - /* It *must* be stale, since there can be no two - processes with the same pid. */ - unlink(fnTempRoots.c_str()); + if (*fdTempRoots) return; - state->fdTempRoots = openLockFile(fnTempRoots, true); + while (1) { + if (pathExists(fnTempRoots)) + /* It *must* be stale, since there can be no two + processes with the same pid. */ + unlink(fnTempRoots.c_str()); - debug("acquiring write lock on '%s'", fnTempRoots); - lockFile(state->fdTempRoots.get(), ltWrite, true); + *fdTempRoots = openLockFile(fnTempRoots, true); - /* Check whether the garbage collector didn't get in our - way. */ - struct stat st; - if (fstat(state->fdTempRoots.get(), &st) == -1) - throw SysError("statting '%1%'", fnTempRoots); - if (st.st_size == 0) break; + debug("acquiring write lock on '%s'", fnTempRoots); + lockFile(fdTempRoots->get(), ltWrite, true); - /* The garbage collector deleted this file before we could - get a lock. (It won't delete the file after we get a - lock.) Try again. */ - } + /* Check whether the garbage collector didn't get in our + way. */ + struct stat st; + if (fstat(fdTempRoots->get(), &st) == -1) + throw SysError("statting '%1%'", fnTempRoots); + if (st.st_size == 0) break; + /* The garbage collector deleted this file before we could get + a lock. (It won't delete the file after we get a lock.) + Try again. */ } +} + + +void LocalStore::addTempRoot(const StorePath & path) +{ + createTempRootsFile(); + + auto state(_state.lock()); if (!state->fdGCLock) state->fdGCLock = openGCLock(); @@ -162,7 +168,7 @@ void LocalStore::addTempRoot(const StorePath & path) /* Append the store path to the temporary roots file. */ auto s = printStorePath(path) + '\0'; - writeFull(state->fdTempRoots.get(), s); + writeFull(_fdTempRoots.lock()->get(), s); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3bab10af9..be21e3ca0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -441,9 +441,9 @@ LocalStore::~LocalStore() } try { - auto state(_state.lock()); - if (state->fdTempRoots) { - state->fdTempRoots = -1; + auto fdTempRoots(_fdTempRoots.lock()); + if (*fdTempRoots) { + *fdTempRoots = -1; unlink(fnTempRoots.c_str()); } } catch (...) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 4579c2f62..9ea0c0bf7 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -62,9 +62,6 @@ private: /* The global GC lock */ AutoCloseFD fdGCLock; - /* The file to which we write our temporary roots. */ - AutoCloseFD fdTempRoots; - /* Connection to the garbage collector. */ AutoCloseFD fdRootsSocket; @@ -156,6 +153,15 @@ public: void addTempRoot(const StorePath & path) override; +private: + + void createTempRootsFile(); + + /* The file to which we write our temporary roots. */ + Sync _fdTempRoots; + +public: + void addIndirectRoot(const Path & path) override; private: -- cgit v1.2.3 From 28d5b5cd453d89642557455d82b98903da2898b7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Jan 2023 15:15:14 +0100 Subject: Fix deadlock between auto-GC and addTempRoot() Previously addTempRoot() acquired the LocalStore state lock and waited for the garbage collector to reply. If the garbage collector is in the same process (as it the case with auto-GC), this would deadlock as soon as the garbage collector thread needs the LocalStore state lock. So now addTempRoot() uses separate Syncs for the state that it needs. As long at the auto-GC thread doesn't call addTempRoot() (which it shouldn't), it shouldn't deadlock. Fixes #3224. --- src/libstore/gc.cc | 36 ++++++++++++++++++++++-------------- src/libstore/local-store.hh | 12 ++++++------ 2 files changed, 28 insertions(+), 20 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f8c29593f..996f26a95 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -113,30 +113,37 @@ void LocalStore::addTempRoot(const StorePath & path) { createTempRootsFile(); - auto state(_state.lock()); - - if (!state->fdGCLock) - state->fdGCLock = openGCLock(); + /* Open/create the global GC lock file. */ + { + auto fdGCLock(_fdGCLock.lock()); + if (!*fdGCLock) + *fdGCLock = openGCLock(); + } restart: - FdLock gcLock(state->fdGCLock.get(), ltRead, false, ""); + /* Try to acquire a shared global GC lock (non-blocking). This + only succeeds if the garbage collector is not currently + running. */ + FdLock gcLock(_fdGCLock.lock()->get(), ltRead, false, ""); if (!gcLock.acquired) { /* We couldn't get a shared global GC lock, so the garbage collector is running. So we have to connect to the garbage collector and inform it about our root. */ - if (!state->fdRootsSocket) { + auto fdRootsSocket(_fdRootsSocket.lock()); + + if (!*fdRootsSocket) { auto socketPath = stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); - state->fdRootsSocket = createUnixDomainSocket(); + *fdRootsSocket = createUnixDomainSocket(); try { - nix::connect(state->fdRootsSocket.get(), socketPath); + nix::connect(fdRootsSocket->get(), socketPath); } catch (SysError & e) { /* The garbage collector may have exited, so we need to restart. */ if (e.errNo == ECONNREFUSED) { debug("GC socket connection refused"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } throw; @@ -145,9 +152,9 @@ void LocalStore::addTempRoot(const StorePath & path) try { debug("sending GC root '%s'", printStorePath(path)); - writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false); + writeFull(fdRootsSocket->get(), printStorePath(path) + "\n", false); char c; - readFull(state->fdRootsSocket.get(), &c, 1); + readFull(fdRootsSocket->get(), &c, 1); assert(c == '1'); debug("got ack for GC root '%s'", printStorePath(path)); } catch (SysError & e) { @@ -155,18 +162,19 @@ void LocalStore::addTempRoot(const StorePath & path) restart. */ if (e.errNo == EPIPE || e.errNo == ECONNRESET) { debug("GC socket disconnected"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } throw; } catch (EndOfFile & e) { debug("GC socket disconnected"); - state->fdRootsSocket.close(); + fdRootsSocket->close(); goto restart; } } - /* Append the store path to the temporary roots file. */ + /* Record the store path in the temporary roots file so it will be + seen by a future run of the garbage collector. */ auto s = printStorePath(path) + '\0'; writeFull(_fdTempRoots.lock()->get(), s); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9ea0c0bf7..06d36a7d5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -59,12 +59,6 @@ private: struct Stmts; std::unique_ptr stmts; - /* The global GC lock */ - AutoCloseFD fdGCLock; - - /* Connection to the garbage collector. */ - AutoCloseFD fdRootsSocket; - /* The last time we checked whether to do an auto-GC, or an auto-GC finished. */ std::chrono::time_point lastGCCheck; @@ -160,6 +154,12 @@ private: /* The file to which we write our temporary roots. */ Sync _fdTempRoots; + /* The global GC lock. */ + Sync _fdGCLock; + + /* Connection to the garbage collector. */ + Sync _fdRootsSocket; + public: void addIndirectRoot(const Path & path) override; -- cgit v1.2.3 From 609a7dc05974c9f86b2e7304762b9e01c5879380 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Jan 2023 04:36:07 -0800 Subject: Include macOS sandbox files in the Nix binary This basically reverts 6e5165b77370c76bfa39d4b55e9f83673f3bd466. It fixes errors like sandbox-exec: :292:47: unable to open sandbox-minimal.sb: not found when trying to run a development Nix installed in a user's home directory. Also, we're trying to minimize the number of installed files to make it possible to deploy Nix as a single statically-linked binary. --- src/libstore/build/local-derivation-goal.cc | 14 +++++++++----- src/libstore/local.mk | 4 ---- src/libstore/sandbox-defaults.sb | 4 ++++ src/libstore/sandbox-minimal.sb | 4 ++++ src/libstore/sandbox-network.sb | 4 ++++ 5 files changed, 21 insertions(+), 9 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9d869d513..488e06d8c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2050,10 +2050,14 @@ void LocalDerivationGoal::runChild() sandboxProfile += "(deny default (with no-log))\n"; } - sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; + sandboxProfile += + #include "sandbox-defaults.sb" + ; if (!derivationType.isSandboxed()) - sandboxProfile += "(import \"sandbox-network.sb\")\n"; + sandboxProfile += + #include "sandbox-network.sb" + ; /* Add the output paths we'll use at build-time to the chroot */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; @@ -2096,7 +2100,9 @@ void LocalDerivationGoal::runChild() sandboxProfile += additionalSandboxProfile; } else - sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; + sandboxProfile += + #include "sandbox-minimal.sb" + ; debug("Generated sandbox profile:"); debug(sandboxProfile); @@ -2121,8 +2127,6 @@ void LocalDerivationGoal::runChild() args.push_back(sandboxFile); args.push_back("-D"); args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - args.push_back("-D"); - args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); if (allowLocalNetworking) { args.push_back("-D"); args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1")); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 8f28bec6c..e5e24501e 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -13,10 +13,6 @@ ifdef HOST_LINUX libstore_LDFLAGS += -ldl endif -ifdef HOST_DARWIN -libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb -endif - $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) ifeq ($(ENABLE_S3), 1) diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index d9d710559..77f013aea 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -1,3 +1,5 @@ +R""( + (define TMPDIR (param "_GLOBAL_TMP_DIR")) (deny default) @@ -104,3 +106,5 @@ (subpath "/System/Library/Apple/usr/libexec/oah") (subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist") (subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) + +)"" diff --git a/src/libstore/sandbox-minimal.sb b/src/libstore/sandbox-minimal.sb index 65f5108b3..976a1f636 100644 --- a/src/libstore/sandbox-minimal.sb +++ b/src/libstore/sandbox-minimal.sb @@ -1,5 +1,9 @@ +R""( + (allow default) ; Disallow creating setuid/setgid binaries, since that ; would allow breaking build user isolation. (deny file-write-setugid) + +)"" diff --git a/src/libstore/sandbox-network.sb b/src/libstore/sandbox-network.sb index 19e9eea9a..335edbaed 100644 --- a/src/libstore/sandbox-network.sb +++ b/src/libstore/sandbox-network.sb @@ -1,3 +1,5 @@ +R""( + ; Allow local and remote network traffic. (allow network* (local ip) (remote ip)) @@ -18,3 +20,5 @@ ; Allow access to trustd. (allow mach-lookup (global-name "com.apple.trustd")) (allow mach-lookup (global-name "com.apple.trustd.agent")) + +)"" -- cgit v1.2.3 From 6991e558ddaaf037954741830078f933a36ec2f2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Jan 2023 04:50:45 -0800 Subject: Move macOS sandbox files to sr/libstore/build --- src/libstore/build/sandbox-defaults.sb | 110 +++++++++++++++++++++++++++++++++ src/libstore/build/sandbox-minimal.sb | 9 +++ src/libstore/build/sandbox-network.sb | 24 +++++++ src/libstore/sandbox-defaults.sb | 110 --------------------------------- src/libstore/sandbox-minimal.sb | 9 --- src/libstore/sandbox-network.sb | 24 ------- 6 files changed, 143 insertions(+), 143 deletions(-) create mode 100644 src/libstore/build/sandbox-defaults.sb create mode 100644 src/libstore/build/sandbox-minimal.sb create mode 100644 src/libstore/build/sandbox-network.sb delete mode 100644 src/libstore/sandbox-defaults.sb delete mode 100644 src/libstore/sandbox-minimal.sb delete mode 100644 src/libstore/sandbox-network.sb (limited to 'src/libstore') diff --git a/src/libstore/build/sandbox-defaults.sb b/src/libstore/build/sandbox-defaults.sb new file mode 100644 index 000000000..77f013aea --- /dev/null +++ b/src/libstore/build/sandbox-defaults.sb @@ -0,0 +1,110 @@ +R""( + +(define TMPDIR (param "_GLOBAL_TMP_DIR")) + +(deny default) + +; Disallow creating setuid/setgid binaries, since that +; would allow breaking build user isolation. +(deny file-write-setugid) + +; Allow forking. +(allow process-fork) + +; Allow reading system information like #CPUs, etc. +(allow sysctl-read) + +; Allow POSIX semaphores and shared memory. +(allow ipc-posix*) + +; Allow socket creation. +(allow system-socket) + +; Allow sending signals within the sandbox. +(allow signal (target same-sandbox)) + +; Allow getpwuid. +(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo")) + +; Access to /tmp. +; The network-outbound/network-inbound ones are for unix domain sockets, which +; we allow access to in TMPDIR (but if we allow them more broadly, you could in +; theory escape the sandbox) +(allow file* process-exec network-outbound network-inbound + (literal "/tmp") (subpath TMPDIR)) + +; Some packages like to read the system version. +(allow file-read* + (literal "/System/Library/CoreServices/SystemVersion.plist") + (literal "/System/Library/CoreServices/SystemVersionCompat.plist")) + +; Without this line clang cannot write to /dev/null, breaking some configure tests. +(allow file-read-metadata (literal "/dev")) + +; Many packages like to do local networking in their test suites, but let's only +; allow it if the package explicitly asks for it. +(if (param "_ALLOW_LOCAL_NETWORKING") + (begin + (allow network* (local ip) (local tcp) (local udp)) + + ; Allow access to /etc/resolv.conf (which is a symlink to + ; /private/var/run/resolv.conf). + ; TODO: deduplicate with sandbox-network.sb + (allow file-read-metadata + (literal "/var") + (literal "/etc") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + + (allow file-read* + (literal "/private/var/run/resolv.conf")) + + ; Allow DNS lookups. This is even needed for localhost, which lots of tests rely on + (allow file-read-metadata (literal "/etc/hosts")) + (allow file-read* (literal "/private/etc/hosts")) + (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))))) + +; Standard devices. +(allow file* + (literal "/dev/null") + (literal "/dev/random") + (literal "/dev/stdin") + (literal "/dev/stdout") + (literal "/dev/tty") + (literal "/dev/urandom") + (literal "/dev/zero") + (subpath "/dev/fd")) + +; Allow pseudo-terminals. +(allow file* + (literal "/dev/ptmx") + (regex #"^/dev/pty[a-z]+") + (regex #"^/dev/ttys[0-9]+")) + +; Does nothing, but reduces build noise. +(allow file* (literal "/dev/dtracehelper")) + +; Allow access to zoneinfo since libSystem needs it. +(allow file-read* (subpath "/usr/share/zoneinfo")) + +(allow file-read* (subpath "/usr/share/locale")) + +; This is mostly to get more specific log messages when builds try to +; access something in /etc or /var. +(allow file-read-metadata + (literal "/etc") + (literal "/var") + (literal "/private/var/tmp")) + +; This is used by /bin/sh on macOS 10.15 and later. +(allow file* + (literal "/private/var/select/sh")) + +; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin (and vice versa). +(allow file-read* + (subpath "/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist") + (subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) + +)"" diff --git a/src/libstore/build/sandbox-minimal.sb b/src/libstore/build/sandbox-minimal.sb new file mode 100644 index 000000000..976a1f636 --- /dev/null +++ b/src/libstore/build/sandbox-minimal.sb @@ -0,0 +1,9 @@ +R""( + +(allow default) + +; Disallow creating setuid/setgid binaries, since that +; would allow breaking build user isolation. +(deny file-write-setugid) + +)"" diff --git a/src/libstore/build/sandbox-network.sb b/src/libstore/build/sandbox-network.sb new file mode 100644 index 000000000..335edbaed --- /dev/null +++ b/src/libstore/build/sandbox-network.sb @@ -0,0 +1,24 @@ +R""( + +; Allow local and remote network traffic. +(allow network* (local ip) (remote ip)) + +; Allow access to /etc/resolv.conf (which is a symlink to +; /private/var/run/resolv.conf). +(allow file-read-metadata + (literal "/var") + (literal "/etc") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + +(allow file-read* + (literal "/private/var/run/resolv.conf")) + +; Allow DNS lookups. +(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))) + +; Allow access to trustd. +(allow mach-lookup (global-name "com.apple.trustd")) +(allow mach-lookup (global-name "com.apple.trustd.agent")) + +)"" diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb deleted file mode 100644 index 77f013aea..000000000 --- a/src/libstore/sandbox-defaults.sb +++ /dev/null @@ -1,110 +0,0 @@ -R""( - -(define TMPDIR (param "_GLOBAL_TMP_DIR")) - -(deny default) - -; Disallow creating setuid/setgid binaries, since that -; would allow breaking build user isolation. -(deny file-write-setugid) - -; Allow forking. -(allow process-fork) - -; Allow reading system information like #CPUs, etc. -(allow sysctl-read) - -; Allow POSIX semaphores and shared memory. -(allow ipc-posix*) - -; Allow socket creation. -(allow system-socket) - -; Allow sending signals within the sandbox. -(allow signal (target same-sandbox)) - -; Allow getpwuid. -(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo")) - -; Access to /tmp. -; The network-outbound/network-inbound ones are for unix domain sockets, which -; we allow access to in TMPDIR (but if we allow them more broadly, you could in -; theory escape the sandbox) -(allow file* process-exec network-outbound network-inbound - (literal "/tmp") (subpath TMPDIR)) - -; Some packages like to read the system version. -(allow file-read* - (literal "/System/Library/CoreServices/SystemVersion.plist") - (literal "/System/Library/CoreServices/SystemVersionCompat.plist")) - -; Without this line clang cannot write to /dev/null, breaking some configure tests. -(allow file-read-metadata (literal "/dev")) - -; Many packages like to do local networking in their test suites, but let's only -; allow it if the package explicitly asks for it. -(if (param "_ALLOW_LOCAL_NETWORKING") - (begin - (allow network* (local ip) (local tcp) (local udp)) - - ; Allow access to /etc/resolv.conf (which is a symlink to - ; /private/var/run/resolv.conf). - ; TODO: deduplicate with sandbox-network.sb - (allow file-read-metadata - (literal "/var") - (literal "/etc") - (literal "/etc/resolv.conf") - (literal "/private/etc/resolv.conf")) - - (allow file-read* - (literal "/private/var/run/resolv.conf")) - - ; Allow DNS lookups. This is even needed for localhost, which lots of tests rely on - (allow file-read-metadata (literal "/etc/hosts")) - (allow file-read* (literal "/private/etc/hosts")) - (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))))) - -; Standard devices. -(allow file* - (literal "/dev/null") - (literal "/dev/random") - (literal "/dev/stdin") - (literal "/dev/stdout") - (literal "/dev/tty") - (literal "/dev/urandom") - (literal "/dev/zero") - (subpath "/dev/fd")) - -; Allow pseudo-terminals. -(allow file* - (literal "/dev/ptmx") - (regex #"^/dev/pty[a-z]+") - (regex #"^/dev/ttys[0-9]+")) - -; Does nothing, but reduces build noise. -(allow file* (literal "/dev/dtracehelper")) - -; Allow access to zoneinfo since libSystem needs it. -(allow file-read* (subpath "/usr/share/zoneinfo")) - -(allow file-read* (subpath "/usr/share/locale")) - -; This is mostly to get more specific log messages when builds try to -; access something in /etc or /var. -(allow file-read-metadata - (literal "/etc") - (literal "/var") - (literal "/private/var/tmp")) - -; This is used by /bin/sh on macOS 10.15 and later. -(allow file* - (literal "/private/var/select/sh")) - -; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin (and vice versa). -(allow file-read* - (subpath "/Library/Apple/usr/libexec/oah") - (subpath "/System/Library/Apple/usr/libexec/oah") - (subpath "/System/Library/LaunchDaemons/com.apple.oahd.plist") - (subpath "/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) - -)"" diff --git a/src/libstore/sandbox-minimal.sb b/src/libstore/sandbox-minimal.sb deleted file mode 100644 index 976a1f636..000000000 --- a/src/libstore/sandbox-minimal.sb +++ /dev/null @@ -1,9 +0,0 @@ -R""( - -(allow default) - -; Disallow creating setuid/setgid binaries, since that -; would allow breaking build user isolation. -(deny file-write-setugid) - -)"" diff --git a/src/libstore/sandbox-network.sb b/src/libstore/sandbox-network.sb deleted file mode 100644 index 335edbaed..000000000 --- a/src/libstore/sandbox-network.sb +++ /dev/null @@ -1,24 +0,0 @@ -R""( - -; Allow local and remote network traffic. -(allow network* (local ip) (remote ip)) - -; Allow access to /etc/resolv.conf (which is a symlink to -; /private/var/run/resolv.conf). -(allow file-read-metadata - (literal "/var") - (literal "/etc") - (literal "/etc/resolv.conf") - (literal "/private/etc/resolv.conf")) - -(allow file-read* - (literal "/private/var/run/resolv.conf")) - -; Allow DNS lookups. -(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))) - -; Allow access to trustd. -(allow mach-lookup (global-name "com.apple.trustd")) -(allow mach-lookup (global-name "com.apple.trustd.agent")) - -)"" -- cgit v1.2.3 From 4e84b532ed5317ec836c54689c73a1fddab0c892 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Jan 2023 04:58:55 -0800 Subject: On macOS with auto-uid-allocation and sandboxing, use the correct gid macOS doesn't have user namespacing, so the gid of the builder needs to be nixbld. The logic got "has sandboxing enabled" confused with "has user namespaces". Fixes #7529. --- src/libstore/lock.cc | 12 ++++++++---- src/libstore/lock.hh | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index d02d20b4c..4fe1fcf56 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -123,8 +123,12 @@ struct AutoUserLock : UserLock std::vector getSupplementaryGIDs() override { return {}; } - static std::unique_ptr acquire(uid_t nrIds, bool useChroot) + static std::unique_ptr acquire(uid_t nrIds, bool useUserNamespace) { + #if !defined(__linux__) + useUserNamespace = false; + #endif + settings.requireExperimentalFeature(Xp::AutoAllocateUids); assert(settings.startId > 0); assert(settings.uidCount % maxIdsPerBuild == 0); @@ -157,7 +161,7 @@ struct AutoUserLock : UserLock auto lock = std::make_unique(); lock->fdUserLock = std::move(fd); lock->firstUid = firstUid; - if (useChroot) + if (useUserNamespace) lock->firstGid = firstUid; else { struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); @@ -174,10 +178,10 @@ struct AutoUserLock : UserLock } }; -std::unique_ptr acquireUserLock(uid_t nrIds, bool useChroot) +std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace) { if (settings.autoAllocateUids) - return AutoUserLock::acquire(nrIds, useChroot); + return AutoUserLock::acquire(nrIds, useUserNamespace); else return SimpleUserLock::acquire(); } diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 49ad86de7..7f1934510 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -31,7 +31,7 @@ struct UserLock /* Acquire a user lock for a UID range of size `nrIds`. Note that this may return nullptr if no user is available. */ -std::unique_ptr acquireUserLock(uid_t nrIds, bool useChroot); +std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace); bool useBuildUsers(); -- cgit v1.2.3