From 3113b13df9afaab918792c725742c6bc3fcec88b Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Wed, 8 Feb 2023 20:03:57 +0100 Subject: buildenv: throw BuildEnvFileConflictError with more context At the moment an Error is thrown that only holds an error message regarding `nix-env` and `nix profile`. These tools make use of builtins.buildEnv, but buildEnv is also used in other places. These places are unrelated to Nix profiles, so the error shouldn't mention these tools. This generic error is now BuildEnvFileConflictError, which holds more contextual information about the files that were conflicting while building the environment. --- src/libstore/builtins/buildenv.cc | 12 +++++------- src/libstore/builtins/buildenv.hh | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index b1fbda13d..7bba33fb9 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -92,13 +92,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, if (S_ISLNK(dstSt.st_mode)) { auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) - throw Error( - "files '%1%' and '%2%' have the same priority %3%; " - "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " - "or type 'nix profile install --help' if using 'nix profile' to find out how " - "to change the priority of one of the conflicting packages" - " (0 being the highest priority)", - srcFile, readLink(dstFile), priority); + throw BuildEnvFileConflictError( + readLink(dstFile), + srcFile, + priority + ); if (prevPriority < priority) continue; if (unlink(dstFile.c_str()) == -1) diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 73c0f5f7f..a018de3af 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -12,6 +12,32 @@ struct Package { Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; +class BuildEnvFileConflictError : public Error +{ +public: + const Path fileA; + const Path fileB; + int priority; + + BuildEnvFileConflictError( + const Path fileA, + const Path fileB, + int priority + ) + : Error( + "Unable to build profile. There is a conflict for the following files:\n" + "\n" + " %1%\n" + " %2%", + fileA, + fileB + ) + , fileA(fileA) + , fileB(fileB) + , priority(priority) + {} +}; + typedef std::vector Packages; void buildProfile(const Path & out, Packages && pkgs); -- cgit v1.2.3 From f86f2b973f3b07134736fd96c1eff1da184dc1af Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Tue, 28 Feb 2023 16:04:17 -0700 Subject: Always set valid in path-info --json output Currently the valid key is only present when the path is invalid, which makes checking path validity more complex than it should be. With this change, the valid key can always be used to check if a path is valid --- src/libstore/store-api.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 601efa1cc..f32c2d30c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -855,6 +855,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths, auto info = queryPathInfo(storePath); jsonPath["path"] = printStorePath(info->path); + jsonPath["valid"] = true; jsonPath["narHash"] = info->narHash.to_string(hashBase, true); jsonPath["narSize"] = info->narSize; -- cgit v1.2.3 From b7f01a82a9b0b2c66bb5d58e1518a113d20e79c5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 1 Mar 2023 11:09:47 -0500 Subject: Remove dead code `RemoteStore::sameMachine` It has been dead code since 9747ea84b43c75f719d719673b9d0a7cb50d34e5. --- src/libstore/remote-store.hh | 2 -- src/libstore/ssh-store.cc | 3 --- src/libstore/uds-remote-store.hh | 3 --- 3 files changed, 8 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 11d089cd2..8cd7cc822 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -38,8 +38,6 @@ class RemoteStore : public virtual RemoteStoreConfig, { public: - virtual bool sameMachine() = 0; - RemoteStore(const Params & params); /* Implementations of abstract store API methods. */ diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index a1d4daafd..cfa063803 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -49,9 +49,6 @@ public: return *uriSchemes().begin() + "://" + host; } - bool sameMachine() override - { return false; } - // FIXME extend daemon protocol, move implementation to RemoteStore std::optional getBuildLogExact(const StorePath & path) override { unsupported("getBuildLogExact"); } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index f8dfcca70..d31a4d592 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -29,9 +29,6 @@ public: static std::set uriSchemes() { return {"unix"}; } - bool sameMachine() override - { return true; } - ref getFSAccessor() override { return LocalFSStore::getFSAccessor(); } -- cgit v1.2.3 From 25300c0ecdedcd8720b996b41e78dfe1037bfcff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Baylac=20Jacqu=C3=A9?= Date: Wed, 1 Mar 2023 20:01:36 +0100 Subject: Treat empty env var paths as unset We make sure the env var paths are actually set (ie. not "") before sending them to the canonicalization function. If we forget to do so, the user will end up facing a puzzled failed assertion internal error. We issue a non-failing warning as a stop-gap measure. We could want to revisit this to issue a detailed failing error message in the future. --- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/globals.cc | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a961d8eed..1c0860993 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2063,7 +2063,7 @@ void LocalDerivationGoal::runChild() /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true); + Path globalTmpDir = canonPath(getEnvNonEmpty("TMPDIR").value_or("/tmp"), true); /* They don't like trailing slashes on subpath directives */ if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 8e33a3dec..fae79c1a0 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -30,15 +30,15 @@ static GlobalConfig::Register rSettings(&settings); Settings::Settings() : nixPrefix(NIX_PREFIX) - , nixStore(canonPath(getEnv("NIX_STORE_DIR").value_or(getEnv("NIX_STORE").value_or(NIX_STORE_DIR)))) - , nixDataDir(canonPath(getEnv("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) - , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) - , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) - , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) + , nixStore(canonPath(getEnvNonEmpty("NIX_STORE_DIR").value_or(getEnvNonEmpty("NIX_STORE").value_or(NIX_STORE_DIR)))) + , nixDataDir(canonPath(getEnvNonEmpty("NIX_DATA_DIR").value_or(NIX_DATA_DIR))) + , nixLogDir(canonPath(getEnvNonEmpty("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) + , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) + , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) + , nixBinDir(canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) - , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) + , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; -- cgit v1.2.3 From 1a825b6919e9a3a98caec58192bb89dfb4e5bcfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 2 Mar 2023 14:59:15 +0100 Subject: Log the decompressed body in case of http errors Don't show the users the raw (possibly compressed) error message as everyone isn't able to decompress brotli in their brain. --- src/libstore/filetransfer.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 1c8676a59..6a4778d1f 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -88,6 +88,10 @@ struct curlFileTransfer : public FileTransfer {request.uri}, request.parentAct) , callback(std::move(callback)) , finalSink([this](std::string_view data) { + if (errorSink) { + (*errorSink)(data); + } + if (this->request.dataCallback) { auto httpStatus = getHTTPStatus(); @@ -163,8 +167,6 @@ struct curlFileTransfer : public FileTransfer } } - if (errorSink) - (*errorSink)({(char *) contents, realSize}); (*decompressionSink)({(char *) contents, realSize}); return realSize; -- cgit v1.2.3 From 29abc8e7647cd9ec6ef5a1df3fcb1dcf4a4495a2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Mar 2023 15:44:19 +0100 Subject: Remove FormatOrString and remaining uses of format() --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/goal.cc | 4 ++-- src/libstore/build/goal.hh | 2 +- src/libstore/build/local-derivation-goal.cc | 7 +++---- src/libstore/daemon.cc | 4 ++-- src/libstore/export-import.cc | 4 ++-- src/libstore/filetransfer.cc | 4 ++-- src/libstore/gc.cc | 9 ++++----- src/libstore/local-store.cc | 6 +++--- src/libstore/optimise-store.cc | 17 ++++++++--------- src/libstore/pathlocks.cc | 8 ++++---- src/libstore/profiles.cc | 4 ++-- src/libstore/references.cc | 3 +-- src/libstore/s3-binary-cache-store.cc | 12 ++++++------ src/libstore/store-api.cc | 4 ++-- 15 files changed, 43 insertions(+), 47 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2021d0023..38b73d531 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -732,7 +732,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) tmpPath (the replacement), so we have to move it out of the way first. We'd better not be interrupted here, because if we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); + Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), random()); if (pathExists(storePath)) movePath(storePath, oldPath); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 58e805f55..d59b94797 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -78,9 +78,9 @@ void Goal::amDone(ExitCode result, std::optional ex) } -void Goal::trace(const FormatOrString & fs) +void Goal::trace(std::string_view s) { - debug("%1%: %2%", name, fs.s); + debug("%1%: %2%", name, s); } } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 35121c5d9..776eb86bc 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -88,7 +88,7 @@ struct Goal : public std::enable_shared_from_this abort(); } - void trace(const FormatOrString & fs); + void trace(std::string_view s); std::string getName() { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a961d8eed..d17715efd 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -650,7 +650,7 @@ void LocalDerivationGoal::startBuilder() /* Clean up the chroot directory automatically. */ autoDelChroot = std::make_shared(chrootRootDir); - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); + printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootRootDir); // FIXME: make this 0700 if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) @@ -753,8 +753,7 @@ void LocalDerivationGoal::startBuilder() throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") - % settings.preBuildHook); + printMsg(lvlChatty, "executing pre-build hook '%1%'", settings.preBuildHook); auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : Strings({ worker.store.printStorePath(drvPath) }); enum BuildHookState { @@ -1104,7 +1103,7 @@ void LocalDerivationGoal::initEnv() env["NIX_STORE"] = worker.store.storeDir; /* The maximum number of cores to utilize for parallel building. */ - env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + env["NIX_BUILD_CORES"] = fmt("%d", settings.buildCores); initTmpDir(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5e6fd011f..7f8b0f905 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -67,12 +67,12 @@ struct TunnelLogger : public Logger state->pendingMsgs.push_back(s); } - void log(Verbosity lvl, const FormatOrString & fs) override + void log(Verbosity lvl, std::string_view s) override { if (lvl > verbosity) return; StringSink buf; - buf << STDERR_NEXT << (fs.s + "\n"); + buf << STDERR_NEXT << (s + "\n"); enqueueMsg(buf.s); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 9875da909..4eb838b68 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -16,7 +16,7 @@ void Store::exportPaths(const StorePathSet & paths, Sink & sink) //logger->incExpected(doneLabel, sorted.size()); for (auto & path : sorted) { - //Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); + //Activity act(*logger, lvlInfo, "exporting path '%s'", path); sink << 1; exportPath(path, sink); //logger->incProgress(doneLabel); @@ -71,7 +71,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) auto path = parseStorePath(readString(source)); - //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); + //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); auto references = worker_proto::read(*this, source, Phantom {}); auto deriver = readString(source); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 1c8676a59..404f980b6 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -183,7 +183,7 @@ struct curlFileTransfer : public FileTransfer { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); + printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); std::smatch match; if (std::regex_match(line, match, statusLine)) { @@ -207,7 +207,7 @@ struct curlFileTransfer : public FileTransfer long httpStatus = 0; curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); if (result.etag == request.expectedETag && httpStatus == 200) { - debug(format("shutting down on 200 HTTP response with expected ETag")); + debug("shutting down on 200 HTTP response with expected ETag"); return 0; } } else if (name == "content-encoding") diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 996f26a95..0aecc2d3b 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -34,8 +34,7 @@ static void makeSymlink(const Path & link, const Path & target) createDirs(dirOf(link)); /* Create the new symlink. */ - Path tempLink = (format("%1%.tmp-%2%-%3%") - % link % getpid() % random()).str(); + Path tempLink = fmt("%1%.tmp-%2%-%3%", link, getpid(), random()); createSymlink(target, tempLink); /* Atomically replace the old one. */ @@ -197,7 +196,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) pid_t pid = std::stoi(i.name); - debug(format("reading temporary root file '%1%'") % path); + debug("reading temporary root file '%1%'", path); AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)); if (!fd) { /* It's okay if the file has disappeared. */ @@ -263,7 +262,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) target = absPath(target, dirOf(path)); if (!pathExists(target)) { if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { - printInfo(format("removing stale link from '%1%' to '%2%'") % path % target); + printInfo("removing stale link from '%1%' to '%2%'", path, target); unlink(path.c_str()); } } else { @@ -863,7 +862,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) continue; } - printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + printMsg(lvlTalkative, "deleting unused link '%1%'", path); if (unlink(path.c_str()) == -1) throw SysError("deleting '%1%'", path); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 82edaa9bf..c9a466ee8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -280,7 +280,7 @@ LocalStore::LocalStore(const Params & params) else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; openDB(*state, true); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true); + writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); } else if (curSchema < nixSchemaVersion) { @@ -329,7 +329,7 @@ LocalStore::LocalStore(const Params & params) txn.commit(); } - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true); + writeFile(schemaPath, fmt("%1%", nixSchemaVersion), 0666, true); lockFile(globalLock.get(), ltRead, true); } @@ -1560,7 +1560,7 @@ void LocalStore::invalidatePathChecked(const StorePath & path) bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { - printInfo(format("reading the Nix store...")); + printInfo("reading the Nix store..."); bool errors = false; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 4d2781180..4a79cf4a1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -55,7 +55,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() } if (errno) throw SysError("reading directory '%1%'", linksDir); - printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); + printMsg(lvlTalkative, "loaded %1% hash inodes", inodeHash.size()); return inodeHash; } @@ -73,7 +73,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa checkInterrupt(); if (inodeHash.count(dirent->d_ino)) { - debug(format("'%1%' is already linked") % dirent->d_name); + debug("'%1%' is already linked", dirent->d_name); continue; } @@ -102,7 +102,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) { - debug(format("'%1%' is not allowed to be linked in macOS") % path); + debug("'%1%' is not allowed to be linked in macOS", path); return; } #endif @@ -146,7 +146,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); + debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true)); /* Check if this is a known hash. */ Path linkPath = linksDir + "/" + hash.to_string(Base32, false); @@ -196,11 +196,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, auto stLink = lstat(linkPath); if (st.st_ino == stLink.st_ino) { - debug(format("'%1%' is already linked to '%2%'") % path % linkPath); + debug("'%1%' is already linked to '%2%'", path, linkPath); return; } - printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); + printMsg(lvlTalkative, "linking '%1%' to '%2%'", path, linkPath); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its @@ -213,8 +213,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); - Path tempLink = (format("%1%/.tmp-link-%2%-%3%") - % realStoreDir % getpid() % random()).str(); + Path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), random()); if (link(linkPath.c_str(), tempLink.c_str()) == -1) { if (errno == EMLINK) { @@ -222,7 +221,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, systems). This is likely to happen with empty files. Just shrug and ignore. */ if (st.st_size) - printInfo(format("'%1%' has maximum number of links") % linkPath); + printInfo("'%1%' has maximum number of links", linkPath); return; } throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 42023cd0a..adc763e6a 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -96,7 +96,7 @@ bool PathLocks::lockPaths(const PathSet & paths, checkInterrupt(); Path lockPath = path + ".lock"; - debug(format("locking path '%1%'") % path); + debug("locking path '%1%'", path); AutoCloseFD fd; @@ -118,7 +118,7 @@ bool PathLocks::lockPaths(const PathSet & paths, } } - debug(format("lock acquired on '%1%'") % lockPath); + debug("lock acquired on '%1%'", lockPath); /* Check that the lock file hasn't become stale (i.e., hasn't been unlinked). */ @@ -130,7 +130,7 @@ bool PathLocks::lockPaths(const PathSet & paths, a lock on a deleted file. This means that other processes may create and acquire a lock on `lockPath', and proceed. So we must retry. */ - debug(format("open lock file '%1%' has become stale") % lockPath); + debug("open lock file '%1%' has become stale", lockPath); else break; } @@ -163,7 +163,7 @@ void PathLocks::unlock() "error (ignored): cannot close lock file on '%1%'", i.second); - debug(format("lock released on '%1%'") % i.second); + debug("lock released on '%1%'", i.second); } fds.clear(); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index c551c5f3e..179161ff7 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -64,7 +64,7 @@ std::pair> findGenerations(Path pro static void makeName(const Path & profile, GenerationNumber num, Path & outLink) { - Path prefix = (format("%1%-%2%") % profile % num).str(); + Path prefix = fmt("%1%-%2%", profile, num); outLink = prefix + "-link"; } @@ -269,7 +269,7 @@ void switchGeneration( void lockProfile(PathLocks & lock, const Path & profile) { - lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str()); + lock.lockPaths({profile}, fmt("waiting for lock on profile '%1%'", profile)); lock.setDeletion(true); } diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 3bb297fc8..345f4528b 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -39,8 +39,7 @@ static void search( if (!match) continue; std::string ref(s.substr(i, refLength)); if (hashes.erase(ref)) { - debug(format("found reference to '%1%' at offset '%2%'") - % ref % i); + debug("found reference to '%1%' at offset '%2%'", ref, i); seen.insert(ref); } ++i; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 8d76eee99..8006bd733 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -40,12 +40,12 @@ struct S3Error : public Error /* Helper: given an Outcome, return R in case of success, or throw an exception in case of an error. */ template -R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome && outcome) +R && checkAws(std::string_view s, Aws::Utils::Outcome && outcome) { if (!outcome.IsSuccess()) throw S3Error( outcome.GetError().GetErrorType(), - fs.s + ": " + outcome.GetError().GetMessage()); + s + ": " + outcome.GetError().GetMessage()); return outcome.GetResultWithOwnership(); } @@ -430,9 +430,9 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual std::string marker; do { - debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker); + debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker); - auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName, + auto res = checkAws(fmt("AWS error listing bucket '%s'", bucketName), s3Helper.client->ListObjects( Aws::S3::Model::ListObjectsRequest() .WithBucket(bucketName) @@ -441,8 +441,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto & contents = res.GetContents(); - debug(format("got %d keys, next marker '%s'") - % contents.size() % res.GetNextMarker()); + debug("got %d keys, next marker '%s'", + contents.size(), res.GetNextMarker()); for (auto object : contents) { auto & key = object.GetKey(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f32c2d30c..226eb9113 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -790,13 +790,13 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, if (showHash) { s += info->narHash.to_string(Base16, false) + "\n"; - s += (format("%1%\n") % info->narSize).str(); + s += fmt("%1%\n", info->narSize); } auto deriver = showDerivers && info->deriver ? printStorePath(*info->deriver) : ""; s += deriver + "\n"; - s += (format("%1%\n") % info->references.size()).str(); + s += fmt("%1%\n", info->references.size()); for (auto & j : info->references) s += printStorePath(j) + "\n"; -- cgit v1.2.3 From 7bfed34367f49ea2d4fe52e5861e5969fc9f4b59 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Mar 2023 13:51:02 +0100 Subject: Fix crash/hang with CA derivations The curl download can outlive DrvOutputSubstitutionGoal (if some other error occurs), so at shutdown setting the promise to an exception will fail because 'this' is no longer valid in the callback. This can manifest itself as a segfault, "corrupted double-linked list" or hang. --- src/libstore/build/drv-output-substitution-goal.cc | 23 +++++++++++++--------- src/libstore/build/drv-output-substitution-goal.hh | 12 +++++++---- 2 files changed, 22 insertions(+), 13 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index b7f7b5ab1..b30957c84 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -61,20 +61,25 @@ void DrvOutputSubstitutionGoal::tryNext() // FIXME: Make async // outputInfo = sub->queryRealisation(id); - outPipe.create(); - promise = decltype(promise)(); + + /* The callback of the curl download below can outlive `this` (if + some other error occurs), so it must not touch `this`. So put + the shared state in a separate refcounted object. */ + downloadState = std::make_shared(); + downloadState->outPipe.create(); sub->queryRealisation( - id, { [&](std::future> res) { + id, + { [downloadState(downloadState)](std::future> res) { try { - Finally updateStats([this]() { outPipe.writeSide.close(); }); - promise.set_value(res.get()); + Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); }); + downloadState->promise.set_value(res.get()); } catch (...) { - promise.set_exception(std::current_exception()); + downloadState->promise.set_exception(std::current_exception()); } } }); - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + worker.childStarted(shared_from_this(), {downloadState->outPipe.readSide.get()}, true, false); state = &DrvOutputSubstitutionGoal::realisationFetched; } @@ -84,7 +89,7 @@ void DrvOutputSubstitutionGoal::realisationFetched() worker.childTerminated(this); try { - outputInfo = promise.get_future().get(); + outputInfo = downloadState->promise.get_future().get(); } catch (std::exception & e) { printError(e.what()); substituterFailed = true; @@ -155,7 +160,7 @@ void DrvOutputSubstitutionGoal::work() void DrvOutputSubstitutionGoal::handleEOF(int fd) { - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); + if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 948dbda8f..e4b044790 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -16,7 +16,7 @@ class Worker; // 2. Substitute the corresponding output path // 3. Register the output info class DrvOutputSubstitutionGoal : public Goal { -private: + // The drv output we're trying to substitue DrvOutput id; @@ -30,9 +30,13 @@ private: /* The current substituter. */ std::shared_ptr sub; - Pipe outPipe; - std::thread thr; - std::promise> promise; + struct DownloadState + { + Pipe outPipe; + std::promise> promise; + }; + + std::shared_ptr downloadState; /* Whether a substituter failed. */ bool substituterFailed = false; -- cgit v1.2.3 From 4bef2016a18e64e378982db914e73d1a8eaa8855 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Thu, 9 Mar 2023 16:03:48 +0400 Subject: Display progress when running copyPaths (nix copy) `nix copy` operations did not show progress. This is quite confusing. Add a `progressSink` which displays the progress during `copyPaths`, pretty much copied from `copyStorePath`. Fixes https://github.com/NixOS/nix/issues/8000 --- src/libstore/store-api.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f32c2d30c..9ef641a3d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1101,6 +1101,8 @@ std::map copyPaths( return storePathForDst; }; + uint64_t total = 0; + for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); @@ -1121,7 +1123,13 @@ std::map copyPaths( {storePathS, srcUri, dstUri}); PushActivity pact(act.id); - srcStore.narFromPath(missingPath, sink); + LambdaSink progressSink([&](std::string_view data) { + total += data.size(); + act.progress(total, info->narSize); + }); + TeeSink tee { sink, progressSink }; + + srcStore.narFromPath(missingPath, tee); }); pathsToCopy.push_back(std::pair{infoForDst, std::move(source)}); } -- cgit v1.2.3 From 4f3a4b732cdc8a0179b0da9cccafef0b215a4e86 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Mar 2023 16:44:51 +0100 Subject: Make findRuntimeRoots() more resilient to disappearing processes I saw this random failure in https://hydra.nixos.org/build/211811692: error: opening /proc/15307/fd: No such process while running nix-collect-garbage in a readfile-context.sh. This is because we're not handling ESRCH errors reading /proc//fd. So just move the read inside the try/catch where we do handle it. --- src/libstore/gc.cc | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0aecc2d3b..0038ec802 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -371,29 +371,29 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { - readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); - readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); - - auto fdStr = fmt("/proc/%s/fd", ent->d_name); - auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); - if (!fdDir) { - if (errno == ENOENT || errno == EACCES) - continue; - throw SysError("opening %1%", fdStr); - } - struct dirent * fd_ent; - while (errno = 0, fd_ent = readdir(fdDir.get())) { - if (fd_ent->d_name[0] != '.') - readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); - } - if (errno) { - if (errno == ESRCH) - continue; - throw SysError("iterating /proc/%1%/fd", ent->d_name); - } - fdDir.reset(); - try { + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); + + auto fdStr = fmt("/proc/%s/fd", ent->d_name); + auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); + if (!fdDir) { + if (errno == ENOENT || errno == EACCES) + continue; + throw SysError("opening %1%", fdStr); + } + struct dirent * fd_ent; + while (errno = 0, fd_ent = readdir(fdDir.get())) { + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); + } + if (errno) { + if (errno == ESRCH) + continue; + throw SysError("iterating /proc/%1%/fd", ent->d_name); + } + fdDir.reset(); + auto mapFile = fmt("/proc/%s/maps", ent->d_name); auto mapLines = tokenizeString>(readFile(mapFile), "\n"); for (const auto & line : mapLines) { -- cgit v1.2.3 From 19326ac2979f0d989835105a5d816a943a6bc7f2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Mar 2023 18:58:12 +0100 Subject: Simplify commonChildInit() --- src/libstore/build/hook-instance.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index cb58a1f02..ea2ae210e 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -35,7 +35,7 @@ HookInstance::HookInstance() /* Fork the hook. */ pid = startProcess([&]() { - commonChildInit(fromHook); + commonChildInit(fromHook.writeSide.get()); if (chdir("/") == -1) throw SysError("changing into /"); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6fb9b86e0..61c1cbccf 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1649,7 +1649,7 @@ void LocalDerivationGoal::runChild() try { /* child */ - commonChildInit(builderOut); + commonChildInit(builderOut.writeSide.get()); try { setupSeccomp(); -- cgit v1.2.3 From c536e00c9deeac58bc4b3299dbc702604c32adbe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Mar 2023 19:10:59 +0100 Subject: Move pseudoterminal slave open to child Hopefully this fixes "unexpected EOF" failures on macOS (#3137, #3605, #7242, #7702). The problem appears to be that under some circumstances, macOS discards the output written to the slave side of the pseudoterminal. Hence the parent never sees the "sandbox initialized" message from the child, even though it succeeded. The conditions are: * The child finishes very quickly. That's why this bug is likely to trigger in nix-env tests, since that uses a builtin builder. Adding a short sleep before the child exits makes the problem go away. * The parent has closed its duplicate of the slave file descriptor. This shouldn't matter, since the child has a duplicate as well, but it does. E.g. moving the close to the bottom of startBuilder() makes the problem go away. However, that's not a solution because it would make Nix hang if the child dies before sending the "sandbox initialized" message. * The system is under high load. E.g. "make installcheck -j16" makes the issue pretty reproducible, while it's very rare under "make installcheck -j1". As a fix/workaround, we now open the pseudoterminal slave in the child, rather than the parent. This removes the second condition (i.e. the parent no longer needs to close the slave fd) and I haven't been able to reproduce the "unexpected EOF" with this. --- src/libstore/build/local-derivation-goal.cc | 58 ++++++++++++++--------------- src/libstore/build/local-derivation-goal.hh | 4 ++ 2 files changed, 31 insertions(+), 31 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 61c1cbccf..84a5a79b0 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -802,15 +802,13 @@ void LocalDerivationGoal::startBuilder() /* Create the log file. */ Path logFile = openLogFile(); - /* Create a pipe to get the output of the builder. */ - //builderOut.create(); - + /* Create a pseudoterminal to get the output of the builder. */ builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); if (!builderOut.readSide) throw SysError("opening pseudoterminal master"); // FIXME: not thread-safe, use ptsname_r - std::string slaveName(ptsname(builderOut.readSide.get())); + slaveName = ptsname(builderOut.readSide.get()); if (buildUser) { if (chmod(slaveName.c_str(), 0600)) @@ -826,30 +824,9 @@ void LocalDerivationGoal::startBuilder() } #endif - #if 0 - // Mount the pt in the sandbox so that the "tty" command works. - // FIXME: this doesn't work with the new devpts in the sandbox. - if (useChroot) - dirsInChroot[slaveName] = {slaveName, false}; - #endif - if (unlockpt(builderOut.readSide.get())) throw SysError("unlocking pseudoterminal"); - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - buildResult.startTime = time(0); /* Fork a child to build the package. */ @@ -897,7 +874,11 @@ void LocalDerivationGoal::startBuilder() usingUserNamespace = userNamespacesSupported(); + Pipe sendPid; + sendPid.create(); + Pid helper = startProcess([&]() { + sendPid.readSide.close(); /* Drop additional groups here because we can't do it after we've created the new user namespace. FIXME: @@ -919,11 +900,12 @@ void LocalDerivationGoal::startBuilder() pid_t child = startProcess([&]() { runChild(); }, options); - writeFull(builderOut.writeSide.get(), - fmt("%d %d\n", usingUserNamespace, child)); + writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); _exit(0); }); + sendPid.writeSide.close(); + if (helper.wait() != 0) throw Error("unable to start build process"); @@ -935,10 +917,9 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.writeSide = -1; }); - auto ss = tokenizeString>(readLine(builderOut.readSide.get())); - assert(ss.size() == 2); - usingUserNamespace = ss[0] == "1"; - pid = string2Int(ss[1]).value(); + auto ss = tokenizeString>(readLine(sendPid.readSide.get())); + assert(ss.size() == 1); + pid = string2Int(ss[0]).value(); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace @@ -1649,6 +1630,21 @@ void LocalDerivationGoal::runChild() try { /* child */ + /* Open the slave side of the pseudoterminal. */ + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); + commonChildInit(builderOut.writeSide.get()); try { diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 34c4e9187..c8537c7e5 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -27,6 +27,10 @@ struct LocalDerivationGoal : public DerivationGoal /* Pipe for the builder's standard output/error. */ Pipe builderOut; + /* Slave side of the pseudoterminal used for the builder's + standard output/error. */ + Path slaveName; + /* Pipe for synchronising updates to the builder namespaces. */ Pipe userNamespaceSync; -- cgit v1.2.3 From 6029c763c2c5998dc3265152425c8ff0ce01b1a0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Mar 2023 10:37:39 +0100 Subject: Change builderOut from Pipe to AutoCloseFD --- src/libstore/build/local-derivation-goal.cc | 31 ++++++++++++++--------------- src/libstore/build/local-derivation-goal.hh | 5 +++-- 2 files changed, 18 insertions(+), 18 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 84a5a79b0..f1ddf753f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -292,7 +292,7 @@ void LocalDerivationGoal::closeReadPipes() if (hook) { DerivationGoal::closeReadPipes(); } else - builderOut.readSide = -1; + builderOut.close(); } @@ -803,12 +803,12 @@ void LocalDerivationGoal::startBuilder() Path logFile = openLogFile(); /* Create a pseudoterminal to get the output of the builder. */ - builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); - if (!builderOut.readSide) + builderOut = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut) throw SysError("opening pseudoterminal master"); // FIXME: not thread-safe, use ptsname_r - slaveName = ptsname(builderOut.readSide.get()); + slaveName = ptsname(builderOut.get()); if (buildUser) { if (chmod(slaveName.c_str(), 0600)) @@ -819,12 +819,12 @@ void LocalDerivationGoal::startBuilder() } #if __APPLE__ else { - if (grantpt(builderOut.readSide.get())) + if (grantpt(builderOut.get())) throw SysError("granting access to pseudoterminal slave"); } #endif - if (unlockpt(builderOut.readSide.get())) + if (unlockpt(builderOut.get())) throw SysError("unlocking pseudoterminal"); buildResult.startTime = time(0); @@ -980,15 +980,14 @@ void LocalDerivationGoal::startBuilder() /* parent */ pid.setSeparatePG(true); - builderOut.writeSide = -1; - worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); + worker.childStarted(shared_from_this(), {builderOut.get()}, true, true); /* Check if setting up the build environment failed. */ std::vector msgs; while (true) { std::string msg = [&]() { try { - return readLine(builderOut.readSide.get()); + return readLine(builderOut.get()); } catch (Error & e) { auto status = pid.wait(); e.addTrace({}, "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", @@ -1000,7 +999,7 @@ void LocalDerivationGoal::startBuilder() }(); if (msg.substr(0, 1) == "\2") break; if (msg.substr(0, 1) == "\1") { - FdSource source(builderOut.readSide.get()); + FdSource source(builderOut.get()); auto ex = readError(source); ex.addTrace({}, "while setting up the build environment"); throw ex; @@ -1631,21 +1630,21 @@ void LocalDerivationGoal::runChild() try { /* child */ /* Open the slave side of the pseudoterminal. */ - builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut.writeSide) + AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut) throw SysError("opening pseudoterminal slave"); // Put the pt into raw mode to prevent \n -> \r\n translation. struct termios term; - if (tcgetattr(builderOut.writeSide.get(), &term)) + if (tcgetattr(builderOut.get(), &term)) throw SysError("getting pseudoterminal attributes"); cfmakeraw(&term); - if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + if (tcsetattr(builderOut.get(), TCSANOW, &term)) throw SysError("putting pseudoterminal into raw mode"); - commonChildInit(builderOut.writeSide.get()); + commonChildInit(builderOut.get()); try { setupSeccomp(); @@ -2887,7 +2886,7 @@ void LocalDerivationGoal::deleteTmpDir(bool force) bool LocalDerivationGoal::isReadDesc(int fd) { return (hook && DerivationGoal::isReadDesc(fd)) || - (!hook && fd == builderOut.readSide.get()); + (!hook && fd == builderOut.get()); } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index c8537c7e5..c3e8562ae 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -24,8 +24,9 @@ struct LocalDerivationGoal : public DerivationGoal /* The path of the temporary directory in the sandbox. */ Path tmpDirInSandbox; - /* Pipe for the builder's standard output/error. */ - Pipe builderOut; + /* Master side of the pseudoterminal used for the builder's + standard output/error. */ + AutoCloseFD builderOut; /* Slave side of the pseudoterminal used for the builder's standard output/error. */ -- cgit v1.2.3 From 5eb8bfd0f17f950ec181d59fb9fbb2330f778935 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Mar 2023 10:41:38 +0100 Subject: Remove a variable in LocalDerivationGoal --- src/libstore/build/local-derivation-goal.cc | 8 ++++---- src/libstore/build/local-derivation-goal.hh | 6 +----- 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index f1ddf753f..538edef74 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -808,7 +808,7 @@ void LocalDerivationGoal::startBuilder() throw SysError("opening pseudoterminal master"); // FIXME: not thread-safe, use ptsname_r - slaveName = ptsname(builderOut.get()); + std::string slaveName = ptsname(builderOut.get()); if (buildUser) { if (chmod(slaveName.c_str(), 0600)) @@ -898,7 +898,7 @@ void LocalDerivationGoal::startBuilder() if (usingUserNamespace) options.cloneFlags |= CLONE_NEWUSER; - pid_t child = startProcess([&]() { runChild(); }, options); + pid_t child = startProcess([&]() { runChild(slaveName); }, options); writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); _exit(0); @@ -974,7 +974,7 @@ void LocalDerivationGoal::startBuilder() #endif { pid = startProcess([&]() { - runChild(); + runChild(slaveName); }); } @@ -1620,7 +1620,7 @@ void setupSeccomp() } -void LocalDerivationGoal::runChild() +void LocalDerivationGoal::runChild(const Path & slaveName) { /* Warning: in the child we should absolutely not make any SQLite calls! */ diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index c3e8562ae..4d2f1ac28 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -28,10 +28,6 @@ struct LocalDerivationGoal : public DerivationGoal standard output/error. */ AutoCloseFD builderOut; - /* Slave side of the pseudoterminal used for the builder's - standard output/error. */ - Path slaveName; - /* Pipe for synchronising updates to the builder namespaces. */ Pipe userNamespaceSync; @@ -173,7 +169,7 @@ struct LocalDerivationGoal : public DerivationGoal int getChildStatus() override; /* Run the builder's process. */ - void runChild(); + void runChild(const std::string & slaveName); /* Check that the derivation outputs all exist and register them as valid. */ -- cgit v1.2.3 From af4cbdafe782ef3064156b33315469c27c36fcff Mon Sep 17 00:00:00 2001 From: Linus Heckemann Date: Thu, 16 Mar 2023 15:21:02 +0100 Subject: LocalDerivationGoal: set NIX_ATTRS_*_FILE correctly for sandboxed builds --- src/libstore/build/local-derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6fb9b86e0..13a977bd1 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1154,10 +1154,10 @@ void LocalDerivationGoal::writeStructuredAttrs() writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); chownToBuilder(tmpDir + "/.attrs.sh"); - env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh"; + env["NIX_ATTRS_SH_FILE"] = tmpDirInSandbox + "/.attrs.sh"; writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); chownToBuilder(tmpDir + "/.attrs.json"); - env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json"; + env["NIX_ATTRS_JSON_FILE"] = tmpDirInSandbox + "/.attrs.json"; } } -- cgit v1.2.3 From e53e5c38d4a6f89dbb7e33448977a95e5b4ab2eb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Mar 2023 18:32:18 +0100 Subject: Add a setting for configuring the SSL certificates file This provides a platform-independent way to configure the SSL certificates file in the Nix daemon. Previously we provided instructions for overriding the environment variable in launchd, but that obviously doesn't work with systemd. Now we can just tell users to add ssl-cert-file = /etc/ssl/my-certificate-bundle.crt to their nix.conf. --- src/libstore/filetransfer.cc | 2 +- src/libstore/globals.cc | 18 ++++++++++-------- src/libstore/globals.hh | 15 +++++++++++++-- 3 files changed, 24 insertions(+), 11 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index b5fe7c03b..1ba399a29 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -318,7 +318,7 @@ struct curlFileTransfer : public FileTransfer if (request.verifyTLS) { if (settings.caFile != "") - curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); + curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.get().c_str()); } else { curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index fae79c1a0..7be5c89b7 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -44,14 +44,9 @@ Settings::Settings() lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; - caFile = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); - if (caFile == "") { - for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) - if (pathExists(fn)) { - caFile = fn; - break; - } - } + auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); + if (sslOverride != "") + caFile = sslOverride; /* Backwards compatibility. */ auto s = getEnv("NIX_REMOTE_SYSTEMS"); @@ -187,6 +182,13 @@ bool Settings::isWSL1() return hasSuffix(utsbuf.release, "-Microsoft"); } +Path Settings::getDefaultSSLCertFile() +{ + for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) + if (pathExists(fn)) return fn; + return ""; +} + const std::string nixVersion = PACKAGE_VERSION; NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 93086eaf8..95f6b5e35 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -64,6 +64,8 @@ class Settings : public Config { bool isWSL1(); + Path getDefaultSSLCertFile(); + public: Settings(); @@ -826,8 +828,17 @@ public: > `.netrc`. )"}; - /* Path to the SSL CA file used */ - Path caFile; + Setting caFile{ + this, getDefaultSSLCertFile(), "ssl-cert-file", + R"( + The path of a file containing CA certificates used to + authenticate `https://` downloads. It defaults to the first + of `/etc/ssl/certs/ca-certificates.crt` and + `/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt` + that exists. It can be overriden using the + `NIX_SSL_CERT_FILE` and `SSL_CERT_FILE` environment variable + (in that order of precedence). + )"}; #if __linux__ Setting filterSyscalls{ -- cgit v1.2.3 From 296831f641b2ce43f179ec710c3ef76726ef96e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 17 Mar 2023 10:33:48 -0400 Subject: Move enabled experimental feature to libutil struct This is needed in subsequent commits to allow the settings and CLI args infrastructure itself to read this setting. --- src/libstore/build/derivation-goal.cc | 12 ++++++------ src/libstore/build/local-derivation-goal.cc | 8 ++++---- src/libstore/daemon.cc | 4 ++-- src/libstore/derivations.cc | 4 ++-- src/libstore/derived-path.cc | 2 +- src/libstore/globals.cc | 12 ------------ src/libstore/globals.hh | 8 -------- src/libstore/local-store.cc | 10 +++++----- src/libstore/lock.cc | 2 +- src/libstore/misc.cc | 2 +- src/libstore/remote-store.cc | 4 ++-- src/libstore/store-api.cc | 8 ++++---- 12 files changed, 28 insertions(+), 48 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 38b73d531..596034c0f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -199,10 +199,10 @@ void DerivationGoal::haveDerivation() parsedDrv = std::make_unique(drvPath, *drv); if (!drv->type().hasKnownOutputPaths()) - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); if (!drv->type().isPure()) { - settings.requireExperimentalFeature(Xp::ImpureDerivations); + experimentalFeatureSettings.require(Xp::ImpureDerivations); for (auto & [outputName, output] : drv->outputs) { auto randomPath = StorePath::random(outputPathName(drv->name, outputName)); @@ -336,7 +336,7 @@ void DerivationGoal::gaveUpOnSubstitution() for (auto & i : dynamic_cast(drv.get())->inputDrvs) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ - if (settings.isExperimentalFeatureEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { + if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { auto inputDrv = worker.evalStore.readDerivation(i.first); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", @@ -477,7 +477,7 @@ void DerivationGoal::inputsRealised() ca.fixed /* Can optionally resolve if fixed, which is good for avoiding unnecessary rebuilds. */ - ? settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + ? experimentalFeatureSettings.isEnabled(Xp::CaDerivations) /* Must resolve if floating and there are any inputs drvs. */ : true); @@ -488,7 +488,7 @@ void DerivationGoal::inputsRealised() }, drvType.raw()); if (resolveDrv && !fullDrv.inputDrvs.empty()) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a @@ -1352,7 +1352,7 @@ std::pair DerivationGoal::checkPathValidity() }; } auto drvOutput = DrvOutput{info.outputHash, i.first}; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { .path = real->outPath, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 521117c68..923530d08 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -413,7 +413,7 @@ void LocalDerivationGoal::startBuilder() ) { #if __linux__ - settings.requireExperimentalFeature(Xp::Cgroups); + experimentalFeatureSettings.require(Xp::Cgroups); auto cgroupFS = getCgroupFS(); if (!cgroupFS) @@ -1393,7 +1393,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void LocalDerivationGoal::startDaemon() { - settings.requireExperimentalFeature(Xp::RecursiveNix); + experimentalFeatureSettings.require(Xp::RecursiveNix); Store::Params params; params["path-info-cache-size"] = "0"; @@ -2268,7 +2268,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { - settings.requireExperimentalFeature(Xp::DiscardReferences); + experimentalFeatureSettings.require(Xp::DiscardReferences); if (auto output = get(*udr, outputName)) { if (!output->is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); @@ -2688,7 +2688,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() }, .outPath = newInfo.path }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && drv->type().isPure()) { signRealisation(thisRealisation); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 7f8b0f905..656ad4587 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -231,10 +231,10 @@ struct ClientSettings try { if (name == "ssh-auth-sock") // obsolete ; - else if (name == settings.experimentalFeatures.name) { + else if (name == experimentalFeatureSettings.experimentalFeatures.name) { // We don’t want to forward the experimental features to // the daemon, as that could cause some pretty weird stuff - if (parseFeatures(tokenizeString(value)) != settings.experimentalFeatures.get()) + if (parseFeatures(tokenizeString(value)) != experimentalFeatureSettings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); } else if (name == settings.pluginFiles.name) { if (tokenizeString(value) != settings.pluginFiles.get()) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 05dc9a3cc..06cc69056 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -221,7 +221,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, } const auto hashType = parseHashType(hashAlgo); if (hash == "impure") { - settings.requireExperimentalFeature(Xp::ImpureDerivations); + experimentalFeatureSettings.require(Xp::ImpureDerivations); assert(pathS == ""); return DerivationOutput::Impure { .method = std::move(method), @@ -236,7 +236,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, }, }; } else { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); assert(pathS == ""); return DerivationOutput::CAFloating { .method = std::move(method), diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index e0d86a42f..e5f0f1b33 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -105,7 +105,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const auto drvHashes = staticOutputHashes(store, store.readDerivation(p.drvPath)); for (auto& [outputName, outputPath] : p.outputs) { - if (settings.isExperimentalFeatureEnabled( + if (experimentalFeatureSettings.isEnabled( Xp::CaDerivations)) { auto drvOutput = get(drvHashes, outputName); if (!drvOutput) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index fae79c1a0..8781e10ea 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -166,18 +166,6 @@ StringSet Settings::getDefaultExtraPlatforms() return extraPlatforms; } -bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature) -{ - auto & f = experimentalFeatures.get(); - return std::find(f.begin(), f.end(), feature) != f.end(); -} - -void Settings::requireExperimentalFeature(const ExperimentalFeature & feature) -{ - if (!isExperimentalFeatureEnabled(feature)) - throw MissingExperimentalFeature(feature); -} - bool Settings::isWSL1() { struct utsname utsbuf; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 93086eaf8..db01ab657 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "config.hh" #include "util.hh" -#include "experimental-features.hh" #include #include @@ -932,13 +931,6 @@ public: are loaded as plugins (non-recursively). )"}; - Setting> experimentalFeatures{this, {}, "experimental-features", - "Experimental Nix features to enable."}; - - bool isExperimentalFeatureEnabled(const ExperimentalFeature &); - - void requireExperimentalFeature(const ExperimentalFeature &); - Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c9a466ee8..7782f7b50 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -336,7 +336,7 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } @@ -366,7 +366,7 @@ LocalStore::LocalStore(const Params & params) state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { state->stmts->RegisterRealisedOutput.create(state->db, R"( insert into Realisations (drvPath, outputName, outputPath, signatures) @@ -754,7 +754,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) registerDrvOutput(info); else @@ -763,7 +763,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check void LocalStore::registerDrvOutput(const Realisation & info) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); retrySQLite([&]() { auto state(_state.lock()); if (auto oldR = queryRealisation_(*state, info.id)) { @@ -1052,7 +1052,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) return outputs; }); - if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) return outputs; auto drv = readInvalidDerivation(path); diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 4fe1fcf56..7202a64b3 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -129,7 +129,7 @@ struct AutoUserLock : UserLock useUserNamespace = false; #endif - settings.requireExperimentalFeature(Xp::AutoAllocateUids); + experimentalFeatureSettings.require(Xp::AutoAllocateUids); assert(settings.startId > 0); assert(settings.uidCount % maxIdsPerBuild == 0); assert((uint64_t) settings.startId + (uint64_t) settings.uidCount <= std::numeric_limits::max()); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index b28768459..89148d415 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -326,7 +326,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, throw Error( "the derivation '%s' doesn't have an output named '%s'", store.printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { DrvOutput outputId { *outputHash, output }; auto realisation = store.queryRealisation(outputId); if (!realisation) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d1296627a..d24d83117 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -265,7 +265,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.buildCores.name); overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); - overrides.erase(settings.experimentalFeatures.name); + overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); overrides.erase(settings.pluginFiles.name); conn.to << overrides.size(); for (auto & i : overrides) @@ -876,7 +876,7 @@ std::vector RemoteStore::buildPathsWithResults( "the derivation '%s' doesn't have an output named '%s'", printStorePath(bfd.drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = queryRealisation(outputId); if (!realisation) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 19b0a7f5f..b0ca1321c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -445,10 +445,10 @@ StringSet StoreConfig::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) res.insert("ca-derivations"); - if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix)) + if (experimentalFeatureSettings.isEnabled(Xp::RecursiveNix)) res.insert("recursive-nix"); return res; @@ -1017,7 +1017,7 @@ std::map copyPaths( for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if(&path.raw)) { - settings.requireExperimentalFeature(Xp::CaDerivations); + experimentalFeatureSettings.require(Xp::CaDerivations); toplevelRealisations.insert(*realisation); } } @@ -1250,7 +1250,7 @@ std::optional Store::getBuildDerivationPath(const StorePath & path) } } - if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations) || !isValidPath(path)) + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations) || !isValidPath(path)) return path; auto drv = readDerivation(path); -- cgit v1.2.3 From 16db8dc96f64a0facbb620907e571f2dfc8e802e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Mar 2023 17:04:57 +0100 Subject: Open slave pseudoterminal before CLONE_NEWUSER Otherwise, when running as root and user namespaces are enabled, opening the slave fails with EPERM. Fixes "opening pseudoterminal slave: Permission denied" followed by a hang (https://hydra.nixos.org/build/213104244), and "error: getting sandbox mount namespace: No such file or directory" (#8072), which happens when the child fails very quickly and consequently reading /proc//ns fails. --- src/libstore/build/local-derivation-goal.cc | 50 ++++++++++++++++++----------- src/libstore/build/local-derivation-goal.hh | 2 +- 2 files changed, 32 insertions(+), 20 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 521117c68..4f21862ca 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -827,6 +827,27 @@ void LocalDerivationGoal::startBuilder() if (unlockpt(builderOut.get())) throw SysError("unlocking pseudoterminal"); + /* Open the slave side of the pseudoterminal and use it as stderr. */ + auto openSlave = [&]() + { + AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); + + if (dup2(builderOut.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + }; + buildResult.startTime = time(0); /* Fork a child to build the package. */ @@ -880,6 +901,11 @@ void LocalDerivationGoal::startBuilder() Pid helper = startProcess([&]() { sendPid.readSide.close(); + /* We need to open the slave early, before + CLONE_NEWUSER. Otherwise we get EPERM when running as + root. */ + openSlave(); + /* Drop additional groups here because we can't do it after we've created the new user namespace. FIXME: this means that if we're not root in the parent @@ -898,7 +924,7 @@ void LocalDerivationGoal::startBuilder() if (usingUserNamespace) options.cloneFlags |= CLONE_NEWUSER; - pid_t child = startProcess([&]() { runChild(slaveName); }, options); + pid_t child = startProcess([&]() { runChild(); }, options); writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); _exit(0); @@ -974,7 +1000,8 @@ void LocalDerivationGoal::startBuilder() #endif { pid = startProcess([&]() { - runChild(slaveName); + openSlave(); + runChild(); }); } @@ -1620,7 +1647,7 @@ void setupSeccomp() } -void LocalDerivationGoal::runChild(const Path & slaveName) +void LocalDerivationGoal::runChild() { /* Warning: in the child we should absolutely not make any SQLite calls! */ @@ -1629,22 +1656,7 @@ void LocalDerivationGoal::runChild(const Path & slaveName) try { /* child */ - /* Open the slave side of the pseudoterminal. */ - AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); - if (!builderOut) - throw SysError("opening pseudoterminal slave"); - - // Put the pt into raw mode to prevent \n -> \r\n translation. - struct termios term; - if (tcgetattr(builderOut.get(), &term)) - throw SysError("getting pseudoterminal attributes"); - - cfmakeraw(&term); - - if (tcsetattr(builderOut.get(), TCSANOW, &term)) - throw SysError("putting pseudoterminal into raw mode"); - - commonChildInit(builderOut.get()); + commonChildInit(-1); try { setupSeccomp(); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 4d2f1ac28..c9ecc8828 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -169,7 +169,7 @@ struct LocalDerivationGoal : public DerivationGoal int getChildStatus() override; /* Run the builder's process. */ - void runChild(const std::string & slaveName); + void runChild(); /* Check that the derivation outputs all exist and register them as valid. */ -- cgit v1.2.3 From 515662ad703cbd7c34df0020947392d233ac82eb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 20 Mar 2023 18:06:08 +0100 Subject: Cleanup --- src/libstore/build/hook-instance.cc | 5 ++++- src/libstore/build/local-derivation-goal.cc | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index ea2ae210e..075ad554f 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -35,7 +35,10 @@ HookInstance::HookInstance() /* Fork the hook. */ pid = startProcess([&]() { - commonChildInit(fromHook.writeSide.get()); + if (dup2(fromHook.writeSide.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + commonChildInit(); if (chdir("/") == -1) throw SysError("changing into /"); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4f21862ca..3484d2044 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1656,7 +1656,7 @@ void LocalDerivationGoal::runChild() try { /* child */ - commonChildInit(-1); + commonChildInit(); try { setupSeccomp(); -- cgit v1.2.3 From 8d6d59cb1ba0a2cfe12f9f444a27833dc531c191 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Mar 2023 12:58:14 +0100 Subject: nix store --help: Include store type documentation --- src/libstore/http-binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 1479822a9..93f25363a 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -12,7 +12,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const std::string name() override { return "Http Binary Cache Store"; } + const std::string name() override { return "HTTP Binary Cache Store"; } }; class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore -- cgit v1.2.3 From 9eb53bbf17037691e1278c71517d74d0962aa43f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Mar 2023 14:03:05 +0100 Subject: Support per-store Markdown documentation --- src/libstore/dummy-store.cc | 7 ++++++ src/libstore/dummy-store.md | 13 +++++++++++ src/libstore/http-binary-cache-store.cc | 7 ++++++ src/libstore/http-binary-cache-store.md | 8 +++++++ src/libstore/legacy-ssh-store.cc | 9 ++++++- src/libstore/legacy-ssh-store.md | 8 +++++++ src/libstore/local-binary-cache-store.cc | 7 ++++++ src/libstore/local-binary-cache-store.md | 16 +++++++++++++ src/libstore/local-store.cc | 7 ++++++ src/libstore/local-store.hh | 3 ++- src/libstore/local-store.md | 40 ++++++++++++++++++++++++++++++++ src/libstore/s3-binary-cache-store.cc | 7 ++++++ src/libstore/s3-binary-cache-store.md | 8 +++++++ src/libstore/ssh-store.cc | 9 ++++++- src/libstore/ssh-store.md | 8 +++++++ src/libstore/store-api.hh | 5 ++++ src/libstore/uds-remote-store.hh | 7 ++++++ src/libstore/uds-remote-store.md | 9 +++++++ 18 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 src/libstore/dummy-store.md create mode 100644 src/libstore/http-binary-cache-store.md create mode 100644 src/libstore/legacy-ssh-store.md create mode 100644 src/libstore/local-binary-cache-store.md create mode 100644 src/libstore/local-store.md create mode 100644 src/libstore/s3-binary-cache-store.md create mode 100644 src/libstore/ssh-store.md create mode 100644 src/libstore/uds-remote-store.md (limited to 'src/libstore') diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index b4fbe0b70..16e5fafd7 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -7,6 +7,13 @@ struct DummyStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; const std::string name() override { return "Dummy Store"; } + + std::string doc() override + { + return + #include "dummy-store.md" + ; + } }; struct DummyStore : public virtual DummyStoreConfig, public virtual Store diff --git a/src/libstore/dummy-store.md b/src/libstore/dummy-store.md new file mode 100644 index 000000000..eb7b4ba0d --- /dev/null +++ b/src/libstore/dummy-store.md @@ -0,0 +1,13 @@ +R"( + +**Store URL format**: `dummy://` + +This store type represents a store that contains no store paths and +cannot be written to. It's useful when you want to use the Nix +evaluator when no actual Nix store exists, e.g. + +```console +# nix eval --store dummy:// --expr '1 + 2' +``` + +)" diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 93f25363a..238fd1d98 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -13,6 +13,13 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig using BinaryCacheStoreConfig::BinaryCacheStoreConfig; const std::string name() override { return "HTTP Binary Cache Store"; } + + std::string doc() override + { + return + #include "http-binary-cache-store.md" + ; + } }; class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore diff --git a/src/libstore/http-binary-cache-store.md b/src/libstore/http-binary-cache-store.md new file mode 100644 index 000000000..20c26d0c2 --- /dev/null +++ b/src/libstore/http-binary-cache-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `http://...`, `https://...` + +This store allows a binary cache to be accessed via the HTTP +protocol. + +)" diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 2c9dd2680..7ce3dac6b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -22,7 +22,14 @@ struct LegacySSHStoreConfig : virtual StoreConfig const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; - const std::string name() override { return "Legacy SSH Store"; } + const std::string name() override { return "SSH Store"; } + + std::string doc() override + { + return + #include "legacy-ssh-store.md" + ; + } }; struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store diff --git a/src/libstore/legacy-ssh-store.md b/src/libstore/legacy-ssh-store.md new file mode 100644 index 000000000..043acebd6 --- /dev/null +++ b/src/libstore/legacy-ssh-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `ssh://[username@]hostname` + +This store type allows limited access to a remote store on another +machine via SSH. + +)" diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index f20b1fa02..e5ee6fc15 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -11,6 +11,13 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig using BinaryCacheStoreConfig::BinaryCacheStoreConfig; const std::string name() override { return "Local Binary Cache Store"; } + + std::string doc() override + { + return + #include "local-binary-cache-store.md" + ; + } }; class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore diff --git a/src/libstore/local-binary-cache-store.md b/src/libstore/local-binary-cache-store.md new file mode 100644 index 000000000..93fddc840 --- /dev/null +++ b/src/libstore/local-binary-cache-store.md @@ -0,0 +1,16 @@ +R"( + +**Store URL format**: `file://`*path* + +This store allows reading and writing a binary cache stored in *path* +in the local filesystem. If *path* does not exist, it will be created. + +For example, the following builds or downloads `nixpkgs#hello` into +the local store and then copies it to the binary cache in +`/tmp/binary-cache`: + +``` +# nix copy --to file:///tmp/binary-cache nixpkgs#hello +``` + +)" diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c9a466ee8..1eada7cc3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -44,6 +44,13 @@ namespace nix { +std::string LocalStoreConfig::doc() +{ + return + #include "local-store.md" + ; +} + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ SQLiteStmt RegisterValidPath; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a84eb7c26..5e5ce86c8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -41,8 +41,9 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig "require-sigs", "whether store paths should have a trusted signature on import"}; const std::string name() override { return "Local Store"; } -}; + std::string doc() override; +}; class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore { diff --git a/src/libstore/local-store.md b/src/libstore/local-store.md new file mode 100644 index 000000000..893457d32 --- /dev/null +++ b/src/libstore/local-store.md @@ -0,0 +1,40 @@ +R"( + +**Store URL format**: `local`, *root* + +This store type accesses a Nix store in the local filesystem directly +(i.e. not via the Nix daemon). *root* is an absolute path that denotes +the "root" of the filesystem; other directories such as the Nix store +directory (as denoted by the `store` setting) are interpreted relative +to *root*. The store pseudo-URL `local` denotes a store that uses `/` +as its root directory. + +A store that uses a *root* other than `/` is called a *chroot +store*. With such stores, the store directory is "logically" still +`/nix/store`, so programs stored in them can only be built and +executed by `chroot`-ing into *root*. Chroot stores only support +building and running on Linux when mount and user namespaces are +enabled. + +For example, the following uses `/tmp/root` as the chroot environment +to build or download `nixpkgs#hello` and then execute it: + +```console +# nix run --store /tmp/root nixpkgs#hello +Hello, world! +``` + +Here, the "physical" store location is `/tmp/root/nix/store`, and +Nix's store metadata is in `/tmp/root/nix/var/nix/db`. + +It is also possible, but not recommended, to change the "logical" +location of the Nix store from its default of `/nix/store`. This makes +it impossible to use default substituters such as +`https://cache.nixos.org/`, and thus you may have to build everything +locally. Here is an example: + +```console +# nix build --store 'local?store=/tmp/my-nix/store&state=/tmp/my-nix/state&log=/tmp/my-nix/log' nixpkgs#hello +``` + +)" diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 8006bd733..05842a2ea 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -205,6 +205,13 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; const std::string name() override { return "S3 Binary Cache Store"; } + + std::string doc() override + { + return + #include "s3-binary-cache-store.md" + ; + } }; struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore diff --git a/src/libstore/s3-binary-cache-store.md b/src/libstore/s3-binary-cache-store.md new file mode 100644 index 000000000..70fe0eb09 --- /dev/null +++ b/src/libstore/s3-binary-cache-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `s3://`*bucket-name* + +This store allows reading and writing a binary cache stored in an AWS +S3 bucket. + +)" diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index cfa063803..6f3ee6262 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -18,7 +18,14 @@ struct SSHStoreConfig : virtual RemoteStoreConfig const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; - const std::string name() override { return "SSH Store"; } + const std::string name() override { return "Experimental SSH Store"; } + + std::string doc() override + { + return + #include "ssh-store.md" + ; + } }; class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md new file mode 100644 index 000000000..881537e71 --- /dev/null +++ b/src/libstore/ssh-store.md @@ -0,0 +1,8 @@ +R"( + +**Store URL format**: `ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a +remote machine. + +)" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4d8db3596..9afb33b9e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -101,6 +101,11 @@ struct StoreConfig : public Config virtual const std::string name() = 0; + virtual std::string doc() + { + return ""; + } + const PathSetting storeDir_{this, false, settings.nixStore, "store", "path to the Nix store"}; const Path storeDir = storeDir_; diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index d31a4d592..caa452919 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -15,6 +15,13 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon } const std::string name() override { return "Local Daemon Store"; } + + std::string doc() override + { + return + #include "uds-remote-store.md" + ; + } }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore diff --git a/src/libstore/uds-remote-store.md b/src/libstore/uds-remote-store.md new file mode 100644 index 000000000..8df0bd6ff --- /dev/null +++ b/src/libstore/uds-remote-store.md @@ -0,0 +1,9 @@ +R"( + +**Store URL format**: `daemon`, `unix://`*path* + +This store type accesses a Nix store by talking to a Nix daemon +listening on the Unix domain socket *path*. The store pseudo-URL +`daemon` is equivalent to `unix:///nix/var/nix/daemon-socket/socket`. + +)" -- cgit v1.2.3 From 233b063b08b6a82921d26fb5a86e15c2b94a72ee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Mar 2023 14:37:09 +0100 Subject: Move store docs to 'nix help-stores' Why not 'nix help stores'? Well, 'nix help ' already means 'show help on the "arg" subcommand'. --- src/libstore/globals.hh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 93086eaf8..4382d1b85 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -97,7 +97,10 @@ public: Path nixDaemonSocketFile; Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", - "The default Nix store to use."}; + R"( + The URL of the Nix store to use. See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) + for supported store types and settings. + )"}; Setting keepFailed{this, false, "keep-failed", "Whether to keep temporary directories of failed builds."}; -- cgit v1.2.3 From 5691bac2025db9dbf637f23d1d2dd502010df37b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Mar 2023 14:23:36 +0100 Subject: Improve store setting descriptions / Markdown formatting --- src/libstore/binary-cache-store.hh | 34 ++++++++++++++----- src/libstore/legacy-ssh-store.cc | 14 ++++---- src/libstore/local-fs-store.hh | 16 ++++++--- src/libstore/local-store.hh | 3 +- src/libstore/remote-store.hh | 10 +++--- src/libstore/s3-binary-cache-store.cc | 64 ++++++++++++++++++++++++++++++----- src/libstore/ssh-store-config.hh | 29 ++++++++++++++++ src/libstore/ssh-store.cc | 9 ++--- src/libstore/store-api.hh | 31 +++++++++++++---- 9 files changed, 161 insertions(+), 49 deletions(-) create mode 100644 src/libstore/ssh-store-config.hh (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index abd92a83c..c1d08926d 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -16,17 +16,33 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"}; - const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; - const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; - const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; - const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting compression{(StoreConfig*) this, "xz", "compression", + "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; + + const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", + "Whether to write a JSON file that lists the files in each NAR."}; + + const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", + R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )"}; + + const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", + "Path to the secret key used to sign the binary cache."}; + + const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", + "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; + const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", - "enable multi-threading compression for NARs, available for xz and zstd only currently"}; + "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + const Setting compressionLevel{(StoreConfig*) this, -1, "compression-level", - "specify 'preset level' of compression to be used with NARs: " - "meaning and accepted range of values depends on compression method selected, " - "other than -1 which we reserve to indicate Nix defaults should be used"}; + R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )"}; }; class BinaryCacheStore : public virtual BinaryCacheStoreConfig, diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 7ce3dac6b..a8ce0b8b0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,3 +1,4 @@ +#include "ssh-store-config.hh" #include "archive.hh" #include "pool.hh" #include "remote-store.hh" @@ -12,15 +13,12 @@ namespace nix { -struct LegacySSHStoreConfig : virtual StoreConfig +struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { - using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; - const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; - const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + using CommonSSHStoreConfig::CommonSSHStoreConfig; + + const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + "Maximum number of concurrent SSH connections."}; const std::string name() override { return "SSH Store"; } diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 947707341..796e72045 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -9,20 +9,28 @@ namespace nix { struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes // it to omit the call to the Setting constructor. Clang works fine // either way. + const PathSetting rootDir{(StoreConfig*) this, true, "", - "root", "directory prefixed to all other paths"}; + "root", + "Directory prefixed to all other paths."}; + const PathSetting stateDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, - "state", "directory where Nix will store state"}; + "state", + "Directory where Nix will store state."}; + const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, - "log", "directory where Nix will store state"}; + "log", + "directory where Nix will store log files."}; + const PathSetting realStoreDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", - "physical path to the Nix store"}; + "Physical path of the Nix store."}; }; class LocalFSStore : public virtual LocalFSStoreConfig, diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 5e5ce86c8..3743f6b10 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -38,7 +38,8 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig Setting requireSigs{(StoreConfig*) this, settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; + "require-sigs", + "Whether store paths copied into this store should have a trusted signature."}; const std::string name() override { return "Local Store"; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 8cd7cc822..999151239 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -22,11 +22,13 @@ struct RemoteStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, - "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{(StoreConfig*) this, std::numeric_limits::max(), - "max-connection-age", "number of seconds to reuse a connection"}; + const Setting maxConnectionAge{(StoreConfig*) this, + std::numeric_limits::max(), + "max-connection-age", + "Maximum age of a connection before it is closed."}; }; /* FIXME: RemoteStore is a misnomer - should be something like diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 05842a2ea..ac82147ee 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -192,17 +192,63 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const Setting profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."}; - const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; - const Setting scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."}; - const Setting endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; - const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"}; - const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"}; - const Setting logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"}; + + const Setting profile{(StoreConfig*) this, "", "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + + const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting scheme{(StoreConfig*) this, "", "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting endpoint{(StoreConfig*) this, "", "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", + "Compression method for `.narinfo` files."}; + + const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", + "Compression method for `.ls` files."}; + + const Setting logCompression{(StoreConfig*) this, "", "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + const Setting multipartUpload{ - (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"}; + (StoreConfig*) this, false, "multipart-upload", + "Whether to use multi-part uploads."}; + const Setting bufferSize{ - (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", + "Size (in bytes) of each part in multi-part uploads."}; const std::string name() override { return "S3 Binary Cache Store"; } diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh new file mode 100644 index 000000000..767fc5e05 --- /dev/null +++ b/src/libstore/ssh-store-config.hh @@ -0,0 +1,29 @@ +#include "store-api.hh" + +namespace nix { + +struct CommonSSHStoreConfig : virtual StoreConfig +{ + using StoreConfig::StoreConfig; + + const Setting sshKey{(StoreConfig*) this, "", "ssh-key", + "Path to the SSH private key used to authenticate to the remote machine."}; + + const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", + "The public host key of the remote machine."}; + + const Setting compress{(StoreConfig*) this, false, "compress", + "Whether to enable SSH compression."}; + + const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", + "Path to the `nix-store` executable on the remote machine."}; + + const Setting remoteStore{(StoreConfig*) this, "", "remote-store", + R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )"}; +}; + +} diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 6f3ee6262..3206bf169 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,3 +1,4 @@ +#include "ssh-store-config.hh" #include "store-api.hh" #include "remote-store.hh" #include "remote-fs-accessor.hh" @@ -8,16 +9,10 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig { using RemoteStoreConfig::RemoteStoreConfig; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"}; - const Setting compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; - const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; - const std::string name() override { return "Experimental SSH Store"; } std::string doc() override diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9afb33b9e..2f4391c43 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -107,16 +107,35 @@ struct StoreConfig : public Config } const PathSetting storeDir_{this, false, settings.nixStore, - "store", "path to the Nix store"}; + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; const Path storeDir = storeDir_; - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; + const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", + "Size of the in-memory store path metadata cache."}; - const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + const Setting isTrusted{this, false, "trusted", + R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )"}; - Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; + Setting priority{this, 0, "priority", + R"( + Priority of this store when used as a substituter. A lower value means a higher priority. + )"}; - Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; + Setting wantMassQuery{this, false, "want-mass-query", + R"( + Whether this store (when used as a substituter) can be + queried efficiently for path validity. + )"}; Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", @@ -130,8 +149,6 @@ public: typedef std::map Params; - - protected: struct PathInfoCacheValue { -- cgit v1.2.3 From b134546f08000ce80722fdc4de58ac5fd20865ed Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Mar 2023 09:11:15 +0100 Subject: Fix clang build --- src/libstore/legacy-ssh-store.cc | 1 + src/libstore/ssh-store.cc | 2 ++ 2 files changed, 3 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a8ce0b8b0..72b8e89fc 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -56,6 +56,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) + , CommonSSHStoreConfig(params) , LegacySSHStoreConfig(params) , Store(params) , host(host) diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 3206bf169..be8a67c13 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -12,6 +12,7 @@ namespace nix { struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig { using RemoteStoreConfig::RemoteStoreConfig; + using CommonSSHStoreConfig::CommonSSHStoreConfig; const std::string name() override { return "Experimental SSH Store"; } @@ -30,6 +31,7 @@ public: SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) , SSHStoreConfig(params) , Store(params) , RemoteStore(params) -- cgit v1.2.3 From 80f0b8d3070f346bb89bd0ab1a541940a7ceeec1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Mar 2023 09:35:35 +0100 Subject: Fix SSHStore --- src/libstore/legacy-ssh-store.cc | 3 +++ src/libstore/ssh-store-config.hh | 3 --- src/libstore/ssh-store.cc | 3 +++ 3 files changed, 6 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 72b8e89fc..98322b045 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -17,6 +17,9 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; + const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", + "Path to the `nix-store` executable on the remote machine."}; + const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", "Maximum number of concurrent SSH connections."}; diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index 767fc5e05..c4232df34 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -15,9 +15,6 @@ struct CommonSSHStoreConfig : virtual StoreConfig const Setting compress{(StoreConfig*) this, false, "compress", "Whether to enable SSH compression."}; - const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", - "Path to the `nix-store` executable on the remote machine."}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", R"( [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index be8a67c13..962221ad2 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -14,6 +14,9 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig using RemoteStoreConfig::RemoteStoreConfig; using CommonSSHStoreConfig::CommonSSHStoreConfig; + const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", + "Path to the `nix-daemon` executable on the remote machine."}; + const std::string name() override { return "Experimental SSH Store"; } std::string doc() override -- cgit v1.2.3 From abc449bc30bd0c8e2aaaa7840e83237729e9af19 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Mar 2023 10:12:57 +0100 Subject: Update src/libstore/local-store.md Co-authored-by: Valentin Gagarin --- src/libstore/local-store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.md b/src/libstore/local-store.md index 893457d32..580cf5358 100644 --- a/src/libstore/local-store.md +++ b/src/libstore/local-store.md @@ -13,7 +13,7 @@ A store that uses a *root* other than `/` is called a *chroot store*. With such stores, the store directory is "logically" still `/nix/store`, so programs stored in them can only be built and executed by `chroot`-ing into *root*. Chroot stores only support -building and running on Linux when mount and user namespaces are +building and running on Linux when [`mount namespaces`](https://man7.org/linux/man-pages/man7/mount_namespaces.7.html) and [`user namespaces`](https://man7.org/linux/man-pages/man7/user_namespaces.7.html) are enabled. For example, the following uses `/tmp/root` as the chroot environment -- cgit v1.2.3 From fb67c1a1fb5eba91abe3ab411a93cb49960454cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 21 Mar 2023 13:37:19 +0100 Subject: Factor out the generation of the profile/channel directory Make sure that all the code paths use the same one, and that the backwards-compatibility measures are probably in place when needed --- src/libstore/profiles.cc | 30 +++++++++++++++++++++++++----- src/libstore/profiles.hh | 9 +++++++++ 2 files changed, 34 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 179161ff7..ba5c8583f 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -282,28 +282,48 @@ std::string optimisticLockProfile(const Path & profile) Path profilesDir() { - auto profileRoot = createNixStateDir() + "/profiles"; + auto profileRoot = + (getuid() == 0) + ? rootProfilesDir() + : createNixStateDir() + "/profiles"; createDirs(profileRoot); return profileRoot; } +Path rootProfilesDir() +{ + return settings.nixStateDir + "/profiles/per-user/root"; +} + Path getDefaultProfile() { Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; try { - auto profile = - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : profilesDir() + "/profile"; + auto profile = profilesDir() + "/profile"; if (!pathExists(profileLink)) { replaceSymlink(profile, profileLink); } + // Backwards compatibiliy measure: Make root's profile available as + // `.../default` as it's what NixOS and most of the init scripts expect + Path globalProfileLink = settings.nixStateDir + "/profiles/default"; + if (getuid() == 0 && !pathExists(globalProfileLink)) { + replaceSymlink(profile, globalProfileLink); + } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { return profileLink; } } +Path defaultChannelsDir() +{ + return profilesDir() + "/channels"; +} + +Path rootChannelsDir() +{ + return rootProfilesDir() + "/channels"; +} } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index fbf95b850..9d5d4779c 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -72,6 +72,15 @@ std::string optimisticLockProfile(const Path & profile); profiles. */ Path profilesDir(); +/* Returns the path to the profile directory for root (but doesn't try creating it) */ +Path rootProfilesDir(); + +/* Creates and returns the path to the file used for storing the users's channels */ +Path defaultChannelsDir(); + +/* Returns the path to the channel directory for root (but doesn't try creating it) */ +Path rootChannelsDir(); + /* Resolve the default profile (~/.nix-profile by default, $XDG_STATE_HOME/ nix/profile if XDG Base Directory Support is enabled), and create if doesn't exist */ -- cgit v1.2.3 From 161f4b0dea82bdaed7cf8928e523702f61cf899b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Mar 2023 10:38:48 +0100 Subject: Document store URLs --- src/libstore/globals.hh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4382d1b85..a74d5a46d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -98,7 +98,9 @@ public: Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", R"( - The URL of the Nix store to use. See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) + The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to use for most operations. + See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for supported store types and settings. )"}; @@ -681,8 +683,9 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of URLs of substituters, separated by whitespace. Substituters - are tried based on their Priority value, which each substituter can set + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + to be used as substituters, separated by whitespace. + Substituters are tried based on their Priority value, which each substituter can set independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. @@ -700,7 +703,8 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of URLs of substituters, separated by whitespace. These are + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), + separated by whitespace. These are not used by default, but can be enabled by users of the Nix daemon by specifying `--option substituters urls` on the command line. Unprivileged users are only allowed to pass a subset of the -- cgit v1.2.3 From b79df9dedce957e083870d72e649262b5a8893a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Mar 2023 15:23:13 +0100 Subject: Register LocalStore to ensure it's included in the manual --- src/libstore/local-store.cc | 8 ++++++++ src/libstore/local-store.hh | 4 ++++ src/libstore/uds-remote-store.cc | 6 +++--- 3 files changed, 15 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1eada7cc3..22a5dd8f5 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -420,6 +420,13 @@ LocalStore::LocalStore(const Params & params) } +LocalStore::LocalStore(std::string scheme, std::string path, const Params & params) + : LocalStore(params) +{ + throw UnimplementedError("LocalStore"); +} + + AutoCloseFD LocalStore::openGCLock() { Path fnGCLock = stateDir + "/gc.lock"; @@ -1957,5 +1964,6 @@ std::optional LocalStore::getVersion() return nixVersion; } +static RegisterStoreImplementation regLocalStore; } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 3743f6b10..639772b36 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -102,9 +102,13 @@ public: /* Initialise the local store, upgrading the schema if necessary. */ LocalStore(const Params & params); + LocalStore(std::string scheme, std::string path, const Params & params); ~LocalStore(); + static std::set uriSchemes() + { return {}; } + /* Implementations of abstract store API methods. */ std::string getUri() override; diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 5c38323cd..0fb7c38e9 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -26,9 +26,9 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) UDSRemoteStore::UDSRemoteStore( - const std::string scheme, - std::string socket_path, - const Params & params) + const std::string scheme, + std::string socket_path, + const Params & params) : UDSRemoteStore(params) { path.emplace(socket_path); -- cgit v1.2.3 From 168b6021c558f317fc01b142c1610742e9b9096b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Mar 2023 15:32:59 +0100 Subject: Tweaks --- src/libstore/local-store.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.md b/src/libstore/local-store.md index 580cf5358..8174df839 100644 --- a/src/libstore/local-store.md +++ b/src/libstore/local-store.md @@ -3,11 +3,10 @@ R"( **Store URL format**: `local`, *root* This store type accesses a Nix store in the local filesystem directly -(i.e. not via the Nix daemon). *root* is an absolute path that denotes -the "root" of the filesystem; other directories such as the Nix store -directory (as denoted by the `store` setting) are interpreted relative -to *root*. The store pseudo-URL `local` denotes a store that uses `/` -as its root directory. +(i.e. not via the Nix daemon). *root* is an absolute path that is +prefixed to other directories such as the Nix store directory. The +store pseudo-URL `local` denotes a store that uses `/` as its root +directory. A store that uses a *root* other than `/` is called a *chroot store*. With such stores, the store directory is "logically" still -- cgit v1.2.3 From 128994509f3d514b6a14f42f7459a40fe2d38bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 27 Mar 2023 10:02:10 +0200 Subject: Make some comments in profiles.hh doxygen-enabled These are proper documentation of the API, so they deserve to be here --- src/libstore/profiles.hh | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 9d5d4779c..3cadd5c2a 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -68,22 +68,32 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ std::string optimisticLockProfile(const Path & profile); -/* Creates and returns the path to a directory suitable for storing the user’s - profiles. */ +/** + * Create and return the path to a directory suitable for storing the user’s + * profiles. + */ Path profilesDir(); -/* Returns the path to the profile directory for root (but doesn't try creating it) */ +/** + * Return the path to the profile directory for root (but don't try creating it) + */ Path rootProfilesDir(); -/* Creates and returns the path to the file used for storing the users's channels */ +/** + * Create and return the path to the file used for storing the users's channels + */ Path defaultChannelsDir(); -/* Returns the path to the channel directory for root (but doesn't try creating it) */ +/** + * Return the path to the channel directory for root (but don't try creating it) + */ Path rootChannelsDir(); -/* Resolve the default profile (~/.nix-profile by default, $XDG_STATE_HOME/ - nix/profile if XDG Base Directory Support is enabled), and create if doesn't - exist */ +/** + * Resolve the default profile (~/.nix-profile by default, + * $XDG_STATE_HOME/nix/profile if XDG Base Directory Support is enabled), + * and create if doesn't exist + */ Path getDefaultProfile(); } -- cgit v1.2.3 From a519436e9d78fa6500999485c2c638553a32696f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Mar 2023 14:08:39 +0200 Subject: ssl-cert-file: Use lists --- src/libstore/globals.hh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 95f6b5e35..6803fb614 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -832,12 +832,17 @@ public: this, getDefaultSSLCertFile(), "ssl-cert-file", R"( The path of a file containing CA certificates used to - authenticate `https://` downloads. It defaults to the first - of `/etc/ssl/certs/ca-certificates.crt` and - `/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt` - that exists. It can be overriden using the - `NIX_SSL_CERT_FILE` and `SSL_CERT_FILE` environment variable - (in that order of precedence). + authenticate `https://` downloads. Nix by default will use + the first of the following files that exists: + + 1. `/etc/ssl/certs/ca-certificates.crt` + 2. `/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt` + + The path can be overridden by the following environment + variables, in order of precedence: + + 1. `NIX_SSL_CERT_FILE` + 2. `SSL_CERT_FILE` )"}; #if __linux__ -- cgit v1.2.3 From 8cab89a94b28a51b8dcbbe4b681df3c633a41aec Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 25 Mar 2023 19:12:44 -0400 Subject: Convert a bunch of comments in headers to Doxygen documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The internal API docs now contain more useful information. Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libstore/content-address.hh | 72 +++-- src/libstore/derivations.hh | 348 ++++++++++++++------- src/libstore/derived-path.hh | 2 +- src/libstore/outputs-spec.hh | 33 +- src/libstore/path.hh | 21 +- src/libstore/store-api.hh | 674 +++++++++++++++++++++++++--------------- 6 files changed, 756 insertions(+), 394 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..634a51c38 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -5,44 +5,75 @@ namespace nix { +/** + * An enumeration of the ways we can serialize file system objects. + */ enum struct FileIngestionMethod : uint8_t { + /** + * Flat-file hashing. Directly ingest the contents of a single file + */ Flat = false, + /** + * Recursive (or NAR) hashing. Serializes the file-system object in Nix + * Archive format and ingest that + */ Recursive = true }; +/** + * Somewhat obscure, used by \ref Derivation derivations and + * `builtins.toFile` currently. + */ struct TextHash { + /** + * Hash of the contents of the text/file. + */ Hash hash; }; -/// Pair of a hash, and how the file system was ingested +/** + * For path computed by makeFixedOutputPath. + */ struct FixedOutputHash { + /** + * How the file system objects are serialized + */ FileIngestionMethod method; + /** + * Hash of that serialization + */ Hash hash; + std::string printMethodAlgo() const; }; -/* - We've accumulated several types of content-addressed paths over the years; - fixed-output derivations support multiple hash algorithms and serialisation - methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms: - - * ‘text:sha256:’: For paths - computed by makeTextPath() / addTextToStore(). - - * ‘fixed:::’: For paths computed by - makeFixedOutputPath() / addToStore(). -*/ +/** + * We've accumulated several types of content-addressed paths over the + * years; fixed-output derivations support multiple hash algorithms and + * serialisation methods (flat file vs NAR). Thus, ‘ca’ has one of the + * following forms: + * + * - ‘text:sha256:’: For paths + * computed by Store::makeTextPath() / Store::addTextToStore(). + * + * - ‘fixed:::’: For paths computed by + * Store::makeFixedOutputPath() / Store::addToStore(). + */ typedef std::variant< - TextHash, // for paths computed by makeTextPath() / addTextToStore - FixedOutputHash // for path computed by makeFixedOutputPath + TextHash, + FixedOutputHash > ContentAddress; -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ +/** + * Compute the prefix to the hash algorithm which indicates how the + * files were ingested. + */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ +/** + * Compute the content-addressability assertion (ValidPathInfo::ca) for + * paths created by Store::makeFixedOutputPath() / Store::addToStore(). + */ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); std::string renderContentAddress(ContentAddress ca); @@ -65,6 +96,11 @@ struct FixedOutputHashMethod { HashType hashType; }; +/** + * Ways of content addressing but not a complete ContentAddress. + * + * A ContentAddress without a Hash. + */ typedef std::variant< TextHashMethod, FixedOutputHashMethod diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 8456b29e7..a5731f18d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -17,42 +17,72 @@ class Store; /* Abstract syntax of derivations. */ -/* The traditional non-fixed-output derivation type. */ +/** + * The traditional non-fixed-output derivation type. + */ struct DerivationOutputInputAddressed { StorePath path; }; -/* Fixed-output derivations, whose output paths are content addressed - according to that fixed output. */ +/** + * Fixed-output derivations, whose output paths are content + * addressed according to that fixed output. + */ struct DerivationOutputCAFixed { - FixedOutputHash hash; /* hash used for expected hash computation */ + /** + * hash used for expected hash computation + */ + FixedOutputHash hash; + + /** + * Return the \ref StorePath "store path" corresponding to this output + * + * @param drvName The name of the derivation this is an output of, without the `.drv`. + * @param outputName The name of this output. + */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; }; -/* Floating-output derivations, whose output paths are content addressed, but - not fixed, and so are dynamically calculated from whatever the output ends - up being. */ +/** + * Floating-output derivations, whose output paths are content + * addressed, but not fixed, and so are dynamically calculated from + * whatever the output ends up being. + * */ struct DerivationOutputCAFloating { - /* information used for expected hash computation */ + /** + * How the file system objects will be serialized for hashing + */ FileIngestionMethod method; + + /** + * How the serialization will be hashed + */ HashType hashType; }; -/* Input-addressed output which depends on a (CA) derivation whose hash isn't - * known yet. +/** + * Input-addressed output which depends on a (CA) derivation whose hash + * isn't known yet. */ struct DerivationOutputDeferred {}; -/* Impure output which is moved to a content-addressed location (like - CAFloating) but isn't registered as a realization. +/** + * Impure output which is moved to a content-addressed location (like + * CAFloating) but isn't registered as a realization. */ struct DerivationOutputImpure { - /* information used for expected hash computation */ + /** + * How the file system objects will be serialized for hashing + */ FileIngestionMethod method; + + /** + * How the serialization will be hashed + */ HashType hashType; }; @@ -64,6 +94,9 @@ typedef std::variant< DerivationOutputImpure > _DerivationOutputRaw; +/** + * A single output of a BasicDerivation (and Derivation). + */ struct DerivationOutput : _DerivationOutputRaw { using Raw = _DerivationOutputRaw; @@ -75,9 +108,12 @@ struct DerivationOutput : _DerivationOutputRaw using Deferred = DerivationOutputDeferred; using Impure = DerivationOutputImpure; - /* Note, when you use this function you should make sure that you're passing - the right derivation name. When in doubt, you should use the safer - interface provided by BasicDerivation::outputsAndOptPaths */ + /** + * \note when you use this function you should make sure that you're + * passing the right derivation name. When in doubt, you should use + * the safer interface provided by + * BasicDerivation::outputsAndOptPaths + */ std::optional path(const Store & store, std::string_view drvName, std::string_view outputName) const; inline const Raw & raw() const { @@ -92,26 +128,61 @@ struct DerivationOutput : _DerivationOutputRaw typedef std::map DerivationOutputs; -/* These are analogues to the previous DerivationOutputs data type, but they - also contains, for each output, the (optional) store path in which it would - be written. To calculate values of these types, see the corresponding - functions in BasicDerivation */ +/** + * These are analogues to the previous DerivationOutputs data type, + * but they also contains, for each output, the (optional) store + * path in which it would be written. To calculate values of these + * types, see the corresponding functions in BasicDerivation. + */ typedef std::map>> DerivationOutputsAndOptPaths; -/* For inputs that are sub-derivations, we specify exactly which - output IDs we are interested in. */ +/** + * For inputs that are sub-derivations, we specify exactly which + * output IDs we are interested in. + */ typedef std::map DerivationInputs; +/** + * Input-addressed derivation types + */ struct DerivationType_InputAddressed { + /** + * True iff the derivation type can't be determined statically, + * for instance because it (transitively) depends on a content-addressed + * derivation. + */ bool deferred; }; +/** + * Content-addressed derivation types + */ struct DerivationType_ContentAddressed { + /** + * Whether the derivation should be built safely inside a sandbox. + */ bool sandboxed; + /** + * Whether the derivation's outputs' content-addresses are "fixed" + * or "floating. + * + * - Fixed: content-addresses are written down as part of the + * derivation itself. If the outputs don't end up matching the + * build fails. + * + * - Floating: content-addresses are not written down, we do not + * know them until we perform the build. + */ bool fixed; }; +/** + * Impure derivation type + * + * This is similar at buil-time to the content addressed, not standboxed, not fixed + * type, but has some restrictions on its usage. + */ struct DerivationType_Impure { }; @@ -128,30 +199,38 @@ struct DerivationType : _DerivationTypeRaw { using ContentAddressed = DerivationType_ContentAddressed; using Impure = DerivationType_Impure; - /* Do the outputs of the derivation have paths calculated from their content, - or from the derivation itself? */ + /** + * Do the outputs of the derivation have paths calculated from their + * content, or from the derivation itself? + */ bool isCA() const; - /* Is the content of the outputs fixed a-priori via a hash? Never true for - non-CA derivations. */ + /** + * Is the content of the outputs fixed a priori via a hash? + * Never true for non-CA derivations. + */ bool isFixed() const; - /* Whether the derivation is fully sandboxed. If false, the - sandbox is opened up, e.g. the derivation has access to the - network. Note that whether or not we actually sandbox the - derivation is controlled separately. Always true for non-CA - derivations. */ + /** + * Whether the derivation is fully sandboxed. If false, the sandbox + * is opened up, e.g. the derivation has access to the network. Note + * that whether or not we actually sandbox the derivation is + * controlled separately. Always true for non-CA derivations. + */ bool isSandboxed() const; - /* Whether the derivation is expected to produce the same result - every time, and therefore it only needs to be built once. This - is only false for derivations that have the attribute '__impure - = true'. */ + /** + * Whether the derivation is expected to produce the same result + * every time, and therefore it only needs to be built once. This is + * only false for derivations that have the attribute '__impure = + * true'. + */ bool isPure() const; - /* Does the derivation knows its own output paths? - Only true when there's no floating-ca derivation involved in the - closure, or if fixed output. + /** + * Does the derivation knows its own output paths? + * Only true when there's no floating-ca derivation involved in the + * closure, or if fixed output. */ bool hasKnownOutputPaths() const; @@ -175,15 +254,21 @@ struct BasicDerivation bool isBuiltin() const; - /* Return true iff this is a fixed-output derivation. */ + /** + * Return true iff this is a fixed-output derivation. + */ DerivationType type() const; - /* Return the output names of a derivation. */ + /** + * Return the output names of a derivation. + */ StringSet outputNames() const; - /* Calculates the maps that contains all the DerivationOutputs, but - augmented with knowledge of the Store paths they would be written - into. */ + /** + * Calculates the maps that contains all the DerivationOutputs, but + * augmented with knowledge of the Store paths they would be written + * into. + */ DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; static std::string_view nameFromPath(const StorePath & storePath); @@ -191,23 +276,33 @@ struct BasicDerivation struct Derivation : BasicDerivation { - DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + /** + * inputs that are sub-derivations + */ + DerivationInputs inputDrvs; - /* Print a derivation. */ + /** + * Print a derivation. + */ std::string unparse(const Store & store, bool maskOutputs, std::map * actualInputs = nullptr) const; - /* Return the underlying basic derivation but with these changes: - - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. - - 2. Input placeholders are replaced with realized input store paths. */ + /** + * Return the underlying basic derivation but with these changes: + * + * 1. Input drvs are emptied, but the outputs of them that were used + * are added directly to input sources. + * + * 2. Input placeholders are replaced with realized input store + * paths. + */ std::optional tryResolve(Store & store) const; - /* Like the above, but instead of querying the Nix database for - realisations, uses a given mapping from input derivation paths - + output names to actual output store paths. */ + /** + * Like the above, but instead of querying the Nix database for + * realisations, uses a given mapping from input derivation paths + + * output names to actual output store paths. + */ std::optional tryResolve( Store & store, const std::map, StorePath> & inputDrvOutputs) const; @@ -222,81 +317,108 @@ struct Derivation : BasicDerivation class Store; -/* Write a derivation to the Nix store, and return its path. */ +/** + * Write a derivation to the Nix store, and return its path. + */ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair = NoRepair, bool readOnly = false); -/* Read a derivation from a file. */ +/** + * Read a derivation from a file. + */ Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); -// FIXME: remove +/** + * \todo Remove. + * + * Use Path::isDerivation instead. + */ bool isDerivation(std::string_view fileName); -/* Calculate the name that will be used for the store path for this - output. - - This is usually -, but is just when - the output name is "out". */ +/** + * Calculate the name that will be used for the store path for this + * output. + * + * This is usually -, but is just when + * the output name is "out". + */ std::string outputPathName(std::string_view drvName, std::string_view outputName); -// The hashes modulo of a derivation. -// -// Each output is given a hash, although in practice only the content-addressed -// derivations (fixed-output or not) will have a different hash for each -// output. +/** + * The hashes modulo of a derivation. + * + * Each output is given a hash, although in practice only the content-addressed + * derivations (fixed-output or not) will have a different hash for each + * output. + */ struct DrvHash { + /** + * Map from output names to hashes + */ std::map hashes; enum struct Kind : bool { - // Statically determined derivations. - // This hash will be directly used to compute the output paths + /** + * Statically determined derivations. + * This hash will be directly used to compute the output paths + */ Regular, - // Floating-output derivations (and their reverse dependencies). + + /** + * Floating-output derivations (and their reverse dependencies). + */ Deferred, }; + /** + * The kind of derivation this is, simplified for just "derivation hash + * modulo" purposes. + */ Kind kind; }; void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; -/* Returns hashes with the details of fixed-output subderivations - expunged. - - A fixed-output derivation is a derivation whose outputs have a - specified content hash and hash algorithm. (Currently they must have - exactly one output (`out'), which is specified using the `outputHash' - and `outputHashAlgo' attributes, but the algorithm doesn't assume - this.) We don't want changes to such derivations to propagate upwards - through the dependency graph, changing output paths everywhere. - - For instance, if we change the url in a call to the `fetchurl' - function, we do not want to rebuild everything depending on it---after - all, (the hash of) the file being downloaded is unchanged. So the - *output paths* should not change. On the other hand, the *derivation - paths* should change to reflect the new dependency graph. - - For fixed-output derivations, this returns a map from the name of - each output to its hash, unique up to the output's contents. - - For regular derivations, it returns a single hash of the derivation - ATerm, after subderivations have been likewise expunged from that - derivation. +/** + * Returns hashes with the details of fixed-output subderivations + * expunged. + * + * A fixed-output derivation is a derivation whose outputs have a + * specified content hash and hash algorithm. (Currently they must have + * exactly one output (`out'), which is specified using the `outputHash' + * and `outputHashAlgo' attributes, but the algorithm doesn't assume + * this.) We don't want changes to such derivations to propagate upwards + * through the dependency graph, changing output paths everywhere. + * + * For instance, if we change the url in a call to the `fetchurl' + * function, we do not want to rebuild everything depending on it---after + * all, (the hash of) the file being downloaded is unchanged. So the + * *output paths* should not change. On the other hand, the *derivation + * paths* should change to reflect the new dependency graph. + * + * For fixed-output derivations, this returns a map from the name of + * each output to its hash, unique up to the output's contents. + * + * For regular derivations, it returns a single hash of the derivation + * ATerm, after subderivations have been likewise expunged from that + * derivation. */ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); -/* - Return a map associating each output to a hash that uniquely identifies its - derivation (modulo the self-references). - - FIXME: what is the Hash in this map? +/** + * Return a map associating each output to a hash that uniquely identifies its + * derivation (modulo the self-references). + * + * \todo What is the Hash in this map? */ std::map staticOutputHashes(Store & store, const Derivation & drv); -/* Memoisation of hashDerivationModulo(). */ +/** + * Memoisation of hashDerivationModulo(). + */ typedef std::map DrvHashes; // FIXME: global, though at least thread-safe. @@ -308,21 +430,25 @@ struct Sink; Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); -/* This creates an opaque and almost certainly unique string - deterministically from the output name. - - It is used as a placeholder to allow derivations to refer to their - own outputs without needing to use the hash of a derivation in - itself, making the hash near-impossible to calculate. */ +/** + * This creates an opaque and almost certainly unique string + * deterministically from the output name. + * + * It is used as a placeholder to allow derivations to refer to their + * own outputs without needing to use the hash of a derivation in + * itself, making the hash near-impossible to calculate. + */ std::string hashPlaceholder(const std::string_view outputName); -/* This creates an opaque and almost certainly unique string - deterministically from a derivation path and output name. - - It is used as a placeholder to allow derivations to refer to - content-addressed paths whose content --- and thus the path - themselves --- isn't yet known. This occurs when a derivation has a - dependency which is a CA derivation. */ +/** + * This creates an opaque and almost certainly unique string + * deterministically from a derivation path and output name. + * + * It is used as a placeholder to allow derivations to refer to + * content-addressed paths whose content --- and thus the path + * themselves --- isn't yet known. This occurs when a derivation has a + * dependency which is a CA derivation. + */ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); extern const Hash impureOutputHash; diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 9e0cce377..72dbcc128 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -105,7 +105,7 @@ using _BuiltPathRaw = std::variant< >; /** - * A built path. Similar to a `DerivedPath`, but enriched with the corresponding + * A built path. Similar to a DerivedPath, but enriched with the corresponding * output path(s). */ struct BuiltPath : _BuiltPathRaw { diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 46bc35ebc..0b7c98ac9 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -9,6 +9,9 @@ namespace nix { +/** + * A non-empty set of outputs, specified by name + */ struct OutputNames : std::set { using std::set::set; @@ -18,6 +21,9 @@ struct OutputNames : std::set { : std::set(s) { assert(!empty()); } + /** + * Needs to be "inherited manually" + */ OutputNames(std::set && s) : std::set(s) { assert(!empty()); } @@ -28,6 +34,9 @@ struct OutputNames : std::set { OutputNames() = delete; }; +/** + * The set of all outputs, without needing to name them explicitly + */ struct AllOutputs : std::monostate { }; typedef std::variant _OutputsSpecRaw; @@ -36,7 +45,9 @@ struct OutputsSpec : _OutputsSpecRaw { using Raw = _OutputsSpecRaw; using Raw::Raw; - /* Force choosing a variant */ + /** + * Force choosing a variant + */ OutputsSpec() = delete; using Names = OutputNames; @@ -52,14 +63,20 @@ struct OutputsSpec : _OutputsSpecRaw { bool contains(const std::string & output) const; - /* Create a new OutputsSpec which is the union of this and that. */ + /** + * Create a new OutputsSpec which is the union of this and that. + */ OutputsSpec union_(const OutputsSpec & that) const; - /* Whether this OutputsSpec is a subset of that. */ + /** + * Whether this OutputsSpec is a subset of that. + */ bool isSubsetOf(const OutputsSpec & outputs) const; - /* Parse a string of the form 'output1,...outputN' or - '*', returning the outputs spec. */ + /** + * Parse a string of the form 'output1,...outputN' or '*', returning + * the outputs spec. + */ static OutputsSpec parse(std::string_view s); static std::optional parseOpt(std::string_view s); @@ -81,8 +98,10 @@ struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw { return static_cast(*this); } - /* Parse a string of the form 'prefix^output1,...outputN' or - 'prefix^*', returning the prefix and the extended outputs spec. */ + /** + * Parse a string of the form 'prefix^output1,...outputN' or + * 'prefix^*', returning the prefix and the extended outputs spec. + */ static std::pair parse(std::string_view s); static std::optional> parseOpt(std::string_view s); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 1e5579b90..2730541c6 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -8,13 +8,22 @@ namespace nix { struct Hash; +/** + * \ref StorePath "Store path" is the fundamental reference type of Nix. + * A store paths refers to a Store object. + * + * See glossary.html#gloss-store-path for more information on a + * conceptual level. + */ class StorePath { std::string baseName; public: - /* Size of the hash part of store paths, in base-32 characters. */ + /** + * Size of the hash part of store paths, in base-32 characters. + */ constexpr static size_t HashLen = 32; // i.e. 160 bits constexpr static size_t MaxPathLen = 211; @@ -45,8 +54,9 @@ public: return baseName != other.baseName; } - /* Check whether a file name ends with the extension for - derivations. */ + /** + * Check whether a file name ends with the extension for derivations. + */ bool isDerivation() const; std::string_view name() const @@ -67,7 +77,10 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; -/* Extension of derivations in the Nix store. */ +/** + * The file extension of \ref Derivation derivations when serialized + * into store objects. + */ const std::string drvExtension = ".drv"; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2f4391c43..194cbe6c4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -55,7 +55,10 @@ namespace nix { */ MakeError(SubstError, Error); -MakeError(BuildError, Error); // denotes a permanent build failure +/** + * denotes a permanent build failure + */ +MakeError(BuildError, Error); MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); @@ -78,7 +81,9 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; -/* Magic header of exportPath() output (obsolete). */ +/** + * Magic header of exportPath() output (obsolete). + */ const uint32_t exportMagic = 0x4558494e; @@ -153,17 +158,26 @@ protected: struct PathInfoCacheValue { - // Time of cache entry creation or update + /** + * Time of cache entry creation or update + */ std::chrono::time_point time_point = std::chrono::steady_clock::now(); - // Null if missing + /** + * Null if missing + */ std::shared_ptr value; - // Whether the value is valid as a cache entry. The path may not exist. + /** + * Whether the value is valid as a cache entry. The path may not + * exist. + */ bool isKnownNow(); - // Past tense, because a path can only be assumed to exists when - // isKnownNow() && didExist() + /** + * Past tense, because a path can only be assumed to exists when + * isKnownNow() && didExist() + */ inline bool didExist() { return value != nullptr; } @@ -197,35 +211,53 @@ public: std::string printStorePath(const StorePath & path) const; - // FIXME: remove + /** + * Deprecated + * + * \todo remove + */ StorePathSet parseStorePathSet(const PathSet & paths) const; PathSet printStorePathSet(const StorePathSet & path) const; - /* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ + /** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ std::string showPaths(const StorePathSet & paths); - /* Return true if ‘path’ is in the Nix store (but not the Nix - store itself). */ + /** + * @return true if ‘path’ is in the Nix store (but not the Nix + * store itself). + */ bool isInStore(PathView path) const; - /* Return true if ‘path’ is a store path, i.e. a direct child of - the Nix store. */ + /** + * @return true if ‘path’ is a store path, i.e. a direct child of the + * Nix store. + */ bool isStorePath(std::string_view path) const; - /* Split a path like /nix/store/-/ into - /nix/store/- and /. */ + /** + * Split a path like /nix/store/-/ into + * /nix/store/- and /. + */ std::pair toStorePath(PathView path) const; - /* Follow symlinks until we end up with a path in the Nix store. */ + /** + * Follow symlinks until we end up with a path in the Nix store. + */ Path followLinksToStore(std::string_view path) const; - /* Same as followLinksToStore(), but apply toStorePath() to the - result. */ + /** + * Same as followLinksToStore(), but apply toStorePath() to the + * result. + */ StorePath followLinksToStorePath(std::string_view path) const; - /* Constructs a unique store path name. */ + /** + * Constructs a unique store path name. + */ StorePath makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const; StorePath makeStorePath(std::string_view type, @@ -246,33 +278,40 @@ public: const StorePathSet & references = {}, bool hasSelfReference = false) const; - /* This is the preparatory part of addToStore(); it computes the - store path to which srcPath is to be copied. Returns the store - path and the cryptographic hash of the contents of srcPath. */ + /** + * Preparatory part of addToStore(). + * + * @return the store path to which srcPath is to be copied + * and the cryptographic hash of the contents of srcPath. + */ std::pair computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; - /* Preparatory part of addTextToStore(). - - !!! Computation of the path should take the references given to - addTextToStore() into account, otherwise we have a (relatively - minor) security hole: a caller can register a source file with - bogus references. If there are too many references, the path may - not be garbage collected when it has to be (not really a problem, - the caller could create a root anyway), or it may be garbage - collected when it shouldn't be (more serious). - - Hashing the references would solve this (bogus references would - simply yield a different store path, so other users wouldn't be - affected), but it has some backwards compatibility issues (the - hashing scheme changes), so I'm not doing that for now. */ + /** + * Preparatory part of addTextToStore(). + * + * !!! Computation of the path should take the references given to + * addTextToStore() into account, otherwise we have a (relatively + * minor) security hole: a caller can register a source file with + * bogus references. If there are too many references, the path may + * not be garbage collected when it has to be (not really a problem, + * the caller could create a root anyway), or it may be garbage + * collected when it shouldn't be (more serious). + * + * Hashing the references would solve this (bogus references would + * simply yield a different store path, so other users wouldn't be + * affected), but it has some backwards compatibility issues (the + * hashing scheme changes), so I'm not doing that for now. + */ StorePath computeStorePathForText( std::string_view name, std::string_view s, const StorePathSet & references) const; - /* Check whether a path is valid. */ + /** + * Check whether a path is valid. + */ bool isValidPath(const StorePath & path); protected: @@ -281,53 +320,68 @@ protected: public: - /* If requested, substitute missing paths. This - implements nix-copy-closure's --use-substitutes - flag. */ + /** + * If requested, substitute missing paths. This + * implements nix-copy-closure's --use-substitutes + * flag. + */ void substitutePaths(const StorePathSet & paths); - /* Query which of the given paths is valid. Optionally, try to - substitute missing paths. */ + /** + * Query which of the given paths is valid. Optionally, try to + * substitute missing paths. + */ virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); - /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be replaced by 'x' - (i.e. you'll get /nix/store/-x rather than - /nix/store/-). Use queryPathInfo() to obtain the - full store path. FIXME: should return a set of - std::variant to get rid of this hack. */ + /** + * Query the set of all valid paths. Note that for some store + * backends, the name part of store paths may be replaced by 'x' + * (i.e. you'll get /nix/store/-x rather than + * /nix/store/-). Use queryPathInfo() to obtain the + * full store path. FIXME: should return a set of + * std::variant to get rid of this hack. + */ virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } constexpr static const char * MissingName = "x"; - /* Query information about a valid path. It is permitted to omit - the name part of the store path. */ + /** + * Query information about a valid path. It is permitted to omit + * the name part of the store path. + */ ref queryPathInfo(const StorePath & path); - /* Asynchronous version of queryPathInfo(). */ + /** + * Asynchronous version of queryPathInfo(). + */ void queryPathInfo(const StorePath & path, Callback> callback) noexcept; - /* Query the information about a realisation. */ + /** + * Query the information about a realisation. + */ std::shared_ptr queryRealisation(const DrvOutput &); - /* Asynchronous version of queryRealisation(). */ + /** + * Asynchronous version of queryRealisation(). + */ void queryRealisation(const DrvOutput &, Callback> callback) noexcept; - /* Check whether the given valid path info is sufficiently attested, by - either being signed by a trusted public key or content-addressed, in - order to be included in the given store. - - These same checks would be performed in addToStore, but this allows an - earlier failure in the case where dependencies need to be added too, but - the addToStore wouldn't fail until those dependencies are added. Also, - we don't really want to add the dependencies listed in a nar info we - don't trust anyyways. - */ + /** + * Check whether the given valid path info is sufficiently attested, by + * either being signed by a trusted public key or content-addressed, in + * order to be included in the given store. + * + * These same checks would be performed in addToStore, but this allows an + * earlier failure in the case where dependencies need to be added too, but + * the addToStore wouldn't fail until those dependencies are added. Also, + * we don't really want to add the dependencies listed in a nar info we + * don't trust anyyways. + */ virtual bool pathInfoIsUntrusted(const ValidPathInfo &) { return true; @@ -347,53 +401,77 @@ protected: public: - /* Queries the set of incoming FS references for a store path. - The result is not cleared. */ + /** + * Queries the set of incoming FS references for a store path. + * The result is not cleared. + */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) { unsupported("queryReferrers"); } - /* Return all currently valid derivations that have `path' as an - output. (Note that the result of `queryDeriver()' is the - derivation that was actually used to produce `path', which may - not exist anymore.) */ + /** + * @return all currently valid derivations that have `path' as an + * output. + * + * (Note that the result of `queryDeriver()' is the derivation that + * was actually used to produce `path', which may not exist + * anymore.) + */ virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; - /* Query the outputs of the derivation denoted by `path'. */ + /** + * Query the outputs of the derivation denoted by `path'. + */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); - /* Query the mapping outputName => outputPath for the given derivation. All - outputs are mentioned so ones mising the mapping are mapped to - `std::nullopt`. */ + /** + * Query the mapping outputName => outputPath for the given + * derivation. All outputs are mentioned so ones mising the mapping + * are mapped to `std::nullopt`. + */ virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); - /* Query the mapping outputName=>outputPath for the given derivation. - Assume every output has a mapping and throw an exception otherwise. */ + /** + * Query the mapping outputName=>outputPath for the given derivation. + * Assume every output has a mapping and throw an exception otherwise. + */ OutputPathMap queryDerivationOutputMap(const StorePath & path); - /* Query the full store path given the hash part of a valid store - path, or empty if the path doesn't exist. */ + /** + * Query the full store path given the hash part of a valid store + * path, or empty if the path doesn't exist. + */ virtual std::optional queryPathFromHashPart(const std::string & hashPart) = 0; - /* Query which of the given paths have substitutes. */ + /** + * Query which of the given paths have substitutes. + */ virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; - /* Query substitute info (i.e. references, derivers and download - sizes) of a map of paths to their optional ca values. The info - of the first succeeding substituter for each path will be - returned. If a path does not have substitute info, it's omitted - from the resulting ‘infos’ map. */ + /** + * Query substitute info (i.e. references, derivers and download + * sizes) of a map of paths to their optional ca values. The info of + * the first succeeding substituter for each path will be returned. + * If a path does not have substitute info, it's omitted from the + * resulting ‘infos’ map. + */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { return; }; - /* Import a path into the store. */ + /** + * Import a path into the store. + */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; - // A list of paths infos along with a source providing the content of the - // associated store path + /** + * A list of paths infos along with a source providing the content + * of the associated store path + */ using PathsSource = std::vector>>; - /* Import multiple paths into the store. */ + /** + * Import multiple paths into the store. + */ virtual void addMultipleToStore( Source & source, RepairFlag repair = NoRepair, @@ -405,10 +483,14 @@ public: RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); - /* Copy the contents of a path to the store and register the - validity the resulting path. The resulting path is returned. - The function object `filter' can be used to exclude files (see - libutil/archive.hh). */ + /** + * Copy the contents of a path to the store and register the + * validity the resulting path. + * + * @return The resulting path is returned. + * @param filter This function can be used to exclude files (see + * libutil/archive.hh). + */ virtual StorePath addToStore( std::string_view name, const Path & srcPath, @@ -418,26 +500,33 @@ public: RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()); - /* Copy the contents of a path to the store and register the - validity the resulting path, using a constant amount of - memory. */ + /** + * Copy the contents of a path to the store and register the + * validity the resulting path, using a constant amount of + * memory. + */ ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, std::optional expectedCAHash = {}); - /* Like addToStore(), but the contents of the path are contained - in `dump', which is either a NAR serialisation (if recursive == - true) or simply the contents of a regular file (if recursive == - false). - `dump` may be drained */ - // FIXME: remove? + /** + * Like addToStore(), but the contents of the path are contained + * in `dump', which is either a NAR serialisation (if recursive == + * true) or simply the contents of a regular file (if recursive == + * false). + * `dump` may be drained + * + * \todo remove? + */ virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } - /* Like addToStore, but the contents written to the output path is - a regular file containing the given string. */ + /** + * Like addToStore, but the contents written to the output path is a + * regular file containing the given string. + */ virtual StorePath addTextToStore( std::string_view name, std::string_view s, @@ -458,140 +547,180 @@ public: virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs) { return registerDrvOutput(output); } - /* Write a NAR dump of a store path. */ + /** + * Write a NAR dump of a store path. + */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; - /* For each path, if it's a derivation, build it. Building a - derivation means ensuring that the output paths are valid. If - they are already valid, this is a no-op. Otherwise, validity - can be reached in two ways. First, if the output paths is - substitutable, then build the path that way. Second, the - output paths can be created by running the builder, after - recursively building any sub-derivations. For inputs that are - not derivations, substitute them. */ + /** + * For each path, if it's a derivation, build it. Building a + * derivation means ensuring that the output paths are valid. If + * they are already valid, this is a no-op. Otherwise, validity + * can be reached in two ways. First, if the output paths is + * substitutable, then build the path that way. Second, the + * output paths can be created by running the builder, after + * recursively building any sub-derivations. For inputs that are + * not derivations, substitute them. + */ virtual void buildPaths( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); - /* Like `buildPaths()`, but return a vector of `BuildResult`s - corresponding to each element in `paths`. Note that in case of - a build/substitution error, this function won't throw an - exception, but return a `BuildResult` containing an error - message. */ + /** + * Like buildPaths(), but return a vector of \ref BuildResult + * BuildResults corresponding to each element in paths. Note that in + * case of a build/substitution error, this function won't throw an + * exception, but return a BuildResult containing an error message. + */ virtual std::vector buildPathsWithResults( const std::vector & paths, BuildMode buildMode = bmNormal, std::shared_ptr evalStore = nullptr); - /* Build a single non-materialized derivation (i.e. not from an - on-disk .drv file). - - ‘drvPath’ is used to deduplicate worker goals so it is imperative that - is correct. That said, it doesn't literally need to be store path that - would be calculated from writing this derivation to the store: it is OK - if it instead is that of a Derivation which would resolve to this (by - taking the outputs of it's input derivations and adding them as input - sources) such that the build time referenceable-paths are the same. - - In the input-addressed case, we usually *do* use an "original" - unresolved derivations's path, as that is what will be used in the - `buildPaths` case. Also, the input-addressed output paths are verified - only by that contents of that specific unresolved derivation, so it is - nice to keep that information around so if the original derivation is - ever obtained later, it can be verified whether the trusted user in fact - used the proper output path. - - In the content-addressed case, we want to always use the - resolved drv path calculated from the provided derivation. This serves - two purposes: - - - It keeps the operation trustless, by ruling out a maliciously - invalid drv path corresponding to a non-resolution-equivalent - derivation. - - - For the floating case in particular, it ensures that the derivation - to output mapping respects the resolution equivalence relation, so - one cannot choose different resolution-equivalent derivations to - subvert dependency coherence (i.e. the property that one doesn't end - up with multiple different versions of dependencies without - explicitly choosing to allow it). - */ + /** + * Build a single non-materialized derivation (i.e. not from an + * on-disk .drv file). + * + * @param drvPath This is used to deduplicate worker goals so it is + * imperative that is correct. That said, it doesn't literally need + * to be store path that would be calculated from writing this + * derivation to the store: it is OK if it instead is that of a + * Derivation which would resolve to this (by taking the outputs of + * it's input derivations and adding them as input sources) such + * that the build time referenceable-paths are the same. + * + * In the input-addressed case, we usually *do* use an "original" + * unresolved derivations's path, as that is what will be used in the + * buildPaths case. Also, the input-addressed output paths are verified + * only by that contents of that specific unresolved derivation, so it is + * nice to keep that information around so if the original derivation is + * ever obtained later, it can be verified whether the trusted user in fact + * used the proper output path. + * + * In the content-addressed case, we want to always use the resolved + * drv path calculated from the provided derivation. This serves two + * purposes: + * + * - It keeps the operation trustless, by ruling out a maliciously + * invalid drv path corresponding to a non-resolution-equivalent + * derivation. + * + * - For the floating case in particular, it ensures that the derivation + * to output mapping respects the resolution equivalence relation, so + * one cannot choose different resolution-equivalent derivations to + * subvert dependency coherence (i.e. the property that one doesn't end + * up with multiple different versions of dependencies without + * explicitly choosing to allow it). + */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - /* Ensure that a path is valid. If it is not currently valid, it - may be made valid by running a substitute (if defined for the - path). */ + /** + * Ensure that a path is valid. If it is not currently valid, it + * may be made valid by running a substitute (if defined for the + * path). + */ virtual void ensurePath(const StorePath & path); - /* Add a store path as a temporary root of the garbage collector. - The root disappears as soon as we exit. */ + /** + * Add a store path as a temporary root of the garbage collector. + * The root disappears as soon as we exit. + */ virtual void addTempRoot(const StorePath & path) { debug("not creating temporary root, store doesn't support GC"); } - /* Return a string representing information about the path that - can be loaded into the database using `nix-store --load-db' or - `nix-store --register-validity'. */ + /** + * @return a string representing information about the path that + * can be loaded into the database using `nix-store --load-db' or + * `nix-store --register-validity'. + */ std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); - /* Write a JSON representation of store path metadata, such as the - hash and the references. If ‘includeImpureInfo’ is true, - variable elements such as the registration time are - included. If ‘showClosureSize’ is true, the closure size of - each path is included. */ + /** + * Write a JSON representation of store path metadata, such as the + * hash and the references. + * + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + * + * @param showClosureSize If true, the closure size of each path is + * included. + */ nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); - /* Return the size of the closure of the specified path, that is, - the sum of the size of the NAR serialisation of each path in - the closure. */ + /** + * @return the size of the closure of the specified path, that is, + * the sum of the size of the NAR serialisation of each path in the + * closure. + */ std::pair getClosureSize(const StorePath & storePath); - /* Optimise the disk space usage of the Nix store by hard-linking files - with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking files + * with the same contents. + */ virtual void optimiseStore() { }; - /* Check the integrity of the Nix store. Returns true if errors - remain. */ + /** + * Check the integrity of the Nix store. + * + * @return true if errors remain. + */ virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; - /* Return an object to access files in the Nix store. */ + /** + * @return An object to access files in the Nix store. + */ virtual ref getFSAccessor() { unsupported("getFSAccessor"); } - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ + /** + * Repair the contents of the given path by redownloading it using + * a substituter (if available). + */ virtual void repairPath(const StorePath & path) { unsupported("repairPath"); } - /* Add signatures to the specified store path. The signatures are - not verified. */ + /** + * Add signatures to the specified store path. The signatures are + * not verified. + */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) { unsupported("addSignatures"); } /* Utility functions. */ - /* Read a derivation, after ensuring its existence through - ensurePath(). */ + /** + * Read a derivation, after ensuring its existence through + * ensurePath(). + */ Derivation derivationFromPath(const StorePath & drvPath); - /* Read a derivation (which must already be valid). */ + /** + * Read a derivation (which must already be valid). + */ Derivation readDerivation(const StorePath & drvPath); - /* Read a derivation from a potentially invalid path. */ + /** + * Read a derivation from a potentially invalid path. + */ Derivation readInvalidDerivation(const StorePath & drvPath); - /* Place in `out' the set of all store paths in the file system - closure of `storePath'; that is, all paths than can be directly - or indirectly reached from it. `out' is not cleared. If - `flipDirection' is true, the set of paths that can reach - `storePath' is returned; that is, the closures under the - `referrers' relation instead of the `references' relation is - returned. */ + /** + * @param [out] out Place in here the set of all store paths in the + * file system closure of `storePath'; that is, all paths than can + * be directly or indirectly reached from it. `out' is not cleared. + * + * @param flipDirection If true, the set of paths that can reach + * `storePath' is returned; that is, the closures under the + * `referrers' relation instead of the `references' relation is + * returned. + */ virtual void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); @@ -600,27 +729,34 @@ public: StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); - /* Given a set of paths that are to be built, return the set of - derivations that will be built, and the set of output paths - that will be substituted. */ + /** + * Given a set of paths that are to be built, return the set of + * derivations that will be built, and the set of output paths that + * will be substituted. + */ virtual void queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize); - /* Sort a set of paths topologically under the references - relation. If p refers to q, then p precedes q in this list. */ + /** + * Sort a set of paths topologically under the references + * relation. If p refers to q, then p precedes q in this list. + */ StorePaths topoSortPaths(const StorePathSet & paths); - /* Export multiple paths in the format expected by ‘nix-store - --import’. */ + /** + * Export multiple paths in the format expected by ‘nix-store + * --import’. + */ void exportPaths(const StorePathSet & paths, Sink & sink); void exportPath(const StorePath & path, Sink & sink); - /* Import a sequence of NAR dumps created by exportPaths() into - the Nix store. Optionally, the contents of the NARs are - preloaded into the specified FS accessor to speed up subsequent - access. */ + /** + * Import a sequence of NAR dumps created by exportPaths() into the + * Nix store. Optionally, the contents of the NARs are preloaded + * into the specified FS accessor to speed up subsequent access. + */ StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); struct Stats @@ -642,8 +778,9 @@ public: const Stats & getStats(); - /* Computes the full closure of of a set of store-paths for e.g. - derivations that need this information for `exportReferencesGraph`. + /** + * Computes the full closure of of a set of store-paths for e.g. + * derivations that need this information for `exportReferencesGraph`. */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); @@ -654,18 +791,24 @@ public: */ std::optional getBuildDerivationPath(const StorePath &); - /* Hack to allow long-running processes like hydra-queue-runner to - occasionally flush their path info cache. */ + /** + * Hack to allow long-running processes like hydra-queue-runner to + * occasionally flush their path info cache. + */ void clearPathInfoCache() { state.lock()->pathInfoCache.clear(); } - /* Establish a connection to the store, for store types that have - a notion of connection. Otherwise this is a no-op. */ + /** + * Establish a connection to the store, for store types that have + * a notion of connection. Otherwise this is a no-op. + */ virtual void connect() { }; - /* Get the protocol version of this store or it's connection. */ + /** + * Get the protocol version of this store or it's connection. + */ virtual unsigned int getProtocol() { return 0; @@ -681,7 +824,7 @@ public: return toRealPath(printStorePath(storePath)); } - /* + /** * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) */ @@ -693,7 +836,13 @@ protected: Stats stats; - /* Unsupported methods. */ + /** + * Helper for methods that are not unsupported: this is used for + * default definitions for virtual methods that are meant to be overriden. + * + * \todo Using this should be a last resort. It is better to make + * the method "virtual pure" and/or move it to a subclass. + */ [[noreturn]] void unsupported(const std::string & op) { throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); @@ -702,7 +851,9 @@ protected: }; -/* Copy a path from one store to another. */ +/** + * Copy a path from one store to another. + */ void copyStorePath( Store & srcStore, Store & dstStore, @@ -711,12 +862,14 @@ void copyStorePath( CheckSigsFlag checkSigs = CheckSigs); -/* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order (i.e. - if A is a reference of B, then A is copied before B), but the set - of store paths is not automatically closed; use copyClosure() for - that. Returns a map of what each path was copied to the dstStore - as. */ +/** + * Copy store paths from one store to another. The paths may be copied + * in parallel. They are copied in a topologically sorted order (i.e. if + * A is a reference of B, then A is copied before B), but the set of + * store paths is not automatically closed; use copyClosure() for that. + * + * @return a map of what each path was copied to the dstStore as. + */ std::map copyPaths( Store & srcStore, Store & dstStore, const RealisedPath::Set &, @@ -731,7 +884,9 @@ std::map copyPaths( CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -/* Copy the closure of `paths` from `srcStore` to `dstStore`. */ +/** + * Copy the closure of `paths` from `srcStore` to `dstStore`. + */ void copyClosure( Store & srcStore, Store & dstStore, const RealisedPath::Set & paths, @@ -746,52 +901,61 @@ void copyClosure( CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -/* Remove the temporary roots file for this process. Any temporary - root becomes garbage after this point unless it has been registered - as a (permanent) root. */ +/** + * Remove the temporary roots file for this process. Any temporary + * root becomes garbage after this point unless it has been registered + * as a (permanent) root. + */ void removeTempRoots(); -/* Resolve the derived path completely, failing if any derivation output - is unknown. */ +/** + * Resolve the derived path completely, failing if any derivation output + * is unknown. + */ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); -/* Return a Store object to access the Nix store denoted by - ‘uri’ (slight misnomer...). Supported values are: - - * ‘local’: The Nix store in /nix/store and database in - /nix/var/nix/db, accessed directly. - - * ‘daemon’: The Nix store accessed via a Unix domain socket - connection to nix-daemon. - - * ‘unix://’: The Nix store accessed via a Unix domain socket - connection to nix-daemon, with the socket located at . - - * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on - whether the user has write access to the local Nix - store/database. - - * ‘file://’: A binary cache stored in . - - * ‘https://’: A binary cache accessed via HTTP. - - * ‘s3://’: A writable binary cache stored on Amazon's Simple - Storage Service. - - * ‘ssh://[user@]’: A remote Nix store accessed by running - ‘nix-store --serve’ via SSH. - - You can pass parameters to the store implementation by appending - ‘?key=value&key=value&...’ to the URI. -*/ +/** + * @return a Store object to access the Nix store denoted by + * ‘uri’ (slight misnomer...). + * + * @param uri Supported values are: + * + * - ‘local’: The Nix store in /nix/store and database in + * /nix/var/nix/db, accessed directly. + * + * - ‘daemon’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon. + * + * - ‘unix://’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon, with the socket located at . + * + * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + * whether the user has write access to the local Nix + * store/database. + * + * - ‘file://’: A binary cache stored in . + * + * - ‘https://’: A binary cache accessed via HTTP. + * + * - ‘s3://’: A writable binary cache stored on Amazon's Simple + * Storage Service. + * + * - ‘ssh://[user@]’: A remote Nix store accessed by running + * ‘nix-store --serve’ via SSH. + * + * You can pass parameters to the store implementation by appending + * ‘?key=value&key=value&...’ to the URI. + */ ref openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); -/* Return the default substituter stores, defined by the - ‘substituters’ option and various legacy options. */ +/** + * @return the default substituter stores, defined by the + * ‘substituters’ option and various legacy options. + */ std::list> getDefaultSubstituters(); struct StoreFactory @@ -834,8 +998,10 @@ struct RegisterStoreImplementation }; -/* Display a set of paths in human-readable form (i.e., between quotes - and separated by commas). */ +/** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ std::string showPaths(const PathSet & paths); @@ -844,7 +1010,9 @@ std::optional decodeValidPathInfo( std::istream & str, std::optional hashGiven = std::nullopt); -/* Split URI into protocol+hierarchy part and its parameter set. */ +/** + * Split URI into protocol+hierarchy part and its parameter set. + */ std::pair splitUriAndParams(const std::string & uri); std::optional getDerivationCA(const BasicDerivation & drv); -- cgit v1.2.3