diff options
Diffstat (limited to 'src/libstore')
40 files changed, 806 insertions, 263 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9058bb8b1..9eae8d534 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -309,7 +309,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = nar.first, }, @@ -380,7 +380,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); getFile(narInfoFile, - {[=](std::future<std::optional<std::string>> fut) { + {[=,this](std::future<std::optional<std::string>> fut) { try { auto data = fut.get(); @@ -427,7 +427,7 @@ StorePath BinaryCacheStore::addToStore( *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = h, }, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6f3048b63..85e9e16ff 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -16,6 +16,7 @@ #include "json-utils.hh" #include "cgroup.hh" #include "personality.hh" +#include "namespaces.hh" #include <regex> #include <queue> @@ -167,7 +168,8 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() { +void LocalDerivationGoal::tryLocalBuild() +{ unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { state = &DerivationGoal::tryToBuild; @@ -205,6 +207,17 @@ void LocalDerivationGoal::tryLocalBuild() { #endif } + #if __linux__ + if (useChroot) { + if (!mountAndPidNamespacesSupported()) { + if (!settings.sandboxFallback) + throw Error("this system does not support the kernel namespaces that are required for sandboxing; use '--no-sandbox' to disable sandboxing"); + debug("auto-disabling sandboxing because the prerequisite namespaces are not available"); + useChroot = false; + } + } + #endif + if (useBuildUsers()) { if (!buildUser) buildUser = acquireUserLock(parsedDrv->useUidRange() ? 65536 : 1, useChroot); @@ -372,12 +385,6 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() } -int childEntry(void * arg) -{ - ((LocalDerivationGoal *) arg)->runChild(); - return 1; -} - #if __linux__ static void linkOrCopy(const Path & from, const Path & to) { @@ -663,7 +670,8 @@ void LocalDerivationGoal::startBuilder() nobody account. The latter is kind of a hack to support Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - chownToBuilder(chrootRootDir + "/etc"); + if (parsedDrv->useUidRange()) + chownToBuilder(chrootRootDir + "/etc"); if (parsedDrv->useUidRange() && (!buildUser || buildUser->getUIDCount() < 65536)) throw Error("feature 'uid-range' requires the setting '%s' to be enabled", settings.autoAllocateUids.name); @@ -888,12 +896,7 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.create(); - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; - static bool userNamespacesEnabled = - pathExists(maxUserNamespaces) - && trim(readFile(maxUserNamespaces)) != "0"; - - usingUserNamespace = userNamespacesEnabled; + usingUserNamespace = userNamespacesSupported(); Pid helper = startProcess([&]() { @@ -908,76 +911,21 @@ void LocalDerivationGoal::startBuilder() if (getuid() == 0 && setgroups(0, 0) == -1) throw SysError("setgroups failed"); - size_t stackSize = 1 * 1024 * 1024; - char * stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + ProcessOptions options; + options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; if (privateNetwork) - flags |= CLONE_NEWNET; + options.cloneFlags |= CLONE_NEWNET; if (usingUserNamespace) - flags |= CLONE_NEWUSER; - - pid_t child = clone(childEntry, stack + stackSize, flags, this); - if (child == -1 && errno == EINVAL) { - /* Fallback for Linux < 2.13 where CLONE_NEWPID and - CLONE_PARENT are not allowed together. */ - flags &= ~CLONE_NEWPID; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) { - /* Some distros patch Linux to not allow unprivileged - * user namespaces. If we get EPERM or EINVAL, try - * without CLONE_NEWUSER and see if that works. - * Details: https://salsa.debian.org/kernel-team/linux/-/commit/d98e00eda6bea437e39b9e80444eee84a32438a6 - */ - usingUserNamespace = false; - flags &= ~CLONE_NEWUSER; - child = clone(childEntry, stack + stackSize, flags, this); - } - if (child == -1) { - switch(errno) { - case EPERM: - case EINVAL: { - int errno_ = errno; - if (!userNamespacesEnabled && errno==EPERM) - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/user/max_user_namespaces"); - if (userNamespacesEnabled) { - Path procSysKernelUnprivilegedUsernsClone = "/proc/sys/kernel/unprivileged_userns_clone"; - if (pathExists(procSysKernelUnprivilegedUsernsClone) - && trim(readFile(procSysKernelUnprivilegedUsernsClone)) == "0") { - notice("user namespaces appear to be disabled; they are required for sandboxing; check /proc/sys/kernel/unprivileged_userns_clone"); - } - } - Path procSelfNsUser = "/proc/self/ns/user"; - if (!pathExists(procSelfNsUser)) - notice("/proc/self/ns/user does not exist; your kernel was likely built without CONFIG_USER_NS=y, which is required for sandboxing"); - /* Otherwise exit with EPERM so we can handle this in the - parent. This is only done when sandbox-fallback is set - to true (the default). */ - if (settings.sandboxFallback) - _exit(1); - /* Mention sandbox-fallback in the error message so the user - knows that having it disabled contributed to the - unrecoverability of this failure */ - throw SysError(errno_, "creating sandboxed builder process using clone(), without sandbox-fallback"); - } - default: - throw SysError("creating sandboxed builder process using clone()"); - } - } + options.cloneFlags |= CLONE_NEWUSER; + + pid_t child = startProcess([&]() { runChild(); }, options); + writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); }); - int res = helper.wait(); - if (res != 0 && settings.sandboxFallback) { - useChroot = false; - initTmpDir(); - goto fallback; - } else if (res != 0) + if (helper.wait() != 0) throw Error("unable to start build process"); userNamespaceSync.readSide = -1; @@ -1045,9 +993,6 @@ void LocalDerivationGoal::startBuilder() } else #endif { -#if __linux__ - fallback: -#endif pid = startProcess([&]() { runChild(); }); @@ -1516,8 +1461,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive, - [&](Store & store) { store.createUser("nobody", 65535); }); + daemon::NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); @@ -1907,6 +1851,10 @@ void LocalDerivationGoal::runChild() } } + /* Make /etc unwritable */ + if (!parsedDrv->useUidRange()) + chmod_(chrootRootDir + "/etc", 0555); + /* Unshare this mount namespace. This is necessary because pivot_root() below changes the root of the mount namespace. This means that the call to setns() in @@ -2323,11 +2271,28 @@ DrvOutputs LocalDerivationGoal::registerOutputs() buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt, inodesSeen); - debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); + bool discardReferences = false; + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { + settings.requireExperimentalFeature(Xp::DiscardReferences); + if (auto output = get(*udr, outputName)) { + if (!output->is_boolean()) + throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); + discardReferences = output->get<bool>(); + } + } + } + + StorePathSet references; + if (discardReferences) + debug("discarding references of output '%s'", outputName); + else { + debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath); - /* Pass blank Sink as we are not ready to hash data at this stage. */ - NullSink blank; - auto references = scanForReferences(blank, actualPath, referenceablePaths); + /* Pass blank Sink as we are not ready to hash data at this stage. */ + NullSink blank; + references = scanForReferences(blank, actualPath, referenceablePaths); + } outputReferencesIfUnregistered.insert_or_assign( outputName, diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b94fb8416..f775f8486 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -276,7 +276,7 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) + if (awake.empty() && 0U == settings.maxBuildJobs) { if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 5452758b3..034a9485b 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -17,8 +17,9 @@ std::string makeFileIngestionPrefix(FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; + default: + throw Error("impossible, caught both cases"); } - assert(false); } std::string makeContentAddressingPrefix(ContentAddressMethod m) { @@ -168,13 +169,13 @@ ContentAddressWithReferences contentAddressFromMethodHashAndRefs( if (refs.self) throw UsageError("Cannot have a self reference with text hashing scheme"); return TextInfo { - { .hash = std::move(hash) }, + .hash = { .hash = std::move(hash) }, .references = std::move(refs.others), }; }, [&](FileIngestionMethod m2) -> ContentAddressWithReferences { return FixedOutputInfo { - { + .hash = { .method = m2, .hash = std::move(hash), }, @@ -191,7 +192,7 @@ ContentAddressMethod getContentAddressMethod(const ContentAddressWithReferences return TextHashMethod {}; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.method; + return fsh.hash.method; }, }, ca); } @@ -222,13 +223,13 @@ ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { return std::visit(overloaded { [&](const TextHash & h) -> ContentAddressWithReferences { return TextInfo { - h, + .hash = h, .references = {}, }; }, [&](const FixedOutputHash & h) -> ContentAddressWithReferences { return FixedOutputInfo { - h, + .hash = h, .references = {}, }; }, @@ -239,10 +240,10 @@ Hash getContentAddressHash(const ContentAddressWithReferences & ca) { return std::visit(overloaded { [](const TextInfo & th) { - return th.hash; + return th.hash.hash; }, [](const FixedOutputInfo & fsh) { - return fsh.hash; + return fsh.hash.hash; }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 729b1078d..962a50c6b 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -118,18 +118,20 @@ struct StoreReferences { */ // This matches the additional info that we need for makeTextPath -struct TextInfo : TextHash { +struct TextInfo { + TextHash hash; // References for the paths, self references disallowed StorePathSet references; - GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); + GENERATE_CMP(TextInfo, me->hash, me->references); }; -struct FixedOutputInfo : FixedOutputHash { +struct FixedOutputInfo { + FixedOutputHash hash; // References for the paths StoreReferences references; - GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); + GENERATE_CMP(FixedOutputInfo, me->hash, me->references); }; typedef std::variant< diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0694f5b90..1745b1e57 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -222,7 +222,8 @@ struct ClientSettings else if (!hasSuffix(s, "/") && trusted.count(s + "/")) subs.push_back(s + "/"); else - warn("ignoring untrusted substituter '%s'", s); + warn("ignoring untrusted substituter '%s', you are not a trusted user.\n" + "Run `man nix.conf` for more information on the `substituters` configuration option.", s); res = subs; return true; }; @@ -235,6 +236,10 @@ struct ClientSettings // the daemon, as that could cause some pretty weird stuff if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); + } else if (name == settings.pluginFiles.name) { + if (tokenizeString<Paths>(value) != settings.pluginFiles.get()) + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name @@ -528,7 +533,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation in this message: the daemon code (i.e. + this file) knows whether a client/connection is trusted, but it + does not how how the client was authenticated. The mechanism + need not be getting the UID of the other end of a Unix Domain + Socket. + */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); } @@ -545,7 +557,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store, mode = (BuildMode) readInt(from); /* Repairing is not atomic, so disallowed for "untrusted" - clients. */ + clients. + + FIXME: layer violation; see above. */ if (mode == bmRepair && !trusted) throw Error("repairing is not allowed because you are not in 'trusted-users'"); @@ -984,8 +998,7 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - std::function<void(Store &)> authHook) + RecursiveFlag recursive) { auto monitor = !recursive ? std::make_unique<MonitorFdHup>(from.fd) : nullptr; @@ -1028,10 +1041,6 @@ void processConnection( try { - /* If we can't accept clientVersion, then throw an error - *here* (not above). */ - authHook(*store); - tunnelLogger->stopWork(); to.flush(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 67755d54e..8c765615c 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -13,11 +13,6 @@ void processConnection( FdSource & from, FdSink & to, TrustedFlag trusted, - RecursiveFlag recursive, - /* Arbitrary hook to check authorization / initialize user data / whatever - after the protocol has been negotiated. The idea is that this function - and everything it calls doesn't know about this stuff, and the - `nix-daemon` handles that instead. */ - std::function<void(Store &)> authHook); + RecursiveFlag recursive); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b19ee5ed1..5461f4a9c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -6,6 +6,7 @@ #include "worker-protocol.hh" #include "fs-accessor.hh" #include <boost/container/small_vector.hpp> +#include <nlohmann/json.hpp> namespace nix { @@ -887,4 +888,64 @@ std::optional<BasicDerivation> Derivation::tryResolve( const Hash impureOutputHash = hashString(htSHA256, "impure"); +nlohmann::json DerivationOutput::toJSON( + const Store & store, std::string_view drvName, std::string_view outputName) const +{ + nlohmann::json res = nlohmann::json::object(); + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doi) { + res["path"] = store.printStorePath(doi.path); + }, + [&](const DerivationOutput::CAFixed & dof) { + res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); + res["hashAlgo"] = printMethodAlgo(dof.ca); + res["hash"] = getContentAddressHash(dof.ca).to_string(Base16, false); + // FIXME print refs? + }, + [&](const DerivationOutput::CAFloating & dof) { + res["hashAlgo"] = makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType); + }, + [&](const DerivationOutput::Deferred &) {}, + [&](const DerivationOutput::Impure & doi) { + res["hashAlgo"] = makeContentAddressingPrefix(doi.method) + printHashType(doi.hashType); + res["impure"] = true; + }, + }, raw()); + return res; +} + +nlohmann::json Derivation::toJSON(const Store & store) const +{ + nlohmann::json res = nlohmann::json::object(); + + { + nlohmann::json & outputsObj = res["outputs"]; + outputsObj = nlohmann::json::object(); + for (auto & [outputName, output] : outputs) { + outputsObj[outputName] = output.toJSON(store, name, outputName); + } + } + + { + auto& inputsList = res["inputSrcs"]; + inputsList = nlohmann::json ::array(); + for (auto & input : inputSrcs) + inputsList.emplace_back(store.printStorePath(input)); + } + + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json ::object(); + for (auto & input : inputDrvs) + inputDrvsObj[store.printStorePath(input.first)] = input.second; + } + + res["system"] = platform; + res["builder"] = builder; + res["args"] = args; + res["env"] = env; + + return res; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 803ab3cd6..80629dadb 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -83,6 +83,11 @@ struct DerivationOutput : _DerivationOutputRaw inline const Raw & raw() const { return static_cast<const Raw &>(*this); } + + nlohmann::json toJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName) const; }; typedef std::map<std::string, DerivationOutput> DerivationOutputs; @@ -210,6 +215,8 @@ struct Derivation : BasicDerivation Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } + + nlohmann::json toJSON(const Store & store) const; }; diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 5f2dfa4f1..9e0cce377 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -4,8 +4,8 @@ #include "path.hh" #include "realisation.hh" #include "outputs-spec.hh" +#include "comparator.hh" -#include <optional> #include <variant> #include <nlohmann/json_fwd.hpp> @@ -28,8 +28,7 @@ struct DerivedPathOpaque { std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); - bool operator < (const DerivedPathOpaque & b) const - { return path < b.path; } + GENERATE_CMP(DerivedPathOpaque, me->path); }; /** @@ -52,8 +51,7 @@ struct DerivedPathBuilt { static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); nlohmann::json toJSON(ref<Store> store) const; - bool operator < (const DerivedPathBuilt & b) const - { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); } + GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); }; using _DerivedPathRaw = std::variant< @@ -97,6 +95,8 @@ struct BuiltPathBuilt { nlohmann::json toJSON(ref<Store> store) const; static BuiltPathBuilt parse(const Store & store, std::string_view); + + GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); }; using _BuiltPathRaw = std::variant< diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 756bd4423..1c8676a59 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -101,6 +101,7 @@ struct curlFileTransfer : public FileTransfer this->result.data.append(data); }) { + requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); if (!request.mimeType.empty()) @@ -828,7 +829,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) { auto state(_state->lock()); - while (state->data.empty()) { + if (state->data.empty()) { if (state->quit) { if (state->exc) std::rethrow_exception(state->exc); @@ -836,6 +837,8 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } state.wait(state->avail); + + if (state->data.empty()) continue; } chunk = std::move(state->data); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 130c5b670..8e33a3dec 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -222,19 +222,19 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s .longName = name, .description = "Enable sandboxing.", .category = category, - .handler = {[=]() { override(smEnabled); }} + .handler = {[this]() { override(smEnabled); }} }); args.addFlag({ .longName = "no-" + name, .description = "Disable sandboxing.", .category = category, - .handler = {[=]() { override(smDisabled); }} + .handler = {[this]() { override(smDisabled); }} }); args.addFlag({ .longName = "relaxed-" + name, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, - .handler = {[=]() { override(smRelaxed); }} + .handler = {[this]() { override(smRelaxed); }} }); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7111def92..93086eaf8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -201,7 +201,16 @@ public: {"build-timeout"}}; PathSetting buildHook{this, true, "", "build-hook", - "The path of the helper program that executes builds to remote machines."}; + R"( + The path to the helper program that executes remote builds. + + Nix communicates with the build hook over `stdio` using a custom protocol to request builds that cannot be performed directly by the Nix daemon. + The default value is the internal Nix binary that implements remote building. + + > **Important** + > + > Change this setting only if you really know what you’re doing. + )"}; Setting<std::string> builders{ this, "@" + nixConfDir + "/machines", "builders", @@ -279,8 +288,8 @@ public: If the build users group is empty, builds will be performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if - `NIX_REMOTE` is `daemon`). Obviously, this should not be used in - multi-user settings with untrusted users. + `NIX_REMOTE` is `daemon`). Obviously, this should not be used + with a nix daemon accessible to untrusted clients. Defaults to `nixbld` when running as root, *empty* otherwise. )", @@ -570,11 +579,15 @@ public: {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, "trusted-public-keys", R"( - A whitespace-separated list of public keys. When paths are copied - from another Nix store (such as a binary cache), they must be - signed with one of these keys. For example: - `cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=`. + A whitespace-separated list of public keys. + + At least one of the following condition must be met + for Nix to accept copying a store object from another + Nix store (such as a substituter): + + - the store object has been signed using a key in the trusted keys list + - the [`require-sigs`](#conf-require-sigs) option has been set to `false` + - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) )", {"binary-cache-public-keys"}}; @@ -670,13 +683,14 @@ public: independently. Lower value means higher priority. The default is `https://cache.nixos.org`, with a Priority of 40. - Nix will copy a store path from a remote store only if one - of the following is true: + At least one of the following conditions must be met for Nix to use + a substituter: - - the store object is signed by one of the [`trusted-public-keys`](#conf-trusted-public-keys) - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - - the [`require-sigs`](#conf-require-sigs) option has been set to `false` - - the store object is [output-addressed](@docroot@/glossary.md#gloss-output-addressed-store-object) + - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + + In addition, each store path should be trusted as described + in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; @@ -691,24 +705,6 @@ public: )", {"trusted-binary-caches"}}; - Setting<Strings> trustedUsers{ - this, {"root"}, "trusted-users", - R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. - - > **Warning** - > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. - )"}; - Setting<unsigned int> ttlNegativeNarInfoCache{ this, 3600, "narinfo-cache-negative-ttl", R"( @@ -731,18 +727,6 @@ public: mismatch if the build isn't reproducible. )"}; - /* ?Who we trust to use the daemon in safe ways */ - Setting<Strings> allowedUsers{ - this, {"*"}, "allowed-users", - R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. - - Note that trusted users are always allowed to connect. - )"}; - Setting<bool> printMissing{this, true, "print-missing", "Whether to print what paths need to be built or downloaded."}; @@ -970,6 +954,27 @@ public: resolves to a different location from that of the build machine. You can enable this setting if you are sure you're not going to do that. )"}; + + Setting<bool> useXDGBaseDirectories{ + this, false, "use-xdg-base-directories", + R"( + If set to `true`, Nix will conform to the [XDG Base Directory Specification] for files in `$HOME`. + The environment variables used to implement this are documented in the [Environment Variables section](@docroot@/installation/env-variables.md). + + [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html + + > **Warning** + > This changes the location of some well-known symlinks that Nix creates, which might break tools that rely on the old, non-XDG-conformant locations. + + In particular, the following locations change: + + | Old | New | + |-------------------|--------------------------------| + | `~/.nix-profile` | `$XDG_STATE_HOME/nix/profile` | + | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | + | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | + )" + }; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 73bcd6e81..1479822a9 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -56,7 +56,7 @@ public: void init() override { // FIXME: do this lazily? - if (auto cacheInfo = diskCache->cacheExists(cacheUri)) { + if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { wantMassQuery.setDefault(cacheInfo->wantMassQuery); priority.setDefault(cacheInfo->priority); } else { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 94c005130..72ba12e0d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -201,8 +201,6 @@ LocalStore::LocalStore(const Params & params) throw SysError("could not set permissions on '%s' to 755", perUserDir); } - createUser(getUserName(), getuid()); - /* Optionally, create directories and set permissions for a multi-user install. */ if (getuid() == 0 && settings.buildUsersGroup != "") { @@ -1417,7 +1415,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); ContentAddressWithReferences desc = FixedOutputInfo { - { + .hash = { .method = method, .hash = hash, }, @@ -1844,20 +1842,6 @@ void LocalStore::signPathInfo(ValidPathInfo & info) } -void LocalStore::createUser(const std::string & userName, uid_t userId) -{ - for (auto & dir : { - fmt("%s/profiles/per-user/%s", stateDir, userName), - fmt("%s/gcroots/per-user/%s", stateDir, userName) - }) { - createDirs(dir); - if (chmod(dir.c_str(), 0755) == -1) - throw SysError("changing permissions of directory '%s'", dir); - if (chown(dir.c_str(), userId, getgid()) == -1) - throw SysError("changing owner of directory '%s'", dir); - } -} - std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_( LocalStore::State & state, const DrvOutput & id) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 06d36a7d5..a84eb7c26 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -281,8 +281,6 @@ private: void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); - void createUser(const std::string & userName, uid_t userId) override; - // XXX: Make a generic `Store` method FixedOutputHash hashCAPath( const FileIngestionMethod & method, diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 3ee64c77a..53fe04704 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -31,11 +31,12 @@ std::map<StorePath, StorePath> makeContentAddressed( for (auto & ref : oldInfo->references) { if (ref == path) refs.self = true; - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) { - rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); refs.others.insert(std::move(replacement)); } } @@ -51,7 +52,7 @@ std::map<StorePath, StorePath> makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - { + .hash = { .method = FileIngestionMethod::Recursive, .hash = narModuloHash, }, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index acd4e0929..e16b60b88 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -21,16 +21,16 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; StorePathSet referrers; queryReferrers(path, referrers); - for (auto & ref : referrers) + for (auto& ref : referrers) if (ref != path) res.insert(ref); if (includeOutputs) - for (auto & i : queryValidDerivers(path)) + for (auto& i : queryValidDerivers(path)) res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); return res; @@ -40,12 +40,12 @@ void Store::computeFSClosure(const StorePathSet & startPaths, std::future<ref<const ValidPathInfo>> & fut) { StorePathSet res; auto info = fut.get(); - for (auto & ref : info->references) + for (auto& ref : info->references) if (ref != path) res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto & [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) if (maybeOutPath && isValidPath(*maybeOutPath)) res.insert(*maybeOutPath); @@ -93,12 +93,12 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv) [&](const TextInfo & ti) -> std::optional<ContentAddress> { if (!ti.references.empty()) return std::nullopt; - return static_cast<TextHash>(ti); + return ti.hash; }, [&](const FixedOutputInfo & fi) -> std::optional<ContentAddress> { if (!fi.references.empty()) return std::nullopt; - return static_cast<FixedOutputHash>(fi); + return fi.hash; }, }, dof->ca); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 3e0689534..2645f468b 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -84,11 +84,10 @@ public: Sync<State> _state; - NarInfoDiskCacheImpl() + NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite") { auto state(_state.lock()); - Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); @@ -98,7 +97,7 @@ public: state->db.exec(schema); state->insertCache.create(state->db, - "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + "insert into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?1, ?2, ?3, ?4, ?5) on conflict (url) do update set timestamp = ?2, storeDir = ?3, wantMassQuery = ?4, priority = ?5 returning id;"); state->queryCache.create(state->db, "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ? and timestamp > ?"); @@ -166,6 +165,8 @@ public: return i->second; } +private: + std::optional<Cache> queryCacheRaw(State & state, const std::string & uri) { auto i = state.caches.find(uri); @@ -173,15 +174,21 @@ public: auto queryCache(state.queryCache.use()(uri)(time(0) - cacheInfoTtl)); if (!queryCache.next()) return std::nullopt; - state.caches.emplace(uri, - Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); + auto cache = Cache { + .id = (int) queryCache.getInt(0), + .storeDir = queryCache.getStr(1), + .wantMassQuery = queryCache.getInt(2) != 0, + .priority = (int) queryCache.getInt(3), + }; + state.caches.emplace(uri, cache); } return getCache(state, uri); } - void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override +public: + int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override { - retrySQLite<void>([&]() { + return retrySQLite<int>([&]() { auto state(_state.lock()); SQLiteTxn txn(state->db); @@ -190,17 +197,29 @@ public: auto cache(queryCacheRaw(*state, uri)); if (cache) - return; + return cache->id; - state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); - assert(sqlite3_changes(state->db) == 1); - state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; + Cache ret { + .id = -1, // set below + .storeDir = storeDir, + .wantMassQuery = wantMassQuery, + .priority = priority, + }; + + { + auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); + assert(r.next()); + ret.id = (int) r.getInt(0); + } + + state->caches[uri] = ret; txn.commit(); + return ret.id; }); } - std::optional<CacheInfo> cacheExists(const std::string & uri) override + std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) override { return retrySQLite<std::optional<CacheInfo>>([&]() -> std::optional<CacheInfo> { auto state(_state.lock()); @@ -208,6 +227,7 @@ public: if (!cache) return std::nullopt; return CacheInfo { + .id = cache->id, .wantMassQuery = cache->wantMassQuery, .priority = cache->priority }; @@ -371,4 +391,9 @@ ref<NarInfoDiskCache> getNarInfoDiskCache() return cache; } +ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath) +{ + return make_ref<NarInfoDiskCacheImpl>(dbPath); +} + } diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 2dcaa76a4..4877f56d8 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -13,16 +13,17 @@ public: virtual ~NarInfoDiskCache() { } - virtual void createCache(const std::string & uri, const Path & storeDir, + virtual int createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) = 0; struct CacheInfo { + int id; bool wantMassQuery; int priority; }; - virtual std::optional<CacheInfo> cacheExists(const std::string & uri) = 0; + virtual std::optional<CacheInfo> upToDateCacheExists(const std::string & uri) = 0; virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( const std::string & uri, const std::string & hashPart) = 0; @@ -45,4 +46,6 @@ public: multiple threads. */ ref<NarInfoDiskCache> getNarInfoDiskCache(); +ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath); + } diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index 6d67b1e03..dc42d0bca 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -6,4 +6,4 @@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lnixstore -lnixutil -Cflags: -I${includedir}/nix -std=c++17 +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 5944afd06..76cab63e0 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -21,7 +21,7 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } -std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferenences() const +std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithReferences() const { if (! ca) return std::nullopt; @@ -30,7 +30,7 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef [&](const TextHash & th) -> ContentAddressWithReferences { assert(references.count(path) == 0); return TextInfo { - th, + .hash = th, .references = references, }; }, @@ -42,7 +42,7 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - foh, + .hash = foh, .references = { .others = std::move(refs), .self = hasSelfReference, @@ -54,7 +54,7 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto fullCaOpt = contentAddressWithReferenences(); + auto fullCaOpt = contentAddressWithReferences(); if (! fullCaOpt) return false; diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 663d94540..97eb6638b 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,6 +29,7 @@ struct ValidPathInfo { StorePath path; std::optional<StorePath> deriver; + // TODO document this Hash narHash; StorePathSet references; time_t registrationTime = 0; @@ -77,7 +78,7 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - std::optional<ContentAddressWithReferences> contentAddressWithReferenences() const; + std::optional<ContentAddressWithReferences> contentAddressWithReferences() const; /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 3e4188188..c551c5f3e 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -280,16 +280,24 @@ std::string optimisticLockProfile(const Path & profile) } +Path profilesDir() +{ + auto profileRoot = createNixStateDir() + "/profiles"; + createDirs(profileRoot); + return profileRoot; +} + + Path getDefaultProfile() { - Path profileLink = getHome() + "/.nix-profile"; + Path profileLink = settings.useXDGBaseDirectories ? createNixStateDir() + "/profile" : getHome() + "/.nix-profile"; try { + auto profile = + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : profilesDir() + "/profile"; if (!pathExists(profileLink)) { - replaceSymlink( - getuid() == 0 - ? settings.nixStateDir + "/profiles/default" - : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), - profileLink); + replaceSymlink(profile, profileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); } catch (Error &) { diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 408ca039c..fbf95b850 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -68,8 +68,13 @@ void lockProfile(PathLocks & lock, const Path & profile); rebuilt. */ std::string optimisticLockProfile(const Path & profile); -/* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create - it. */ +/* Creates and returns the path to a directory suitable for storing the user’s + profiles. */ +Path profilesDir(); + +/* Resolve the default profile (~/.nix-profile by default, $XDG_STATE_HOME/ + nix/profile if XDG Base Directory Support is enabled), and create if doesn't + exist */ Path getDefaultProfile(); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5f1bb73ac..e19d512c0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -266,6 +266,7 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); overrides.erase(settings.experimentalFeatures.name); + overrides.erase(settings.pluginFiles.name); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 844553ad3..8d76eee99 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -238,7 +238,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual void init() override { - if (auto cacheInfo = diskCache->cacheExists(getUri())) { + if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { wantMassQuery.setDefault(cacheInfo->wantMassQuery); priority.setDefault(cacheInfo->priority); } else { diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 353dff9fa..871f2f3be 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -41,6 +41,15 @@ SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int ex throw SQLiteError(path, errMsg, err, exterr, offset, std::move(hf)); } +static void traceSQL(void * x, const char * sql) +{ + // wacky delimiters: + // so that we're quite unambiguous without escaping anything + // notice instead of trace: + // so that this can be enabled without getting the firehose in our face. + notice("SQL<[%1%]>", sql); +}; + SQLite::SQLite(const Path & path, bool create) { // useSQLiteWAL also indicates what virtual file system we need. Using @@ -58,6 +67,11 @@ SQLite::SQLite(const Path & path, bool create) if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) SQLiteError::throw_(db, "setting timeout"); + if (getEnv("NIX_DEBUG_SQLITE_TRACES") == "1") { + // To debug sqlite statements; trace all of them + sqlite3_trace(db, &traceSQL, nullptr); + } + exec("pragma foreign_keys = 1"); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3c0c26706..b8a77b324 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -99,10 +99,12 @@ StorePath Store::followLinksToStorePath(std::string_view path) const silly, but it's done that way for compatibility). <id> is the name of the output (usually, "out"). - <h2> = base-16 representation of a SHA-256 hash of: + <h2> = base-16 representation of a SHA-256 hash of <s2> + + <s2> = if <type> = "text:...": the string written to the resulting store path - if <type> = "source": + if <type> = "source:...": the serialisation of the path from which this store path is copied, as returned by hashPath() if <type> = "output:<id>": @@ -164,8 +166,8 @@ StorePath Store::makeOutputPath(std::string_view id, /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ + hacky, but we can't put them in, say, <s2> (per the grammar above) + since that would be ambiguous. */ static std::string makeType( const Store & store, std::string && type, @@ -182,15 +184,15 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", info.references), info.hash, name); + if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(info.method) - + info.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.hash.method) + + info.hash.hash.to_string(Base16, true) + ":"), name); } } @@ -198,13 +200,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.type == htSHA256); + assert(info.hash.hash.type == htSHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, .self = false, }), - info.hash, + info.hash.hash, name); } @@ -230,7 +232,7 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); FixedOutputInfo caInfo { - { + .hash = { .method = method, .hash = h, }, @@ -439,7 +441,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, *this, name, FixedOutputInfo { - { + .hash = { .method = method, .hash = hash, }, @@ -762,7 +764,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(path, {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { auto state(state_.lock()); try { auto info = fut.get(); @@ -996,7 +998,7 @@ void copyStorePath( auto info2 = make_ref<ValidPathInfo>(*info); info2->path = dstStore.makeFixedOutputPathFromCA( info->path.name(), - info->contentAddressWithReferenences().value()); + info->contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1110,7 +1112,7 @@ std::map<StorePath, StorePath> copyPaths( if (currentPathInfo.ca && currentPathInfo.references.empty()) { storePathForDst = dstStore.makeFixedOutputPathFromCA( currentPathInfo.path.name(), - currentPathInfo.contentAddressWithReferenences().value()); + currentPathInfo.contentAddressWithReferences().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePathForSrc); if (storePathForDst != storePathForSrc) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2d252db84..72517c6e4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -653,9 +653,6 @@ public: return toRealPath(printStorePath(storePath)); } - virtual void createUser(const std::string & userName, uid_t userId) - { } - /* * Synchronises the options of the client with those of the daemon * (a no-op when there’s no daemon) diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc new file mode 100644 index 000000000..4dd14dcce --- /dev/null +++ b/src/libstore/tests/derivation.cc @@ -0,0 +1,143 @@ +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> + +#include "derivations.hh" + +#include "tests/libstore.hh" + +namespace nix { + +class DerivationTest : public LibStoreTest +{ +}; + +#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \ + TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \ + } + +TEST_JSON(DerivationOutput, inputAddressed, + R"({ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + })", + (DerivationOutput::InputAddressed { + .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, caFixed, + R"({ + "hashAlgo": "r:sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .ca = FixedOutputInfo { + .hash = { + .method = FileIngestionMethod::Recursive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + .references = {}, + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, caFixedText, + R"({ + "hashAlgo": "text:sha256", + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" + })", + (DerivationOutput::CAFixed { + .ca = TextInfo { + .hash = { + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + .references = {}, + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, caFloating, + R"({ + "hashAlgo": "r:sha256" + })", + (DerivationOutput::CAFloating { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, deferred, + R"({ })", + DerivationOutput::Deferred { }, + "drv-name", "output-name") + +TEST_JSON(DerivationOutput, impure, + R"({ + "hashAlgo": "r:sha256", + "impure": true + })", + (DerivationOutput::Impure { + .method = FileIngestionMethod::Recursive, + .hashType = htSHA256, + }), + "drv-name", "output-name") + +TEST_JSON(Derivation, impure, + R"({ + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [ + "cat", + "dog" + ] + }, + "system": "wasm-sel4", + "builder": "foo", + "args": [ + "bar", + "baz" + ], + "env": { + "BIG_BAD": "WOLF" + }, + "outputs": {} + })", + ({ + Derivation drv; + drv.inputSrcs = { + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + { + store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + "cat", + "dog", + }, + } + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + drv; + })) + +#undef TEST_JSON + +} diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc new file mode 100644 index 000000000..d1ac2c5e7 --- /dev/null +++ b/src/libstore/tests/derived-path.cc @@ -0,0 +1,62 @@ +#include <regex> + +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> + +#include "tests/derived-path.hh" +#include "tests/libstore.hh" + +namespace rc { +using namespace nix; + +Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary<StorePath>(), + }); +} + +Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = *gen::arbitrary<StorePath>(), + .outputs = *gen::arbitrary<OutputsSpec>(), + }); +} + +Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary() +{ + switch (*gen::inRange<uint8_t>(0, 1)) { + case 0: + return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Opaque>()); + default: + return gen::just<DerivedPath>(*gen::arbitrary<DerivedPath::Built>()); + } +} + +} + +namespace nix { + +class DerivedPathTest : public LibStoreTest +{ +}; + +// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is +// no a real fixture. +// +// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args +TEST_F(DerivedPathTest, force_init) +{ +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, + prop_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store))); +} + +} diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh new file mode 100644 index 000000000..3bc812440 --- /dev/null +++ b/src/libstore/tests/derived-path.hh @@ -0,0 +1,28 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <derived-path.hh> + +#include "tests/path.hh" +#include "tests/outputs-spec.hh" + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<DerivedPath::Opaque> { + static Gen<DerivedPath::Opaque> arbitrary(); +}; + +template<> +struct Arbitrary<DerivedPath::Built> { + static Gen<DerivedPath::Built> arbitrary(); +}; + +template<> +struct Arbitrary<DerivedPath> { + static Gen<DerivedPath> arbitrary(); +}; + +} diff --git a/src/libstore/tests/libstoretests.hh b/src/libstore/tests/libstore.hh index 05397659b..05397659b 100644 --- a/src/libstore/tests/libstoretests.hh +++ b/src/libstore/tests/libstore.hh diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk index a2cf8a0cf..03becc7d1 100644 --- a/src/libstore/tests/local.mk +++ b/src/libstore/tests/local.mk @@ -1,6 +1,20 @@ -check: libstore-tests_RUN +check: libstore-tests-exe_RUN -programs += libstore-tests +programs += libstore-tests-exe + +libstore-tests-exe_NAME = libnixstore-tests + +libstore-tests-exe_DIR := $(d) + +libstore-tests-exe_INSTALL_DIR := + +libstore-tests-exe_LIBS = libstore-tests + +libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) + +libraries += libstore-tests + +libstore-tests_NAME = libnixstore-tests libstore-tests_DIR := $(d) @@ -10,6 +24,6 @@ libstore-tests_SOURCES := $(wildcard $(d)/*.cc) libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil -libstore-tests_LIBS = libstore libutil +libstore-tests_LIBS = libutil-tests libstore libutil libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/src/libstore/tests/nar-info-disk-cache.cc new file mode 100644 index 000000000..b4bdb8329 --- /dev/null +++ b/src/libstore/tests/nar-info-disk-cache.cc @@ -0,0 +1,123 @@ +#include "nar-info-disk-cache.hh" + +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> +#include "sqlite.hh" +#include <sqlite3.h> + + +namespace nix { + +TEST(NarInfoDiskCacheImpl, create_and_read) { + // This is a large single test to avoid some setup overhead. + + int prio = 12345; + bool wantMassQuery = true; + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir); + Path dbPath(tmpDir + "/test-narinfo-disk-cache.sqlite"); + + int savedId; + int barId; + SQLite db; + SQLiteStmt getIds; + + { + auto cache = getTestNarInfoDiskCache(dbPath); + + // Set up "background noise" and check that different caches receive different ids + { + auto bc1 = cache->createCache("https://bar", "/nix/storedir", wantMassQuery, prio); + auto bc2 = cache->createCache("https://xyz", "/nix/storedir", false, 12); + ASSERT_NE(bc1, bc2); + barId = bc1; + } + + // Check that the fields are saved and returned correctly. This does not test + // the select statement yet, because of in-memory caching. + savedId = cache->createCache("http://foo", "/nix/storedir", wantMassQuery, prio);; + { + auto r = cache->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(savedId, r->id); + } + + // We're going to pay special attention to the id field because we had a bug + // that changed it. + db = SQLite(dbPath); + getIds.create(db, "select id from BinaryCaches where url = 'http://foo'"); + + { + auto q(getIds.use()); + ASSERT_TRUE(q.next()); + ASSERT_EQ(savedId, q.getInt(0)); + ASSERT_FALSE(q.next()); + } + + // Pretend that the caches are older, but keep one up to date, as "background noise" + db.exec("update BinaryCaches set timestamp = timestamp - 1 - 7 * 24 * 3600 where url <> 'https://xyz';"); + + // This shows that the in-memory cache works + { + auto r = cache->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + } + } + + { + // We can't clear the in-memory cache, so we use a new cache object. This is + // more realistic anyway. + auto cache2 = getTestNarInfoDiskCache(dbPath); + + { + auto r = cache2->upToDateCacheExists("http://foo"); + ASSERT_FALSE(r); + } + + // "Update", same data, check that the id number is reused + cache2->createCache("http://foo", "/nix/storedir", wantMassQuery, prio); + + { + auto r = cache2->upToDateCacheExists("http://foo"); + ASSERT_TRUE(r); + ASSERT_EQ(r->priority, prio); + ASSERT_EQ(r->wantMassQuery, wantMassQuery); + ASSERT_EQ(r->id, savedId); + } + + { + auto q(getIds.use()); + ASSERT_TRUE(q.next()); + auto currentId = q.getInt(0); + ASSERT_FALSE(q.next()); + ASSERT_EQ(currentId, savedId); + } + + // Check that the fields can be modified, and the id remains the same + { + auto r0 = cache2->upToDateCacheExists("https://bar"); + ASSERT_FALSE(r0); + + cache2->createCache("https://bar", "/nix/storedir", !wantMassQuery, prio + 10); + auto r = cache2->upToDateCacheExists("https://bar"); + ASSERT_EQ(r->wantMassQuery, !wantMassQuery); + ASSERT_EQ(r->priority, prio + 10); + ASSERT_EQ(r->id, barId); + } + + // // Force update (no use case yet; we only retrieve cache metadata when stale based on timestamp) + // { + // cache2->createCache("https://bar", "/nix/storedir", wantMassQuery, prio + 20); + // auto r = cache2->upToDateCacheExists("https://bar"); + // ASSERT_EQ(r->wantMassQuery, wantMassQuery); + // ASSERT_EQ(r->priority, prio + 20); + // } + } +} + +} diff --git a/src/libstore/tests/outputs-spec.cc b/src/libstore/tests/outputs-spec.cc index 06e4cabbd..984d1d963 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/src/libstore/tests/outputs-spec.cc @@ -2,6 +2,7 @@ #include <nlohmann/json.hpp> #include <gtest/gtest.h> +#include <rapidcheck/gtest.h> namespace nix { @@ -199,3 +200,34 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl #undef TEST_JSON } + +namespace rc { +using namespace nix; + +Gen<OutputsSpec> Arbitrary<OutputsSpec>::arbitrary() +{ + switch (*gen::inRange<uint8_t>(0, 1)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + default: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container<StringSet>(gen::map( + gen::arbitrary<StorePathName>(), + [](StorePathName n) { return n.name; }))), + }); + } +} + +} + +namespace nix { + +RC_GTEST_PROP( + OutputsSpec, + prop_round_rip, + (const OutputsSpec & o)) +{ + RC_ASSERT(o == OutputsSpec::parse(o.to_string())); +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/src/libstore/tests/outputs-spec.hh new file mode 100644 index 000000000..2d455c817 --- /dev/null +++ b/src/libstore/tests/outputs-spec.hh @@ -0,0 +1,17 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <outputs-spec.hh> + +#include <tests/path.hh> + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<OutputsSpec> { + static Gen<OutputsSpec> arbitrary(); +}; + +} diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index 8ea252c92..430aa0099 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -7,7 +7,9 @@ #include "path-regex.hh" #include "store-api.hh" -#include "libstoretests.hh" +#include "tests/hash.hh" +#include "tests/libstore.hh" +#include "tests/path.hh" namespace nix { @@ -73,17 +75,14 @@ void showValue(const StorePath & p, std::ostream & os) { namespace rc { using namespace nix; -template<> -struct Arbitrary<StorePath> { - static Gen<StorePath> arbitrary(); -}; - -Gen<StorePath> Arbitrary<StorePath>::arbitrary() +Gen<StorePathName> Arbitrary<StorePathName>::arbitrary() { - auto len = *gen::inRange<size_t>(1, StorePath::MaxPathLen); + auto len = *gen::inRange<size_t>( + 1, + StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); - std::string pre { HASH_PART "-" }; - pre.reserve(pre.size() + len); + std::string pre; + pre.reserve(len); for (size_t c = 0; c < len; ++c) { switch (auto i = *gen::inRange<uint8_t>(0, 10 + 2 * 26 + 6)) { @@ -118,7 +117,17 @@ Gen<StorePath> Arbitrary<StorePath>::arbitrary() } } - return gen::just(StorePath { pre }); + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen<StorePath> Arbitrary<StorePath>::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary<Hash>(), + (*gen::arbitrary<StorePathName>()).name, + }); } } // namespace rc diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh new file mode 100644 index 000000000..d7f1a8988 --- /dev/null +++ b/src/libstore/tests/path.hh @@ -0,0 +1,28 @@ +#pragma once + +#include <rapidcheck/gen/Arbitrary.h> + +#include <path.hh> + +namespace nix { + +struct StorePathName { + std::string name; +}; + +} + +namespace rc { +using namespace nix; + +template<> +struct Arbitrary<StorePathName> { + static Gen<StorePathName> arbitrary(); +}; + +template<> +struct Arbitrary<StorePath> { + static Gen<StorePath> arbitrary(); +}; + +} |