From 5c74a6147b4b81dc5b173f190f02f6681ec4b0fe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 11 Oct 2020 17:07:14 +0000 Subject: Properly type the derivation and substitution goal maps As a bonus, Worker::removeGoal is less inefficient. --- src/libstore/build.hh | 11 +++++------ src/libstore/build/worker.cc | 33 +++++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build.hh b/src/libstore/build.hh index 8027c61b1..894feab5b 100644 --- a/src/libstore/build.hh +++ b/src/libstore/build.hh @@ -28,7 +28,6 @@ struct HookInstance; /* A pointer to a goal. */ struct Goal; -class DerivationGoal; typedef std::shared_ptr GoalPtr; typedef std::weak_ptr WeakGoalPtr; @@ -140,6 +139,8 @@ struct Child steady_time_point timeStarted; }; +class DerivationGoal; +class SubstitutionGoal; /* The worker class. */ class Worker @@ -167,8 +168,8 @@ private: /* Maps used to prevent multiple instantiations of a goal for the same derivation / path. */ - WeakGoalMap derivationGoals; - WeakGoalMap substitutionGoals; + std::map> derivationGoals; + std::map> substitutionGoals; /* Goals waiting for busy paths to be unlocked. */ WeakGoals waitingForAnyGoal; @@ -242,7 +243,7 @@ public: const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); /* substitution goal */ - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); + std::shared_ptr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -305,8 +306,6 @@ public: typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -class SubstitutionGoal; - /* Unless we are repairing, we don't both to test validity and just assume it, so the choices are `Absent` or `Valid`. */ enum struct PathStatus { diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 2fc9f6982..47403580e 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -39,16 +39,13 @@ std::shared_ptr Worker::makeDerivationGoalCommon( const StringSet & wantedOutputs, std::function()> mkDrvGoal) { - WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath]; - GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME - std::shared_ptr goal; - if (!abstract_goal) { + std::weak_ptr & goal_weak = derivationGoals[drvPath]; + std::shared_ptr goal = goal_weak.lock(); + if (!goal) { goal = mkDrvGoal(); - abstract_goal_weak = goal; + goal_weak = goal; wakeUp(goal); } else { - goal = std::dynamic_pointer_cast(abstract_goal); - assert(goal); goal->addWantedOutputs(wantedOutputs); } return goal; @@ -73,10 +70,10 @@ std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) +std::shared_ptr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional ca) { - WeakGoalPtr & goal_weak = substitutionGoals[path]; - GoalPtr goal = goal_weak.lock(); // FIXME + std::weak_ptr & goal_weak = substitutionGoals[path]; + auto goal = goal_weak.lock(); // FIXME if (!goal) { goal = std::make_shared(path, *this, repair, ca); goal_weak = goal; @@ -85,14 +82,14 @@ GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, return goal; } - -static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) +template +static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - for (WeakGoalMap::iterator i = goalMap.begin(); + for (typename std::map>::iterator i = goalMap.begin(); i != goalMap.end(); ) if (i->second.lock() == goal) { - WeakGoalMap::iterator j = i; ++j; + typename std::map>::iterator j = i; ++j; goalMap.erase(i); i = j; } @@ -102,8 +99,12 @@ static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) void Worker::removeGoal(GoalPtr goal) { - nix::removeGoal(goal, derivationGoals); - nix::removeGoal(goal, substitutionGoals); + if (auto drvGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(drvGoal, derivationGoals); + else if (auto subGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(subGoal, substitutionGoals); + else + assert(false); if (topGoals.find(goal) != topGoals.end()) { topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals -- cgit v1.2.3 From 55592b253f3dddb121c1072ca584e95c37729b6d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 13 Oct 2020 18:04:24 +0000 Subject: Add some more docs --- src/libstore/build/worker.hh | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 07c0c0f16..f8bacc514 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -11,6 +11,11 @@ namespace nix { class DerivationGoal; class SubstitutionGoal; +/* Workaround for not being able to declare a something like + + class SubstitutionGoal : public Goal; + + even when Goal is a complete type; */ GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; -- cgit v1.2.3 From 11882d7c7ce3b6dc51dd7c0536f9662dc254ac0a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 Oct 2020 12:20:58 +0200 Subject: Create /etc/passwd *after* figuring out the sandbox uid/gid Fixes build failures like # nix log /nix/store/gjaa0psfcmqvw7ivggsncx9w364p3s8s-sshd.conf-validated.drv No user exists for uid 30012 --- src/libstore/build/derivation-goal.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 241e52116..fda05f0e9 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1420,12 +1420,6 @@ void DerivationGoal::startBuilder() Samba-in-QEMU. */ createDirs(chrootRootDir + "/etc"); - writeFile(chrootRootDir + "/etc/passwd", fmt( - "root:x:0:0:Nix build user:%3%:/noshell\n" - "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" - "nobody:x:65534:65534:Nobody:/:/noshell\n", - sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - /* Declare the build user's group so that programs get a consistent view of the system (e.g., "id -gn"). */ writeFile(chrootRootDir + "/etc/group", @@ -1730,6 +1724,14 @@ void DerivationGoal::startBuilder() throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces"); } + /* Now that we now the sandbox uid, we can write + /etc/passwd. */ + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); + /* Save the mount namespace of the child. We have to do this *before* the child does a chroot. */ sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); -- cgit v1.2.3 From e6f8ae56d82813edbcf09fda58305de47369f964 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 17 Oct 2020 21:45:31 +0000 Subject: tab -> space --- src/libstore/build/worker.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 97ffb9a34..4d3df26f3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -455,7 +455,7 @@ void Worker::markContentsGood(const StorePath & path) GoalPtr upcast_goal(std::shared_ptr subGoal) { - return subGoal; + return subGoal; } } -- cgit v1.2.3 From 57d0432b395cc4d70792d2df5794ff2e0dd02d3d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 17 Oct 2020 21:47:52 +0000 Subject: Just use `auto` in two places. --- src/libstore/build/worker.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 4d3df26f3..17c10cd71 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -90,10 +90,10 @@ template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - for (typename std::map>::iterator i = goalMap.begin(); + for (auto i = goalMap.begin(); i != goalMap.end(); ) if (i->second.lock() == goal) { - typename std::map>::iterator j = i; ++j; + auto j = i; ++j; goalMap.erase(i); i = j; } -- cgit v1.2.3 From 7ed46c15744461534478e2ed0aa25a2b2e536c6f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 17 Oct 2020 21:50:12 +0000 Subject: Explain that `upcast_goal` is still a static cast --- src/libstore/build/worker.hh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index f8bacc514..3a53a8def 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -15,7 +15,11 @@ class SubstitutionGoal; class SubstitutionGoal : public Goal; - even when Goal is a complete type; */ + even when Goal is a complete type. + + This is still a static cast. The purpose of exporting it is to define it in + a place where `SubstitutionGoal` is concrete, and use it in a place where it + is opaque. */ GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; -- cgit v1.2.3 From 94f1e4a441ee64bcc7a961d941ec901de750d880 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 18 Oct 2020 14:26:23 +0200 Subject: Typo --- src/libstore/build/goal.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 360c160ce..0781a9d38 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -46,7 +46,7 @@ struct Goal : public std::enable_shared_from_this unsigned int nrNoSubstituters; /* Number of substitution goals we are/were waiting for that - failed because othey had unsubstitutable references. */ + failed because they had unsubstitutable references. */ unsigned int nrIncompleteClosure; /* Name of this goal for debugging purposes. */ -- cgit v1.2.3 From bd9eb5c743faf1b3c33f4e1c2ccf317977d4be9d Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 18 Oct 2020 14:21:53 +0200 Subject: DerivationGoal: only retry if output closure incomplete is only problem --- src/libstore/build/derivation-goal.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index fda05f0e9..1c9217537 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -330,8 +330,13 @@ void DerivationGoal::outputsSubstitutionTried() /* If the substitutes form an incomplete closure, then we should build the dependencies of this derivation, but after that, we - can still use the substitutes for this derivation itself. */ - if (nrIncompleteClosure > 0) retrySubstitution = true; + can still use the substitutes for this derivation itself. + + If the nrIncompleteClosure != nrFailed, we have another issue as well. + In particular, it may be the case that the hole in the closure is + an output of the current derivation, which causes a loop if retried. + */ + if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true; nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; -- cgit v1.2.3 From 461cf2b85601e4510bff303e454059d80b9df8c0 Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Mon, 19 Oct 2020 23:08:50 +0200 Subject: Add NIX_CONFIG env var for applying nix.conf overrides --- src/libstore/globals.cc | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 1238dc530..4df68d0c9 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -86,6 +86,12 @@ void loadConfFile() for (auto file = files.rbegin(); file != files.rend(); file++) { globalConfig.applyConfigFile(*file); } + + auto nixConfEnv = getEnv("NIX_CONFIG"); + if (nixConfEnv.has_value()) { + globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); + } + } std::vector getUserConfigFiles() -- cgit v1.2.3 From c092fa4702215fdb61611c5dd28194401d056170 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 23 Sep 2020 16:30:42 +0200 Subject: Allow non-CA derivations to depend on CA derivations --- src/libstore/build/derivation-goal.cc | 12 ++++++- src/libstore/derivations.cc | 65 +++++++++++++++++++++++++++++++---- src/libstore/derivations.hh | 14 ++++++-- src/libstore/local-store.cc | 4 ++- 4 files changed, 84 insertions(+), 11 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index db0c2bb6c..cc8737fd5 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -493,7 +493,8 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) { + if ((!fullDrv.inputDrvs.empty() && + fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) { /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ @@ -3166,6 +3167,15 @@ void DerivationGoal::registerOutputs() [&](DerivationOutputCAFloating dof) { return newInfoFromCA(dof); }, + [&](DerivationOutputDeferred) { + // No derivation should reach that point without having been + // rewritten first + assert(false); + // Ugly, but the compiler insists on having this return a value + // of type `ValidPathInfo` despite the `assert(false)`, so + // let's provide it + return *(ValidPathInfo*)0; + }, }, output.output); /* Calculate where we'll move the output files. In the checking case we diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 07b4e772b..3b3a25391 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -21,6 +21,9 @@ std::optional DerivationOutput::path(const Store & store, std::string [](DerivationOutputCAFloating dof) -> std::optional { return std::nullopt; }, + [](DerivationOutputDeferred) -> std::optional { + return std::nullopt; + }, }, output); } @@ -37,6 +40,7 @@ bool derivationIsCA(DerivationType dt) { case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return true; + case DerivationType::DeferredInputAddressed: return false; }; // Since enums can have non-variant values, but making a `default:` would // disable exhaustiveness warnings. @@ -48,6 +52,7 @@ bool derivationIsFixed(DerivationType dt) { case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return false; + case DerivationType::DeferredInputAddressed: return false; }; assert(false); } @@ -57,6 +62,7 @@ bool derivationIsImpure(DerivationType dt) { case DerivationType::InputAddressed: return false; case DerivationType::CAFixed: return true; case DerivationType::CAFloating: return false; + case DerivationType::DeferredInputAddressed: return false; }; assert(false); } @@ -180,6 +186,11 @@ static DerivationOutput parseDerivationOutput(const Store & store, }; } } else { + if (pathS == "") { + return DerivationOutput { + .output = DerivationOutputDeferred { } + }; + } validatePath(pathS); return DerivationOutput { .output = DerivationOutputInputAddressed { @@ -325,6 +336,11 @@ string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, + [&](DerivationOutputDeferred) { + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, ""); + } }, i.second.output); s += ')'; } @@ -389,7 +405,7 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName DerivationType BasicDerivation::type() const { - std::set inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs; + std::set inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs; std::optional floatingHashType; for (auto & i : outputs) { std::visit(overloaded { @@ -408,22 +424,27 @@ DerivationType BasicDerivation::type() const throw Error("All floating outputs must use the same hash type"); } }, + [&](DerivationOutputDeferred _) { + deferredIAOutputs.insert(i.first); + }, }, i.second.output); } - if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { throw Error("Must have at least one output"); - } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { return DerivationType::InputAddressed; - } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) { + } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { if (fixedCAOutputs.size() > 1) // FIXME: Experimental feature? throw Error("Only one fixed output is allowed for now"); if (*fixedCAOutputs.begin() != "out") throw Error("Single fixed output must be named \"out\""); return DerivationType::CAFixed; - } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) { + } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) { return DerivationType::CAFloating; + } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) { + return DerivationType::DeferredInputAddressed; } else { throw Error("Can't mix derivation output types"); } @@ -454,6 +475,8 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath return h->second; } +UnknownHashes unknownHashes; + /* See the header for interface details. These are the implementation details. For fixed-output derivations, each hash in the map is not the @@ -476,7 +499,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { case DerivationType::CAFloating: - throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations"); + return unknownHashes; case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { @@ -491,12 +514,15 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } case DerivationType::InputAddressed: break; + case DerivationType::DeferredInputAddressed: + break; } /* For other derivations, replace the inputs paths with recursive calls to this function. */ std::map inputs2; for (auto & i : drv.inputDrvs) { + bool hasUnknownHash = false; const auto & res = pathDerivationModulo(store, i.first); std::visit(overloaded { // Regular non-CA derivation, replace derivation @@ -514,7 +540,13 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m justOut); } }, + [&](UnknownHashes) { + hasUnknownHash = true; + }, }, res); + if (hasUnknownHash) { + return unknownHashes; + } } return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); @@ -620,6 +652,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << ""; }, + [&](DerivationOutputDeferred) { + out << "" + << "" + << ""; + }, }, i.second.output); } worker_proto::write(store, out, drv.inputSrcs); @@ -645,7 +682,6 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath } -// N.B. Outputs are left unchanged static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { debug("Rewriting the derivation"); @@ -666,6 +702,21 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String newEnv.emplace(envName, envValue); } drv.env = newEnv; + + auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); + for (auto & [outputName, output] : drv.outputs) { + if (std::holds_alternative(output.output)) { + Hash h = std::get(hashModulo); + auto outPath = store.makeOutputPath(outputName, h, drv.name); + drv.env[outputName] = store.printStorePath(outPath); + output = DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(outPath), + }, + }; + } + } + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 6d292b2e5..f9ba935e6 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -41,12 +41,18 @@ struct DerivationOutputCAFloating HashType hashType; }; +/* Input-addressed output which depends on a (CA) derivation whose hash isn't + * known atm + */ +struct DerivationOutputDeferred {}; + struct DerivationOutput { std::variant< DerivationOutputInputAddressed, DerivationOutputCAFixed, - DerivationOutputCAFloating + DerivationOutputCAFloating, + DerivationOutputDeferred > output; std::optional hashAlgoOpt(const Store & store) const; /* Note, when you use this function you should make sure that you're passing @@ -72,6 +78,7 @@ typedef std::map StringPairs; enum struct DerivationType : uint8_t { InputAddressed, + DeferredInputAddressed, CAFixed, CAFloating, }; @@ -167,9 +174,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName // whose output hashes are always known since they are fixed up-front. typedef std::map CaOutputHashes; +struct UnknownHashes {}; + typedef std::variant< Hash, // regular DRV normalized hash - CaOutputHashes + CaOutputHashes, // Fixed-output derivation hashes + UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies > DrvHashModulo; /* Returns hashes with the details of fixed-output subderivations diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d29236a9c..d29a68179 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -573,6 +573,8 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat [&](DerivationOutputCAFloating _) { /* Nothing to check */ }, + [&](DerivationOutputDeferred) { + }, }, i.second.output); } } @@ -817,7 +819,7 @@ std::map> LocalStore::queryPartialDerivati } /* can't just use else-if instead of `!haveCached` because we need to unlock `drvPathResolutions` before it is locked in `Derivation::resolve`. */ - if (!haveCached && drv.type() == DerivationType::CAFloating) { + if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) { /* Try resolve drv and use that path instead. */ auto attempt = drv.tryResolve(*this); if (!attempt) -- cgit v1.2.3 From bc081bcd816542d66f1578788b93df4d7e07b135 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 24 Sep 2020 10:11:58 +0200 Subject: Inline `unkownHashes` See https://github.com/NixOS/nix/pull/4056#discussion_r493661632 --- src/libstore/derivations.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 3b3a25391..517ecfaa2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -475,8 +475,6 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath return h->second; } -UnknownHashes unknownHashes; - /* See the header for interface details. These are the implementation details. For fixed-output derivations, each hash in the map is not the @@ -499,7 +497,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { case DerivationType::CAFloating: - return unknownHashes; + return UnknownHashes {}; case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { @@ -545,7 +543,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m }, }, res); if (hasUnknownHash) { - return unknownHashes; + return UnknownHashes {}; } } -- cgit v1.2.3 From 869c0321ff4829f56e10316f401a0f9268c1a8b0 Mon Sep 17 00:00:00 2001 From: stev Date: Thu, 29 Oct 2020 00:33:14 +0100 Subject: Alter "wanted:" to "specified:" in hash mismatch output This makes it even clearer which of the two hashes was specified in the nix files. Some may think that "wanted" and "got" is obvious, but: "got" could mean "got in nix file" and "wanted" could mean "want to see in nix file". --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/local-store.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index cc8737fd5..7db83c8be 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3157,7 +3157,7 @@ void DerivationGoal::registerOutputs() valid. */ worker.hashMismatch = true; delayedException = std::make_exception_ptr( - BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", + BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", worker.store.printStorePath(drvPath), wanted.to_string(SRI, true), got.to_string(SRI, true))); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d29a68179..bfad8fb21 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1085,11 +1085,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto hashResult = hashSink->finish(); if (hashResult.first != info.narHash) - throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", + throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) - throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", + throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), info.narSize, hashResult.second); autoGC(); -- cgit v1.2.3 From ff4dea63c9403880500f82ce273713ecf793d2d9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 29 Oct 2020 18:17:39 +0100 Subject: Generalize extra-* settings This removes the extra-substituters and extra-sandbox-paths settings and instead makes every array setting extensible by setting "extra- = " in the configuration file or passing "-- " on the command line. --- src/libstore/build/derivation-goal.cc | 6 +----- src/libstore/daemon.cc | 2 -- src/libstore/globals.cc | 9 +++++++-- src/libstore/globals.hh | 21 +-------------------- src/libstore/store-api.cc | 3 --- 5 files changed, 9 insertions(+), 32 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7db83c8be..3dacb218c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1332,13 +1332,9 @@ void DerivationGoal::startBuilder() /* Allow a user-configurable set of directories from the host file system. */ - PathSet dirs = settings.sandboxPaths; - PathSet dirs2 = settings.extraSandboxPaths; - dirs.insert(dirs2.begin(), dirs2.end()); - dirsInChroot.clear(); - for (auto i : dirs) { + for (auto i : settings.sandboxPaths.get()) { if (i.empty()) continue; bool optional = false; if (i[i.size() - 1] == '?') { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 99d8add92..4dbc7ba38 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -231,8 +231,6 @@ struct ClientSettings settings.set(name, value); else if (setSubstituters(settings.substituters)) ; - else if (setSubstituters(settings.extraSubstituters)) - ; else debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); } catch (UsageError & e) { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4df68d0c9..f38601d6d 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -160,7 +160,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smDisabled, false}, }); -template<> void BaseSetting::set(const std::string & str) +template<> void BaseSetting::set(const std::string & str, bool append) { if (str == "true") value = smEnabled; else if (str == "relaxed") value = smRelaxed; @@ -168,6 +168,11 @@ template<> void BaseSetting::set(const std::string & str) else throw UsageError("option '%s' has invalid value '%s'", name, str); } +template<> bool BaseSetting::isAppendable() +{ + return false; +} + template<> std::string BaseSetting::to_string() const { if (value == smEnabled) return "true"; @@ -198,7 +203,7 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s }); } -void MaxBuildJobsSetting::set(const std::string & str) +void MaxBuildJobsSetting::set(const std::string & str, bool append) { if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); else if (!string2Int(str, value)) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8c63c5b34..eabd83e3f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -25,7 +25,7 @@ struct MaxBuildJobsSetting : public BaseSetting options->addSetting(this); } - void set(const std::string & str) override; + void set(const std::string & str, bool append = false) override; }; class Settings : public Config { @@ -413,14 +413,6 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting extraSandboxPaths{ - this, {}, "extra-sandbox-paths", - R"( - A list of additional paths appended to `sandbox-paths`. Useful if - you want to extend its default value. - )", - {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; - Setting buildRepeat{ this, 0, "repeat", R"( @@ -599,17 +591,6 @@ public: )", {"binary-caches"}}; - // FIXME: provide a way to add to option values. - Setting extraSubstituters{ - this, {}, "extra-substituters", - R"( - Additional binary caches appended to those specified in - `substituters`. When used by unprivileged users, untrusted - substituters (i.e. those not listed in `trusted-substituters`) are - silently ignored. - )", - {"extra-binary-caches"}}; - Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9f21f0434..83d3a1fa1 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1114,9 +1114,6 @@ std::list> getDefaultSubstituters() for (auto uri : settings.substituters.get()) addStore(uri); - for (auto uri : settings.extraSubstituters.get()) - addStore(uri); - stores.sort([](ref & a, ref & b) { return a->priority < b->priority; }); -- cgit v1.2.3 From 2192cac634dcd13c3365f4dfeb5f393f4fd9e327 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 30 Oct 2020 21:47:34 +0100 Subject: Fix RemoteStore pool deadlock in filterSource etc --- src/libstore/remote-store.cc | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 488270f48..ccbd3bb1d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -471,9 +471,14 @@ ref RemoteStore::addCAToStore( worker_proto::write(*this, conn->to, references); conn->to << repair; - conn.withFramedSink([&](Sink & sink) { - dump.drainInto(sink); - }); + // The dump source may invoke the store, so we need to make some room. + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + conn.withFramedSink([&](Sink & sink) { + dump.drainInto(sink); + }); + } auto path = parseStorePath(readString(conn->from)); return readValidPathInfo(conn, path); -- cgit v1.2.3 From e8a45d07bccc315de88b8434393082eee6e944a8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 31 Oct 2020 23:52:09 +0100 Subject: Restore RestrictedStore.addToStoreFromDump implementation It was accidentally removed in commit ca30abb3fb36440e5a13161c39647189036fc18f --- src/libstore/build/derivation-goal.cc | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3dacb218c..19d96dd8f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2074,6 +2074,14 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf return path; } + StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + { + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + goal.addDependency(path); + return path; + } + void narFromPath(const StorePath & path, Sink & sink) override { if (!goal.isAllowed(path)) -- cgit v1.2.3 From db5424bf09886afc1c81db36766522f68fc66ba8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Nov 2020 13:57:58 +0100 Subject: Don't send eval-related settings to the daemon --- src/libstore/remote-store.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 488270f48..fb52ca6d0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -11,6 +11,7 @@ #include "finally.hh" #include "logging.hh" #include "callback.hh" +#include "filetransfer.hh" namespace nix { @@ -171,7 +172,8 @@ void RemoteStore::setOptions(Connection & conn) if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { std::map overrides; - globalConfig.getSettings(overrides, true); + settings.getSettings(overrides, true); // libstore settings + fileTransferSettings.getSettings(overrides, true); overrides.erase(settings.keepFailed.name); overrides.erase(settings.keepGoing.name); overrides.erase(settings.tryFallback.name); -- cgit v1.2.3 From 7cf874c17d466d5cffdb0eb6215fcfe8930ed757 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 Nov 2020 18:46:44 +0100 Subject: Don't use readDerivation() in addValidPath() readDerivation() requires a valid path. Fixes #4210. --- src/libstore/local-store.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bfad8fb21..2892b0407 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -623,7 +623,10 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = readDerivation(info.path); + auto drv = parseDerivation( + *this, + readFile(Store::toRealPath(info.path)), + Derivation::nameFromPath(info.path)); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -1000,7 +1003,11 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, readDerivation(i.path)); + checkDerivationOutputs(i.path, + parseDerivation( + *this, + readFile(Store::toRealPath(i.path)), + Derivation::nameFromPath(i.path))); } /* Do a topological sort of the paths. This will throw an -- cgit v1.2.3 From 797a52e31d97381e0be7e0a80e43f8cb259f8a6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Nov 2020 12:26:53 +0100 Subject: Add FIXME --- src/libstore/local-store.hh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f1e2ab7f9..5e07261a6 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -90,6 +90,8 @@ private: std::unique_ptr publicKeys; }; + // FIXME: get rid of recursive_mutex, it hides recursive SQLite + // queries. Sync _state; public: -- cgit v1.2.3 From e8c379555fa0441c5ab83b8e5a3a106d69fe2507 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 3 Nov 2020 12:52:57 +0100 Subject: LocalStore: Get rid of recursive_mutex --- src/libstore/build/derivation-goal.cc | 6 +++--- src/libstore/local-store.cc | 23 ++++++++++++++++------- src/libstore/local-store.hh | 4 +--- src/libstore/path-info.hh | 2 +- 4 files changed, 21 insertions(+), 14 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 19d96dd8f..bf2bad62c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3245,7 +3245,7 @@ void DerivationGoal::registerOutputs() if (!oldInfo.ultimate) { oldInfo.ultimate = true; worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({ std::move(oldInfo) }); + worker.store.registerValidPaths({{oldInfo.path, oldInfo}}); } continue; @@ -3275,7 +3275,7 @@ void DerivationGoal::registerOutputs() isn't statically known so that we can safely unlock the path before the next iteration */ if (newInfo.ca) - worker.store.registerValidPaths({newInfo}); + worker.store.registerValidPaths({{newInfo.path, newInfo}}); infos.emplace(outputName, std::move(newInfo)); } @@ -3350,7 +3350,7 @@ void DerivationGoal::registerOutputs() { ValidPathInfos infos2; for (auto & [outputName, newInfo] : infos) { - infos2.push_back(newInfo); + infos2.insert_or_assign(newInfo.path, newInfo); } worker.store.registerValidPaths(infos2); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2892b0407..e4e404dca 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -7,6 +7,7 @@ #include "nar-info.hh" #include "references.hh" #include "callback.hh" +#include "topo-sort.hh" #include #include @@ -962,9 +963,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst void LocalStore::registerValidPath(const ValidPathInfo & info) { - ValidPathInfos infos; - infos.push_back(info); - registerValidPaths(infos); + registerValidPaths({{info.path, info}}); } @@ -982,7 +981,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) SQLiteTxn txn(state->db); StorePathSet paths; - for (auto & i : infos) { + for (auto & [_, i] : infos) { assert(i.narHash.type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); @@ -991,7 +990,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) paths.insert(i.path); } - for (auto & i : infos) { + for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); for (auto & j : i.references) state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); @@ -1000,7 +999,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) /* Check that the derivation outputs are correct. We can't do this in addValidPath() above, because the references might not be valid yet. */ - for (auto & i : infos) + for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). checkDerivationOutputs(i.path, @@ -1014,7 +1013,17 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) error if a cycle is detected and roll back the transaction. Cycles can only occur when a derivation has multiple outputs. */ - topoSortPaths(paths); + topoSort(paths, + {[&](const StorePath & path) { + auto i = infos.find(path); + return i == infos.end() ? StorePathSet() : i->second.references; + }}, + {[&](const StorePath & path, const StorePath & parent) { + return BuildError( + "cycle detected in the references of '%s' from '%s'", + printStorePath(path), + printStorePath(parent)); + }}); txn.commit(); }); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 5e07261a6..d4435220d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -90,9 +90,7 @@ private: std::unique_ptr publicKeys; }; - // FIXME: get rid of recursive_mutex, it hides recursive SQLite - // queries. - Sync _state; + Sync _state; public: diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 8ff5c466e..de87f8b33 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -107,6 +107,6 @@ struct ValidPathInfo virtual ~ValidPathInfo() { } }; -typedef list ValidPathInfos; +typedef std::map ValidPathInfos; } -- cgit v1.2.3 From 3a63fc6cd515fb009b17d864fede23a356832a5e Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 21 Oct 2020 21:31:19 +0200 Subject: Allow substituting paths when building remotely using `ssh-ng://` Until now, it was not possible to substitute missing paths from e.g. `https://cache.nixos.org` on a remote server when building on it using the new `ssh-ng` protocol. This is because every store implementation except legacy `ssh://` ignores the substitution flag passed to `Store::queryValidPaths` while the `legacy-ssh-store` substitutes the remote store using `cmdQueryValidPaths` when the remote store is opened with `nix-store --serve`. This patch slightly modifies the daemon protocol to allow passing an integer value suggesting whether to substitute missing paths during `wopQueryValidPaths`. To implement this on the daemon-side, the substitution logic from `nix-store --serve` has been moved into a protected method named `Store::substitutePaths` which gets currently called from `LocalStore::queryValidPaths` and `Store::queryValidPaths` if `maybeSubstitute` is `true`. Fixes #2770 --- src/libstore/daemon.cc | 11 ++++++++++- src/libstore/remote-store.cc | 3 +++ src/libstore/store-api.cc | 22 ++++++++++++++++++++++ src/libstore/store-api.hh | 5 +++++ src/libstore/worker-protocol.hh | 2 +- 5 files changed, 41 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4dbc7ba38..60cca4fda 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -274,8 +274,17 @@ static void performOp(TunnelLogger * logger, ref store, case wopQueryValidPaths: { auto paths = worker_proto::read(*store, from, Phantom {}); + + SubstituteFlag substitute = NoSubstitute; + if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { + substitute = readInt(from) ? Substitute : NoSubstitute; + } + logger->startWork(); - auto res = store->queryValidPaths(paths); + if (substitute) { + store->substitutePaths(paths); + } + auto res = store->queryValidPaths(paths, substitute); logger->stopWork(); worker_proto::write(*store, to, res); break; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b6f70057d..c27a0f278 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -259,6 +259,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute } else { conn->to << wopQueryValidPaths; worker_proto::write(*this, conn->to, paths); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { + conn->to << (settings.buildersUseSubstitutes ? 1 : 0); + } conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 83d3a1fa1..3129d6637 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -522,6 +522,28 @@ void Store::queryPathInfo(const StorePath & storePath, } +void Store::substitutePaths(const StorePathSet & paths) +{ + std::vector paths2; + for (auto & path : paths) + if (!path.isDerivation()) + paths2.push_back({path}); + uint64_t downloadSize, narSize; + StorePathSet willBuild, willSubstitute, unknown; + queryMissing(paths2, + willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willSubstitute.empty()) + try { + std::vector subs; + for (auto & p : willSubstitute) subs.push_back({p}); + buildPaths(subs); + } catch (Error & e) { + logWarning(e.info()); + } +} + + StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { struct State diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f77bc21d1..9a373b561 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -360,6 +360,11 @@ protected: public: + /* If requested, substitute missing paths. This + implements nix-copy-closure's --use-substitutes + flag. */ + void substitutePaths(const StorePathSet & paths); + /* Query which of the given paths is valid. Optionally, try to substitute missing paths. */ virtual StorePathSet queryValidPaths(const StorePathSet & paths, diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b3705578e..63bd6ea49 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x11a +#define PROTOCOL_VERSION 0x11b #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -- cgit v1.2.3 From 01db455733899728947a15f8d50942f10432d61e Mon Sep 17 00:00:00 2001 From: Jake Waksbaum Date: Mon, 16 Nov 2020 02:35:50 -0500 Subject: Fix deadlock in nix-store when max-connections=1 This fixes a bug I encountered where `nix-store -qR` will deadlock when the `--include-outputs` flag is passed and `max-connections=1`. The deadlock occurs because `RemoteStore::queryDerivationOutputs` takes the only connection from the connection pool and uses it to check the daemon version. If the version is new enough, it calls `Store::queryDerivationOutputs`, which eventually calls `RemoteStore::queryPartialDerivationOutputMap`, where we take another connection from the connection pool to check the version again. Because we still haven't released the connection from the caller, this waits for a connection to be available, causing a deadlock. This diff solves the issue by using `getProtocol` to check the protocol version in the caller `RemoteStore::queryDerivationOutputs`, which immediately frees the connection back to the pool before returning the protocol version. That way we've already freed the connection by the time we call `RemoteStore::queryPartialDerivationOutputMap`. --- src/libstore/remote-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b6f70057d..a392f3b8c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -409,10 +409,10 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) { - auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) { + if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { return Store::queryDerivationOutputs(path); } + auto conn(getConnection()); conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); -- cgit v1.2.3 From 7de21f6664ffd4a66e5ef70058a4c450985a1981 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 13 Nov 2020 17:00:32 +0100 Subject: Make the sql debug statements more useful Print the expanded sql query (with the variables bound to their value) rather than the original one in case of error --- src/libstore/sqlite.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 31a1f0cac..f5935ee5c 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -147,14 +147,14 @@ void SQLiteStmt::Use::exec() int r = step(); assert(r != SQLITE_ROW); if (r != SQLITE_DONE) - throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql)); + throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt))); } bool SQLiteStmt::Use::next() { int r = step(); if (r != SQLITE_DONE && r != SQLITE_ROW) - throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql)); + throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt))); return r == SQLITE_ROW; } -- cgit v1.2.3 From bccff827dc968b08bddda03aadcb3d9cc41c2719 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Nov 2020 13:50:36 +0100 Subject: Fix deadlock in IFD through the daemon Fixes #4235. --- src/libstore/derivations.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 517ecfaa2..eea129df3 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -463,13 +463,15 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath { auto h = drvHashes.find(drvPath); if (h == drvHashes.end()) { - assert(store.isValidPath(drvPath)); // Cache it h = drvHashes.insert_or_assign( drvPath, hashDerivationModulo( store, - store.readDerivation(drvPath), + parseDerivation( + store, + readFile(store.toRealPath(drvPath)), + Derivation::nameFromPath(drvPath)), false)).first; } return h->second; -- cgit v1.2.3 From e6b7c7b79c697f1d8508930964e8c810f04a8963 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Nov 2020 13:58:55 +0100 Subject: Cleanup --- src/libstore/derivations.cc | 5 +---- src/libstore/local-store.cc | 10 ++-------- src/libstore/store-api.cc | 8 ++++++++ src/libstore/store-api.hh | 3 +++ 4 files changed, 14 insertions(+), 12 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index eea129df3..106ec5daa 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -468,10 +468,7 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath drvPath, hashDerivationModulo( store, - parseDerivation( - store, - readFile(store.toRealPath(drvPath)), - Derivation::nameFromPath(drvPath)), + store.readInvalidDerivation(drvPath), false)).first; } return h->second; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e4e404dca..93d073768 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -624,10 +624,7 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = parseDerivation( - *this, - readFile(Store::toRealPath(info.path)), - Derivation::nameFromPath(info.path)); + auto drv = readInvalidDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -1003,10 +1000,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). checkDerivationOutputs(i.path, - parseDerivation( - *this, - readFile(Store::toRealPath(i.path)), - Derivation::nameFromPath(i.path))); + readInvalidDerivation(i.path)); } /* Do a topological sort of the paths. This will throw an diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 83d3a1fa1..7f808f45a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1007,6 +1007,14 @@ Derivation Store::readDerivation(const StorePath & drvPath) } } +Derivation Store::readInvalidDerivation(const StorePath & drvPath) +{ + return parseDerivation( + *this, + readFile(Store::toRealPath(drvPath)), + Derivation::nameFromPath(drvPath)); +} + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f77bc21d1..30c9e7d0a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -611,6 +611,9 @@ public: /* Read a derivation (which must already be valid). */ Derivation readDerivation(const StorePath & drvPath); + /* Read a derivation from a potentially invalid path. */ + Derivation readInvalidDerivation(const StorePath & drvPath); + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly or indirectly reached from it. `out' is not cleared. If -- cgit v1.2.3 From 2113ae2d856a208350ccbafdc19e8dda322515b8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 19 Nov 2020 16:50:06 +0000 Subject: Make drv hash modulo memo table thread-safe Let's get one step closer to the daemon not needing to fork. --- src/libstore/derivations.cc | 27 +++++++++++++++------------ src/libstore/derivations.hh | 3 ++- 2 files changed, 17 insertions(+), 13 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 106ec5daa..231ca26c2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -451,7 +451,7 @@ DerivationType BasicDerivation::type() const } -DrvHashes drvHashes; +Sync drvHashes; /* pathDerivationModulo and hashDerivationModulo are mutually recursive */ @@ -459,19 +459,22 @@ DrvHashes drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath) +static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) { - auto h = drvHashes.find(drvPath); - if (h == drvHashes.end()) { - // Cache it - h = drvHashes.insert_or_assign( - drvPath, - hashDerivationModulo( - store, - store.readInvalidDerivation(drvPath), - false)).first; + { + auto hashes = drvHashes.lock(); + auto h = hashes->find(drvPath); + if (h != hashes->end()) { + return h->second; + } } - return h->second; + auto h = hashDerivationModulo( + store, + store.readInvalidDerivation(drvPath), + false); + // Cache it + drvHashes.lock()->insert_or_assign(drvPath, h); + return h; } /* See the header for interface details. These are the implementation details. diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index f9ba935e6..b966d6d90 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -210,7 +210,8 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m /* Memoisation of hashDerivationModulo(). */ typedef std::map DrvHashes; -extern DrvHashes drvHashes; // FIXME: global, not thread-safe +// FIXME: global, though at least thread-safe. +extern Sync drvHashes; /* Memoisation of `readDerivation(..).resove()`. */ typedef std::map< -- cgit v1.2.3 From 13c557fe823549db75c3dc24c99c46e1c4e1378e Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 25 Nov 2020 11:20:03 +0100 Subject: fix the hash rewriting for ca-derivations --- src/libstore/build/derivation-goal.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bf2bad62c..0e4504857 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3121,6 +3121,20 @@ void DerivationGoal::registerOutputs() newInfo0.references = refs.second; if (refs.first) newInfo0.references.insert(newInfo0.path); + if (scratchPath != newInfo0.path) { + // Also rewrite the output path + auto source = sinkToSource([&](Sink & nextSink) { + StringSink sink; + dumpPath(actualPath, sink); + RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); + rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); + deletePath(actualPath); + movePath(tmpPath, actualPath); + } assert(newInfo0.ca); return newInfo0; -- cgit v1.2.3 From 9bd8184f1fb2e91ac4fb7207abe56f2a30a81d97 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 25 Nov 2020 18:20:35 +0100 Subject: Allow fixed-output derivations to depend on (floating) content-addressed ones Fix an overlook of https://github.com/NixOS/nix/pull/4056 --- src/libstore/build/derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 0e4504857..76c49f92c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -493,8 +493,8 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if ((!fullDrv.inputDrvs.empty() && - fullDrv.type() == DerivationType::CAFloating) || fullDrv.type() == DerivationType::DeferredInputAddressed) { + if ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) + || fullDrv.type() == DerivationType::DeferredInputAddressed) { /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ -- cgit v1.2.3 From 3b7e00ce2215b742d9fdb1b8d4a4d76d349028c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 18 Nov 2020 14:36:15 +0100 Subject: Move primeCache() to Worker::run() We need the missing path info to communicate the worker's remaining goals to the progress bar. --- src/libstore/build/derivation-goal.hh | 12 +----------- src/libstore/build/local-store-build.cc | 22 ++-------------------- src/libstore/build/substitution-goal.hh | 8 +------- src/libstore/build/worker.cc | 21 ++++++++++++++++++++- 4 files changed, 24 insertions(+), 39 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 4976207e0..8ee0be9e1 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -40,9 +40,8 @@ struct InitialOutput { std::optional known; }; -class DerivationGoal : public Goal +struct DerivationGoal : public Goal { -private: /* Whether to use an on-disk .drv file. */ bool useDerivation; @@ -246,7 +245,6 @@ private: friend struct RestrictedStore; -public: DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); @@ -264,17 +262,11 @@ public: void work() override; - StorePath getDrvPath() - { - return drvPath; - } - /* Add wanted outputs to an already existing derivation goal. */ void addWantedOutputs(const StringSet & outputs); BuildResult getResult() { return result; } -private: /* The states. */ void getDerivation(); void loadDerivation(); @@ -318,8 +310,6 @@ private: /* Run the builder's process. */ void runChild(); - friend int childEntry(void *); - /* Check that the derivation outputs all exist and register them as valid. */ void registerOutputs(); diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/local-store-build.cc index a05fb5805..c91cda2fd 100644 --- a/src/libstore/build/local-store-build.cc +++ b/src/libstore/build/local-store-build.cc @@ -5,25 +5,10 @@ namespace nix { -static void primeCache(Store & store, const std::vector & paths) -{ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; - store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); - - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); -} - - void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { Worker worker(*this); - primeCache(*this, drvPaths); - Goals goals; for (auto & path : drvPaths) { if (path.path.isDerivation()) @@ -44,9 +29,8 @@ void LocalStore::buildPaths(const std::vector & drvPaths, ex = i->ex; } if (i->exitCode != Goal::ecSuccess) { - DerivationGoal * i2 = dynamic_cast(i.get()); - if (i2) failed.insert(i2->getDrvPath()); - else failed.insert(dynamic_cast(i.get())->getStorePath()); + if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); + else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); } } @@ -84,8 +68,6 @@ void LocalStore::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {{path}}); - Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); Goals goals = {goal}; diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 3ae9a9e6b..dee2cecbf 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -8,11 +8,8 @@ namespace nix { class Worker; -class SubstitutionGoal : public Goal +struct SubstitutionGoal : public Goal { - friend class Worker; - -private: /* The store path that should be realised through a substitute. */ StorePath storePath; @@ -56,7 +53,6 @@ private: /* Content address for recomputing store path */ std::optional ca; -public: SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~SubstitutionGoal(); @@ -82,8 +78,6 @@ public: /* Callback used by the worker to write to the log. */ void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; - - StorePath getStorePath() { return storePath; } }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 17c10cd71..1f8999a4b 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -207,7 +207,26 @@ void Worker::waitForAWhile(GoalPtr goal) void Worker::run(const Goals & _topGoals) { - for (auto & i : _topGoals) topGoals.insert(i); + std::vector topPaths; + + for (auto & i : _topGoals) { + topGoals.insert(i); + if (auto goal = dynamic_cast(i.get())) { + topPaths.push_back({goal->drvPath, goal->wantedOutputs}); + } else if (auto goal = dynamic_cast(i.get())) { + topPaths.push_back({goal->storePath}); + } + } + + /* Call queryMissing() efficiently query substitutes. */ + StorePathSet willBuild, willSubstitute, unknown; + uint64_t downloadSize, narSize; + store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) + throw Error( + "%d derivations need to be built, but neither local builds ('--max-jobs') " + "nor remote builds ('--builders') are enabled", willBuild.size()); debug("entered goal loop"); -- cgit v1.2.3 From 438977731cf049cf47873d5825456d47a1aac541 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 1 Dec 2020 14:57:56 +0100 Subject: shut up clang warnings - Fix some class/struct discrepancies - Explicit the overloading of `run` in the `Cmd*` classes - Ignore a warning in the generated lexer --- src/libstore/build/goal.hh | 2 +- src/libstore/build/worker.hh | 4 ++-- src/libstore/local-store.hh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 0781a9d38..fca4f2d00 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -7,7 +7,7 @@ namespace nix { /* Forward definition. */ struct Goal; -struct Worker; +class Worker; /* A pointer to a goal. */ typedef std::shared_ptr GoalPtr; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 3a53a8def..bf8cc4586 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -8,8 +8,8 @@ namespace nix { /* Forward definition. */ -class DerivationGoal; -class SubstitutionGoal; +struct DerivationGoal; +struct SubstitutionGoal; /* Workaround for not being able to declare a something like diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d4435220d..58ec93f27 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -296,8 +296,8 @@ private: void createUser(const std::string & userName, uid_t userId) override; - friend class DerivationGoal; - friend class SubstitutionGoal; + friend struct DerivationGoal; + friend struct SubstitutionGoal; }; -- cgit v1.2.3 From faa31f40846f7a4dbc2487d000b112a6aef69d1b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 14:00:43 +0100 Subject: Sink: Use std::string_view --- src/libstore/binary-cache-store.cc | 3 +-- src/libstore/build/derivation-goal.cc | 8 +++--- src/libstore/daemon.cc | 4 +-- src/libstore/filetransfer.cc | 18 ++++++------- src/libstore/filetransfer.hh | 2 +- src/libstore/nar-accessor.cc | 2 +- src/libstore/references.cc | 50 +++++++++++++++-------------------- src/libstore/references.hh | 4 +-- src/libstore/remote-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/store-api.cc | 4 +-- 11 files changed, 45 insertions(+), 54 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f6224d6a0..a918b7208 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -86,8 +86,7 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) promise.set_exception(std::current_exception()); } }}); - auto data = promise.get_future().get(); - sink((unsigned char *) data->data(), data->size()); + sink(*promise.get_future().get()); } std::shared_ptr BinaryCacheStore::getFile(const std::string & path) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 76c49f92c..1db85bd37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -916,10 +916,8 @@ void DerivationGoal::buildDone() LogSink(Activity & act) : act(act) { } - void operator() (const unsigned char * data, size_t len) override { - for (size_t i = 0; i < len; i++) { - auto c = data[i]; - + void operator() (std::string_view data) override { + for (auto c : data) { if (c == '\n') { flushLine(); } else { @@ -3127,7 +3125,7 @@ void DerivationGoal::registerOutputs() StringSink sink; dumpPath(actualPath, sink); RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2(*sink.s); rsink2.flush(); }); Path tmpPath = actualPath + ".tmp"; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 60cca4fda..cb214bee3 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -153,10 +153,10 @@ struct TunnelSink : Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } - virtual void operator () (const unsigned char * data, size_t len) + void operator () (std::string_view data) { to << STDERR_WRITE; - writeString(data, len, to); + writeString(data, to); } }; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c2c65af05..31b4215a9 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -95,18 +95,18 @@ struct curlFileTransfer : public FileTransfer fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) - , finalSink([this](const unsigned char * data, size_t len) { + , finalSink([this](std::string_view data) { if (this->request.dataCallback) { auto httpStatus = getHTTPStatus(); /* Only write data to the sink if this is a successful response. */ if (successfulStatuses.count(httpStatus)) { - writtenToSink += len; - this->request.dataCallback((char *) data, len); + writtenToSink += data.size(); + this->request.dataCallback(data); } } else - this->result.data->append((char *) data, len); + this->result.data->append(data); }) { if (!request.expectedETag.empty()) @@ -171,8 +171,8 @@ struct curlFileTransfer : public FileTransfer } if (errorSink) - (*errorSink)((unsigned char *) contents, realSize); - (*decompressionSink)((unsigned char *) contents, realSize); + (*errorSink)({(char *) contents, realSize}); + (*decompressionSink)({(char *) contents, realSize}); return realSize; } catch (...) { @@ -776,7 +776,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) state->request.notify_one(); }); - request.dataCallback = [_state](char * buf, size_t len) { + request.dataCallback = [_state](std::string_view data) { auto state(_state->lock()); @@ -794,7 +794,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) /* Append data to the buffer and wake up the calling thread. */ - state->data.append(buf, len); + state->data.append(data); state->avail.notify_one(); }; @@ -840,7 +840,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) if it's blocked on a full buffer. We don't hold the state lock while doing this to prevent blocking the download thread if sink() takes a long time. */ - sink((unsigned char *) chunk.data(), chunk.size()); + sink(chunk); } } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index c89c51a21..afc7e7aa6 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -61,7 +61,7 @@ struct FileTransferRequest bool decompress = true; std::shared_ptr data; std::string mimeType; - std::function dataCallback; + std::function dataCallback; FileTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index a9efdd0b6..e35031f7a 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -87,7 +87,7 @@ struct NarAccessor : public FSAccessor parents.top()->start = pos; } - void receiveContents(unsigned char * data, size_t len) override + void receiveContents(std::string_view data) override { } void createSymlink(const Path & path, const string & target) override diff --git a/src/libstore/references.cc b/src/libstore/references.cc index d2096cb49..eb117b5ba 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -55,27 +55,23 @@ struct RefScanSink : Sink RefScanSink() { } - void operator () (const unsigned char * data, size_t len); + void operator () (std::string_view data) override + { + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + string s = tail + std::string(data, 0, refLength); + search((const unsigned char *) s.data(), s.size(), hashes, seen); + + search((const unsigned char *) data.data(), data.size(), hashes, seen); + + size_t tailLen = data.size() <= refLength ? data.size() : refLength; + tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)); + tail.append({data.data() + data.size() - tailLen, tailLen}); + } }; -void RefScanSink::operator () (const unsigned char * data, size_t len) -{ - /* It's possible that a reference spans the previous and current - fragment, so search in the concatenation of the tail of the - previous fragment and the start of the current fragment. */ - string s = tail + string((const char *) data, len > refLength ? refLength : len); - search((const unsigned char *) s.data(), s.size(), hashes, seen); - - search(data, len, hashes, seen); - - size_t tailLen = len <= refLength ? len : refLength; - tail = - string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + - string((const char *) data + len - tailLen, tailLen); -} - - std::pair scanForReferences(const string & path, const PathSet & refs) { @@ -129,10 +125,10 @@ RewritingSink::RewritingSink(const std::string & from, const std::string & to, S assert(from.size() == to.size()); } -void RewritingSink::operator () (const unsigned char * data, size_t len) +void RewritingSink::operator () (std::string_view data) { std::string s(prev); - s.append((const char *) data, len); + s.append(data); size_t j = 0; while ((j = s.find(from, j)) != string::npos) { @@ -146,14 +142,14 @@ void RewritingSink::operator () (const unsigned char * data, size_t len) pos += consumed; - if (consumed) nextSink((unsigned char *) s.data(), consumed); + if (consumed) nextSink(s.substr(0, consumed)); } void RewritingSink::flush() { if (prev.empty()) return; pos += prev.size(); - nextSink((unsigned char *) prev.data(), prev.size()); + nextSink(prev); prev.clear(); } @@ -163,9 +159,9 @@ HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) { } -void HashModuloSink::operator () (const unsigned char * data, size_t len) +void HashModuloSink::operator () (std::string_view data) { - rewritingSink(data, len); + rewritingSink(data); } HashResult HashModuloSink::finish() @@ -176,10 +172,8 @@ HashResult HashModuloSink::finish() NAR with self-references and a NAR with some of the self-references already zeroed out do not produce a hash collision. FIXME: proof. */ - for (auto & pos : rewritingSink.matches) { - auto s = fmt("|%d", pos); - hashSink((unsigned char *) s.data(), s.size()); - } + for (auto & pos : rewritingSink.matches) + hashSink(fmt("|%d", pos)); auto h = hashSink.finish(); return {h.first, rewritingSink.pos}; diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c2efd095c..4f12e6b21 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -19,7 +19,7 @@ struct RewritingSink : Sink RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); - void operator () (const unsigned char * data, size_t len) override; + void operator () (std::string_view data) override; void flush(); }; @@ -31,7 +31,7 @@ struct HashModuloSink : AbstractHashSink HashModuloSink(HashType ht, const std::string & modulus); - void operator () (const unsigned char * data, size_t len) override; + void operator () (std::string_view data) override; HashResult finish() override; }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 48af9f0c4..6e0c85237 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -857,7 +857,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * if (!source) throw Error("no source"); size_t len = readNum(from); auto buf = std::make_unique(len); - writeString(buf.get(), source->read(buf.get(), len), to); + writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); to.flush(); } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 552c4aac7..4519dd5b5 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -398,7 +398,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName, path, res.data->size(), res.durationMs); - sink((unsigned char *) res.data->data(), res.data->size()); + sink(*res.data); } else throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3b2be05cb..27be66cac 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -772,8 +772,8 @@ void copyStorePath(ref srcStore, ref dstStore, } auto source = sinkToSource([&](Sink & sink) { - LambdaSink progressSink([&](const unsigned char * data, size_t len) { - total += len; + LambdaSink progressSink([&](std::string_view data) { + total += data.size(); act.progress(total, info->narSize); }); TeeSink tee { sink, progressSink }; -- cgit v1.2.3 From 1b79b5b983a6c775766bd0d1c7881042188998b8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 2 Dec 2020 14:10:56 +0100 Subject: read(): Use char * instead of unsigned char * This gets rid of some pointless casts. --- src/libstore/daemon.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/nar-accessor.cc | 2 +- src/libstore/remote-fs-accessor.cc | 2 +- src/libstore/remote-store.cc | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index cb214bee3..e5cfe94cb 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -165,7 +165,7 @@ struct TunnelSource : BufferedSource Source & from; BufferedSink & to; TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { } - size_t readUnbuffered(unsigned char * data, size_t len) override + size_t readUnbuffered(char * data, size_t len) override { to << STDERR_READ << len; to.flush(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 93d073768..348e5d0d4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1143,7 +1143,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, dump.resize(oldSize + want); auto got = 0; try { - got = source.read((uint8_t *) dump.data() + oldSize, want); + got = source.read(dump.data() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index e35031f7a..1427a0f98 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -96,7 +96,7 @@ struct NarAccessor : public FSAccessor NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); } - size_t read(unsigned char * data, size_t len) override + size_t read(char * data, size_t len) override { auto n = source.read(data, len); pos += n; diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 2d02a181b..63bde92de 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -75,7 +75,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) throw SysError("seeking in '%s'", cacheFile); std::string buf(length, 0); - readFull(fd.get(), (unsigned char *) buf.data(), length); + readFull(fd.get(), buf.data(), length); return buf; }); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6e0c85237..be29f8e6f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -856,7 +856,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * else if (msg == STDERR_READ) { if (!source) throw Error("no source"); size_t len = readNum(from); - auto buf = std::make_unique(len); + auto buf = std::make_unique(len); writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); to.flush(); } -- cgit v1.2.3 From d8fc1bb7b00dd7b13d667d3cb41bfcbe0df699d0 Mon Sep 17 00:00:00 2001 From: Greg Hale Date: Wed, 2 Dec 2020 10:15:18 -0500 Subject: fix tokens documentation --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index eabd83e3f..4655ca058 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -867,7 +867,7 @@ public: Example `~/.config/nix/nix.conf`: ``` - access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk" + access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk ``` Example `~/code/flake.nix`: -- cgit v1.2.3 From 0afab668fa3f20f091dec0a31fc0b0fbaac2afde Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 3 Dec 2020 13:19:20 +0100 Subject: Don't fail early when -j0 is passed If the build closure contains some CA derivations, then we can't know ahead-of-time that we won't build anything as early-cutoff might come-in at a laster stage --- src/libstore/build/worker.cc | 5 ----- 1 file changed, 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 1f8999a4b..6c96a93bd 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -223,11 +223,6 @@ void Worker::run(const Goals & _topGoals) uint64_t downloadSize, narSize; store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize); - if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) - throw Error( - "%d derivations need to be built, but neither local builds ('--max-jobs') " - "nor remote builds ('--builders') are enabled", willBuild.size()); - debug("entered goal loop"); while (1) { -- cgit v1.2.3 From 9b1824ecbd222b4bdc8fa2b6f345dc55ef4872d0 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:35:38 -0600 Subject: Add extraPlatforms for Rosetta 2 macOS macOS systems with ARM64 can utilize a translation layer at /Library/Apple/usr/libexec/oah to run x86_64 binaries. This change makes Nix recognize that and it to "extra-platforms". Note that there are two cases here since Nix could be built for either x86_64 or aarch64. In either case, we can switch to the other architecture. Unfortunately there is not a good way to prevent aarch64 binaries from being run in x86_64 contexts or vice versa - programs can always execute programs for the other architecture. --- src/libstore/globals.cc | 22 ++++++++++++++++++++++ src/libstore/globals.hh | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f38601d6d..59c49af8a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -131,6 +131,28 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +StringSet Settings::getDefaultExtraPlatforms() +{ + if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1()) + return StringSet{"i686-linux"}; +#if __APPLE__ + // Rosetta 2 emulation layer can run x86_64 binaries on aarch64 + // machines. Note that we can’t force processes from executing + // x86_64 in aarch64 environments or vice versa since they can + // always exec with their own binary preferences. + else if (pathExists("/Library/Apple/usr/libexec/oah")) { + if (std::string{SYSTEM} == "x86_64-darwin") + return StringSet{"aarch64-darwin"}; + else if (std::string{SYSTEM} == "aarch64-darwin") + return StringSet{"x86_64-darwin"}; + else + return StringSet{}; + } +#endif + else + return StringSet{}; +} + bool Settings::isExperimentalFeatureEnabled(const std::string & name) { auto & f = experimentalFeatures.get(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4655ca058..8666a7d28 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -34,6 +34,8 @@ class Settings : public Config { StringSet getDefaultSystemFeatures(); + StringSet getDefaultExtraPlatforms(); + bool isWSL1(); public: @@ -545,7 +547,7 @@ public: Setting extraPlatforms{ this, - std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, + getDefaultExtraPlatforms(), "extra-platforms", R"( Platforms other than the native one which this machine is capable of -- cgit v1.2.3 From 4b9acf4e21a834276b7d061942e7b5d3692662b6 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Thu, 3 Dec 2020 15:41:59 -0600 Subject: Use posix_spawn_setbinpref_np to advise which architecture to run When running universal binaries like /bin/bash, Darwin XNU will choose which architecture of the binary to use based on "binary preferences". This change sets that to the current platform for aarch64 and x86_64 builds. In addition it now uses posix_spawn instead of the usual execve. Note, that this does not prevent the other architecture from being run, just advises which to use. Unfortunately, posix_spawnattr_setbinpref_np does not appear to be inherited by child processes in x86_64 Rosetta 2 translations, meaning that this will not always work as expected. For example: { arm = derivation { name = "test"; system = "aarch64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; rosetta = derivation { name = "test"; system = "x86_64-darwin"; builder = "/bin/bash"; args = [ "-e" (builtins.toFile "test" '' set -x /usr/sbin/sysctl sysctl.proc_translated /usr/sbin/sysctl sysctl.proc_native [ "$(/usr/bin/arch)" = i386 ] echo It works! /usr/bin/touch $out '') ]; }; } `arm' fails on x86_64-compiled Nix, but `arm' and `rosetta' succeed on aarch64-compiled Nix. I suspect there is a way to fix this since: $ /usr/bin/arch -arch x86_64 /bin/bash \ -c '/usr/bin/arch -arch arm64e /bin/bash -c /usr/bin/arch' arm64 seems to work correctly. We may need to wait for Apple to update system_cmds in opensource.apple.com to find out how though. --- src/libstore/build/derivation-goal.cc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..f370fd82d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -50,6 +50,10 @@ #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif +#if __APPLE__ +#include +#endif + #include #include @@ -2844,7 +2848,27 @@ void DerivationGoal::runChild() } } +#if __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif throw SysError("executing '%1%'", drv->builder); -- cgit v1.2.3 From e20a3ec756c61b076d5fbaea6d976c8ee7fa4c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phan=20Kochen?= Date: Fri, 4 Dec 2020 19:32:35 +0100 Subject: Fix compatibility with newer AWS SDKs Tested against AWS SDK 1.8.99. Fixes #3201. --- src/libstore/s3-binary-cache-store.cc | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 4519dd5b5..27253fc12 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -57,6 +57,10 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem { debug("AWS: %s", chomp(statement)); } + +#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115) + void Flush() override {} +#endif }; static void initAWS() -- cgit v1.2.3 From 3c9b7029ba88e8b831f2054c085ab1fc55c31673 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 13:26:53 -0600 Subject: Use com.apple.oahd.plist for rosetta 2 detection --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 59c49af8a..ad66ef8a8 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -140,7 +140,7 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - else if (pathExists("/Library/Apple/usr/libexec/oah")) { + else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") return StringSet{"aarch64-darwin"}; else if (std::string{SYSTEM} == "aarch64-darwin") -- cgit v1.2.3 From b9a00fd15bb158c293b40aec49c9a426cc4c8921 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 4 Dec 2020 22:17:19 -0600 Subject: =?UTF-8?q?Canonicalize=20binary=20caches=20with=20=E2=80=98/?= =?UTF-8?q?=E2=80=99=20when=20one=20is=20missing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This checks if there is a trusted substituter with a slash, so trusting https://cache.nixos.org also implies https://cache.nixos.org/ is trusted. --- src/libstore/daemon.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e5cfe94cb..2224d54d5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -215,6 +215,8 @@ struct ClientSettings for (auto & s : ss) if (trusted.count(s)) subs.push_back(s); + else if (!hasSuffix(s, "/") && trusted.count(s + "/")) + subs.push_back(s + "/"); else warn("ignoring untrusted substituter '%s'", s); res = subs; -- cgit v1.2.3 From aa07502009625fe0d38fde1a23c50dd34f1996eb Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Sun, 6 Dec 2020 23:04:42 -0600 Subject: Always default to cache.nixos.org even when different nix store dir Since 0744f7f, it is now useful to have cache.nixos.org in substituers even if /nix/store is not the Nix Store Dir. This can always be overridden via configuration, though. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4655ca058..6b4775683 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -583,7 +583,7 @@ public: Setting substituters{ this, - nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), + Strings{"https://cache.nixos.org/"}, "substituters", R"( A list of URLs of substituters, separated by whitespace. The default -- cgit v1.2.3 From 1b1e0760335832c87516b9103b670b34662d5daf Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 11:11:02 +0100 Subject: Re-query for the derivation outputs in the post-build-hook We can't assume that the runtime state knows about them as they might have been built remotely, in which case we must query the db again to get them. --- src/libstore/build/derivation-goal.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..fdf777c27 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -899,8 +899,10 @@ void DerivationGoal::buildDone() Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); + for (auto& [_, maybeOutPath] : + worker.store.queryPartialDerivationOutputMap(drvPath)) { + if (maybeOutPath) + outputPaths.insert(*maybeOutPath); } std::map hookEnvironment = getEnv(); -- cgit v1.2.3 From c0f21f08f817745fcf3e9301749b7e237634521c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 06:57:31 +0100 Subject: Hide the sqlite statements declarations for the local store These have no need to be in the public interface and it causes spurious rebuilds each time one wants to add or remove a new statement. --- src/libstore/local-store.cc | 76 +++++++++++++++++++++++++++------------------ src/libstore/local-store.hh | 15 ++------- 2 files changed, 48 insertions(+), 43 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 348e5d0d4..2a47b3956 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -42,6 +42,21 @@ namespace nix { +struct LocalStore::State::Stmts { + /* Some precompiled SQLite statements. */ + SQLiteStmt RegisterValidPath; + SQLiteStmt UpdatePathInfo; + SQLiteStmt AddReference; + SQLiteStmt QueryPathInfo; + SQLiteStmt QueryReferences; + SQLiteStmt QueryReferrers; + SQLiteStmt InvalidatePath; + SQLiteStmt AddDerivationOutput; + SQLiteStmt QueryValidDerivers; + SQLiteStmt QueryDerivationOutputs; + SQLiteStmt QueryPathFromHashPart; + SQLiteStmt QueryValidPaths; +}; LocalStore::LocalStore(const Params & params) : StoreConfig(params) @@ -60,6 +75,7 @@ LocalStore::LocalStore(const Params & params) , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) { auto state(_state.lock()); + state->stmts = std::make_unique(); /* Create missing state directories if they don't already exist. */ createDirs(realStoreDir); @@ -223,31 +239,31 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); /* Prepare SQL statements. */ - state->stmtRegisterValidPath.create(state->db, + state->stmts->RegisterValidPath.create(state->db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); - state->stmtUpdatePathInfo.create(state->db, + state->stmts->UpdatePathInfo.create(state->db, "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); - state->stmtAddReference.create(state->db, + state->stmts->AddReference.create(state->db, "insert or replace into Refs (referrer, reference) values (?, ?);"); - state->stmtQueryPathInfo.create(state->db, + state->stmts->QueryPathInfo.create(state->db, "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); - state->stmtQueryReferences.create(state->db, + state->stmts->QueryReferences.create(state->db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); - state->stmtQueryReferrers.create(state->db, + state->stmts->QueryReferrers.create(state->db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); - state->stmtInvalidatePath.create(state->db, + state->stmts->InvalidatePath.create(state->db, "delete from ValidPaths where path = ?;"); - state->stmtAddDerivationOutput.create(state->db, + state->stmts->AddDerivationOutput.create(state->db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); - state->stmtQueryValidDerivers.create(state->db, + state->stmts->QueryValidDerivers.create(state->db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); - state->stmtQueryDerivationOutputs.create(state->db, + state->stmts->QueryDerivationOutputs.create(state->db, "select id, path from DerivationOutputs where drv = ?;"); // Use "path >= ?" with limit 1 rather than "path like '?%'" to // ensure efficient lookup. - state->stmtQueryPathFromHashPart.create(state->db, + state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); - state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); + state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); } @@ -590,7 +606,7 @@ void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & out void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output) { retrySQLite([&]() { - state.stmtAddDerivationOutput.use() + state.stmts->AddDerivationOutput.use() (deriver) (outputName) (printStorePath(output)) @@ -607,7 +623,7 @@ uint64_t LocalStore::addValidPath(State & state, throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", printStorePath(info.path)); - state.stmtRegisterValidPath.use() + state.stmts->RegisterValidPath.use() (printStorePath(info.path)) (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) @@ -659,7 +675,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, auto state(_state.lock()); /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path))); + auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path))); if (!useQueryPathInfo.next()) return std::shared_ptr(); @@ -679,7 +695,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, info->registrationTime = useQueryPathInfo.getInt(2); - auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); + auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3); if (s) info->deriver = parseStorePath(s); /* Note that narSize = NULL yields 0. */ @@ -687,14 +703,14 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, info->ultimate = useQueryPathInfo.getInt(5) == 1; - s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); + s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6); if (s) info->sigs = tokenizeString(s, " "); - s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); + s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7); if (s) info->ca = parseContentAddressOpt(s); /* Get the references. */ - auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); + auto useQueryReferences(state->stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) info->references.insert(parseStorePath(useQueryReferences.getStr(0))); @@ -709,7 +725,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, /* Update path info in the database. */ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { - state.stmtUpdatePathInfo.use() + state.stmts->UpdatePathInfo.use() (info.narSize, info.narSize != 0) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) @@ -722,7 +738,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { - auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); + auto use(state.stmts->QueryPathInfo.use()(printStorePath(path))); if (!use.next()) throw InvalidPath("path '%s' is not valid", printStorePath(path)); return use.getInt(0); @@ -731,7 +747,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) bool LocalStore::isValidPath_(State & state, const StorePath & path) { - return state.stmtQueryPathInfo.use()(printStorePath(path)).next(); + return state.stmts->QueryPathInfo.use()(printStorePath(path)).next(); } @@ -757,7 +773,7 @@ StorePathSet LocalStore::queryAllValidPaths() { return retrySQLite([&]() { auto state(_state.lock()); - auto use(state->stmtQueryValidPaths.use()); + auto use(state->stmts->QueryValidPaths.use()); StorePathSet res; while (use.next()) res.insert(parseStorePath(use.getStr(0))); return res; @@ -767,7 +783,7 @@ StorePathSet LocalStore::queryAllValidPaths() void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers) { - auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path))); + auto useQueryReferrers(state.stmts->QueryReferrers.use()(printStorePath(path))); while (useQueryReferrers.next()) referrers.insert(parseStorePath(useQueryReferrers.getStr(0))); @@ -788,7 +804,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) return retrySQLite([&]() { auto state(_state.lock()); - auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path))); + auto useQueryValidDerivers(state->stmts->QueryValidDerivers.use()(printStorePath(path))); StorePathSet derivers; while (useQueryValidDerivers.next()) @@ -848,7 +864,7 @@ std::map> LocalStore::queryPartialDerivati } auto useQueryDerivationOutputs { - state->stmtQueryDerivationOutputs.use() + state->stmts->QueryDerivationOutputs.use() (drvId) }; @@ -872,11 +888,11 @@ std::optional LocalStore::queryPathFromHashPart(const std::string & h return retrySQLite>([&]() -> std::optional { auto state(_state.lock()); - auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); + auto useQueryPathFromHashPart(state->stmts->QueryPathFromHashPart.use()(prefix)); if (!useQueryPathFromHashPart.next()) return {}; - const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); + const char * s = (const char *) sqlite3_column_text(state->stmts->QueryPathFromHashPart, 0); if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0) return parseStorePath(s); return {}; @@ -990,7 +1006,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); for (auto & j : i.references) - state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); + state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } /* Check that the derivation outputs are correct. We can't do @@ -1030,7 +1046,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { debug("invalidating path '%s'", printStorePath(path)); - state.stmtInvalidatePath.use()(printStorePath(path)).exec(); + state.stmts->InvalidatePath.use()(printStorePath(path)).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 58ec93f27..332718af4 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -55,19 +55,8 @@ private: /* The SQLite database object. */ SQLite db; - /* Some precompiled SQLite statements. */ - SQLiteStmt stmtRegisterValidPath; - SQLiteStmt stmtUpdatePathInfo; - SQLiteStmt stmtAddReference; - SQLiteStmt stmtQueryPathInfo; - SQLiteStmt stmtQueryReferences; - SQLiteStmt stmtQueryReferrers; - SQLiteStmt stmtInvalidatePath; - SQLiteStmt stmtAddDerivationOutput; - SQLiteStmt stmtQueryValidDerivers; - SQLiteStmt stmtQueryDerivationOutputs; - SQLiteStmt stmtQueryPathFromHashPart; - SQLiteStmt stmtQueryValidPaths; + struct Stmts; + std::unique_ptr stmts; /* The file to which we write our temporary roots. */ AutoCloseFD fdTempRoots; -- cgit v1.2.3 From 6758e65612b990805d3d7d2039cd92647730e900 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 09:44:07 +0100 Subject: Revert "Re-query for the derivation outputs in the post-build-hook" This reverts commit 1b1e0760335832c87516b9103b670b34662d5daf. Using `queryPartialDerivationOutputMap` assumes that the derivation exists locally which isn't the case for remote builders. --- src/libstore/build/derivation-goal.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index fdf777c27..1db85bd37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -899,10 +899,8 @@ void DerivationGoal::buildDone() Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); StorePathSet outputPaths; - for (auto& [_, maybeOutPath] : - worker.store.queryPartialDerivationOutputMap(drvPath)) { - if (maybeOutPath) - outputPaths.insert(*maybeOutPath); + for (auto i : drv->outputs) { + outputPaths.insert(finalOutputs.at(i.first)); } std::map hookEnvironment = getEnv(); -- cgit v1.2.3 From c87267c2a4621a42d6bfcdb7944cd546f194787a Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 10:38:04 +0100 Subject: Store the final drv outputs in memory when building remotely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `DerivationGoal` has a variable storing the “final” derivation output paths that is used (amongst other things) to fill the environment for the post build hook. However this variable wasn't set when the build-hook is used, causing a crash when both hooks are used together. Fix this by setting this variable (from the informations in the db) after a run of the post build hook. --- src/libstore/build/derivation-goal.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 1db85bd37..de58d9f06 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2872,6 +2872,8 @@ void DerivationGoal::registerOutputs() for (auto & i : drv->outputsAndOptPaths(worker.store)) { if (!i.second.second || !worker.store.isValidPath(*i.second.second)) allValid = false; + else + finalOutputs.insert_or_assign(i.first, *i.second.second); } if (allValid) return; } -- cgit v1.2.3 From 93a8a005defbe5204f739853403ef11dbc016f33 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 5 Dec 2020 15:33:16 +0100 Subject: libstore/openStore: fix stores with IPv6 addresses In `nixStable` (2.3.7 to be precise) it's possible to connect to stores using an IPv6 address: nix ping-store --store ssh://root@2001:db8::1 This is also useful for `nixops(1)` where you could specify an IPv6 address in `deployment.targetHost`. However, this behavior is broken on `nixUnstable` and fails with the following error: $ nix store ping --store ssh://root@2001:db8::1 don't know how to open Nix store 'ssh://root@2001:db8::1' This happened because `openStore` from `libstore` uses the `parseURL` function from `libfetchers` which expects a valid URL as defined in RFC2732. However, this is unsupported by `ssh(1)`: $ nix store ping --store 'ssh://root@[2001:db8::1]' cannot connect to 'root@[2001:db8::1]' This patch now allows both ways of specifying a store (`root@2001:db8::1`) and also `root@[2001:db8::1]` since the latter one is useful to pass query parameters to the remote store. In order to achieve this, the following changes were made: * The URL regex from `url-parts.hh` now allows an IPv6 address in the form `2001:db8::1` and also `[2001:db8::1]`. * In `libstore`, a new function named `extractConnStr` ensures that a proper URL is passed to e.g. `ssh(1)`: * If a URL looks like either `[2001:db8::1]` or `root@[2001:db8::1]`, the brackets will be removed using a regex. No additional validation is done here as only strings parsed by `parseURL` are expected. * In any other case, the string will be left untouched. * The rules above only apply for `LegacySSHStore` and `SSHStore` (a.k.a `ssh://` and `ssh-ng://`). Unresolved questions: * I'm not really sure whether we want to allow both variants of IPv6 addresses in the URL parser. However it should be noted that both seem to be possible according to RFC2732: > This document incudes an update to the generic syntax for Uniform > Resource Identifiers defined in RFC 2396 [URL]. It defines a syntax > for IPv6 addresses and allows the use of "[" and "]" within a URI > explicitly for this reserved purpose. * Currently, it's not supported to specify a port number behind the hostname, however it seems as this is not really supported by the URL parser. Hence, this is probably out of scope here. --- src/libstore/store-api.cc | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 27be66cac..7bf9235b2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -10,6 +10,8 @@ #include "archive.hh" #include "callback.hh" +#include + namespace nix { @@ -1091,6 +1093,34 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para } } +// The `parseURL` function supports both IPv6 URIs as defined in +// RFC2732, but also pure addresses. The latter one is needed here to +// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`). +// +// This function now ensures that a usable connection string is available: +// * If the store to be opened is not an SSH store, nothing will be done. +// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably +// needed to pass further flags), it +// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`). +// * If the URL looks like `root@::1` it will be left as-is. +// * In any other case, the string will be left as-is. +static std::string extractConnStr(const std::string &proto, const std::string &connStr) +{ + if (proto.rfind("ssh") != std::string::npos) { + std::smatch result; + std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$"); + + if (std::regex_match(connStr, result, v6AddrRegex)) { + if (result[1].matched) { + return result.str(1) + result.str(3); + } + return result.str(3); + } + } + + return connStr; +} + ref openStore(const std::string & uri_, const Store::Params & extraParams) { @@ -1099,7 +1129,10 @@ ref openStore(const std::string & uri_, auto parsedUri = parseURL(uri_); params.insert(parsedUri.query.begin(), parsedUri.query.end()); - auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; + auto baseURI = extractConnStr( + parsedUri.scheme, + parsedUri.authority.value_or("") + parsedUri.path + ); for (auto implem : *Implementations::registered) { if (implem.uriSchemes.count(parsedUri.scheme)) { -- cgit v1.2.3 From 63b3536f50f124cdcd7592b344eac157a1439d42 Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Fri, 28 Aug 2020 10:28:35 -0300 Subject: treat s3 permission errors as file-not-found Signed-off-by: Jonathan Ringer --- src/libstore/s3-binary-cache-store.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 27253fc12..d6edafd7e 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -166,7 +166,8 @@ S3Helper::FileTransferResult S3Helper::getObject( dynamic_cast(result.GetBody()).str()); } catch (S3Error & e) { - if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; + if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) && + (e.err != Aws::S3::S3Errors::ACCESS_DENIED)) throw; } auto now2 = std::chrono::steady_clock::now(); -- cgit v1.2.3 From 58cdab64acd4807f73768fb32acdde39b501799f Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 8 Oct 2020 17:36:51 +0200 Subject: Store metadata about drv outputs realisations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For each known realisation, store: - its output - its output path This comes with a set of needed changes: - New `realisations` module declaring the types needed for describing these mappings - New `Store::registerDrvOutput` method registering all the needed informations about a derivation output (also replaces `LocalStore::linkDeriverToPath`) - new `Store::queryRealisation` method to retrieve the informations for a derivations This introcudes some redundancy on the remote-store side between `wopQueryDerivationOutputMap` and `wopQueryRealisation`. However we might need to keep both (regardless of backwards compat) because we sometimes need to get some infos for all the outputs of a derivation (where `wopQueryDerivationOutputMap` is handy), but all the stores can't implement it − because listing all the outputs of a derivation isn't really possible for binary caches where the server doesn't allow to list a directory. --- src/libstore/binary-cache-store.cc | 17 ++++++++ src/libstore/binary-cache-store.hh | 7 ++++ src/libstore/build/derivation-goal.cc | 19 ++++++++- src/libstore/daemon.cc | 22 ++++++++++ src/libstore/dummy-store.cc | 3 ++ src/libstore/legacy-ssh-store.cc | 4 ++ src/libstore/local-binary-cache-store.cc | 1 + src/libstore/local-store.cc | 21 +++++++--- src/libstore/local-store.hh | 12 +++--- src/libstore/realisation.cc | 72 ++++++++++++++++++++++++++++++++ src/libstore/realisation.hh | 34 +++++++++++++++ src/libstore/remote-store.cc | 21 ++++++++++ src/libstore/remote-store.hh | 4 ++ src/libstore/store-api.hh | 15 +++++++ src/libstore/worker-protocol.hh | 5 +++ 15 files changed, 246 insertions(+), 11 deletions(-) create mode 100644 src/libstore/realisation.cc create mode 100644 src/libstore/realisation.hh (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a918b7208..085dc7ba1 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -443,6 +443,23 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s })->path; } +std::optional BinaryCacheStore::queryRealisation(const DrvOutput & id) +{ + auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; + auto rawOutputInfo = getFile(outputInfoFilePath); + + if (rawOutputInfo) { + return { Realisation::parse(*rawOutputInfo, outputInfoFilePath) }; + } else { + return std::nullopt; + } +} + +void BinaryCacheStore::registerDrvOutput(const Realisation& info) { + auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; + upsertFile(filePath, info.to_string(), "text/x-nix-derivertopath"); +} + ref BinaryCacheStore::getFSAccessor() { return make_ref(ref(shared_from_this()), localNarCache); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 5224d7ec8..07a8b2beb 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -33,6 +33,9 @@ private: protected: + // The prefix under which realisation infos will be stored + const std::string realisationsPrefix = "/realisations"; + BinaryCacheStore(const Params & params); public: @@ -99,6 +102,10 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; + void registerDrvOutput(const Realisation & info) override; + + std::optional queryRealisation(const DrvOutput &) override; + void narFromPath(const StorePath & path, Sink & sink) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index de58d9f06..a0f10c33d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2094,6 +2094,20 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf /* Nothing to be done; 'path' must already be valid. */ } + void registerDrvOutput(const Realisation & info) override + { + if (!goal.isAllowed(info.id.drvPath)) + throw InvalidPath("cannot register unknown drv output '%s' in recursive Nix", printStorePath(info.id.drvPath)); + next->registerDrvOutput(info); + } + + std::optional queryRealisation(const DrvOutput & id) override + { + if (!goal.isAllowed(id.drvPath)) + throw InvalidPath("cannot query the output info for unknown derivation '%s' in recursive Nix", printStorePath(id.drvPath)); + return next->queryRealisation(id); + } + void buildPaths(const std::vector & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); @@ -3393,7 +3407,10 @@ void DerivationGoal::registerOutputs() if (useDerivation || isCaFloating) for (auto & [outputName, newInfo] : infos) - worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path); + worker.store.registerDrvOutput( + DrvOutputId{drvPathResolved, outputName}, + DrvOutputInfo{.outPath = newInfo.path, + .resolvedDrv = drvPathResolved}); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2224d54d5..ba5788b64 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -868,6 +868,28 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case wopRegisterDrvOutput: { + logger->startWork(); + auto outputId = DrvOutput::parse(readString(from)); + auto outputPath = StorePath(readString(from)); + auto resolvedDrv = StorePath(readString(from)); + store->registerDrvOutput(Realisation{ + .id = outputId, .outPath = outputPath}); + logger->stopWork(); + break; + } + + case wopQueryRealisation: { + logger->startWork(); + auto outputId = DrvOutput::parse(readString(from)); + auto info = store->queryRealisation(outputId); + logger->stopWork(); + std::set outPaths; + if (info) outPaths.insert(info->outPath); + worker_proto::write(*store, to, outPaths); + break; + } + default: throw Error("invalid operation %1%", op); } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 98b745c3a..91fc178db 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -60,6 +60,9 @@ struct DummyStore : public Store, public virtual DummyStoreConfig BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { unsupported("buildDerivation"); } + + std::optional queryRealisation(const DrvOutput&) override + { unsupported("queryRealisation"); } }; static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 467169ce8..ad1779aea 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -333,6 +333,10 @@ public: auto conn(connections->get()); return conn->remoteVersion; } + + std::optional queryRealisation(const DrvOutput&) override + // TODO: Implement + { unsupported("queryRealisation"); } }; static RegisterStoreImplementation regLegacySSHStore; diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 7d979c5c2..bb7464989 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -87,6 +87,7 @@ protected: void LocalBinaryCacheStore::init() { createDirs(binaryCacheDir + "/nar"); + createDirs(binaryCacheDir + realisationsPrefix); if (writeDebugInfo) createDirs(binaryCacheDir + "/debuginfo"); BinaryCacheStore::init(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2a47b3956..418b3ab9c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -597,13 +597,16 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat } -void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output) +void LocalStore::registerDrvOutput(const Realisation & info) { auto state(_state.lock()); - return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output); + // XXX: This ignores the references of the output because we can + // recompute them later from the drv and the references of the associated + // store path, but doing so is both inefficient and fragile. + return registerDrvOutput_(*state, queryValidPathId(*state, id.drvPath), id.outputName, info.outPath); } -void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output) +void LocalStore::registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output) { retrySQLite([&]() { state.stmts->AddDerivationOutput.use() @@ -653,7 +656,7 @@ uint64_t LocalStore::addValidPath(State & state, /* Floating CA derivations have indeterminate output paths until they are built, so don't register anything in that case */ if (i.second.second) - linkDeriverToPath(state, id, i.first, *i.second.second); + registerDrvOutput_(state, id, i.first, *i.second.second); } } @@ -1612,5 +1615,13 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } } - +std::optional LocalStore::queryDrvOutputInfo(const DrvOutputId& id) { + auto outputPath = queryOutputPathOf(id.drvPath, id.outputName); + if (!(outputPath && isValidPath(*outputPath))) + return std::nullopt; + else + return {DrvOutputInfo{ + .outPath = *outputPath, + }}; } +} // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 332718af4..440411f01 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -208,6 +208,13 @@ public: garbage until it exceeds maxFree. */ void autoGC(bool sync = true); + /* Register the store path 'output' as the output named 'outputName' of + derivation 'deriver'. */ + void registerDrvOutput(const DrvOutputId & outputId, const DrvOutputInfo & info) override; + void registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output); + + std::optional queryRealisation(const DrvOutput&) override; + private: int getSchema(); @@ -276,11 +283,6 @@ private: specified by the ‘secret-key-files’ option. */ void signPathInfo(ValidPathInfo & info); - /* Register the store path 'output' as the output named 'outputName' of - derivation 'deriver'. */ - void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output); - void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output); - Path getRealStoreDir() override { return realStoreDir; } void createUser(const std::string & userName, uid_t userId) override; diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc new file mode 100644 index 000000000..fcc1a3825 --- /dev/null +++ b/src/libstore/realisation.cc @@ -0,0 +1,72 @@ +#include "realisation.hh" +#include "store-api.hh" + +namespace nix { + +MakeError(InvalidDerivationOutputId, Error); + +DrvOutput DrvOutput::parse(const std::string &strRep) { + const auto &[rawPath, outputs] = parsePathWithOutputs(strRep); + if (outputs.size() != 1) + throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); + + return DrvOutput{ + .drvPath = StorePath(rawPath), + .outputName = *outputs.begin(), + }; +} + +std::string DrvOutput::to_string() const { + return std::string(drvPath.to_string()) + "!" + outputName; +} + +std::string Realisation::to_string() const { + std::string res; + + res += "Id: " + id.to_string() + '\n'; + res += "OutPath: " + std::string(outPath.to_string()) + '\n'; + + return res; +} + +Realisation Realisation::parse(const std::string & s, const std::string & whence) +{ + // XXX: Copy-pasted from NarInfo::NarInfo. Should be factored out + auto corrupt = [&]() { + return Error("Drv output info file '%1%' is corrupt", whence); + }; + + std::optional id; + std::optional outPath; + + size_t pos = 0; + while (pos < s.size()) { + + size_t colon = s.find(':', pos); + if (colon == std::string::npos) throw corrupt(); + + std::string name(s, pos, colon - pos); + + size_t eol = s.find('\n', colon + 2); + if (eol == std::string::npos) throw corrupt(); + + std::string value(s, colon + 2, eol - colon - 2); + + if (name == "Id") + id = DrvOutput::parse(value); + + if (name == "OutPath") + outPath = StorePath(value); + + pos = eol + 1; + } + + if (!outPath) corrupt(); + if (!id) corrupt(); + return Realisation { + .id = *id, + .outPath = *outPath, + }; +} + +} // namespace nix diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh new file mode 100644 index 000000000..c573e1bb4 --- /dev/null +++ b/src/libstore/realisation.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "path.hh" + +namespace nix { + +struct DrvOutput { + StorePath drvPath; + std::string outputName; + + std::string to_string() const; + + static DrvOutput parse(const std::string &); + + bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); } + bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); } + +private: + // Just to make comparison operators easier to write + std::pair to_pair() const + { return std::make_pair(drvPath, outputName); } +}; + +struct Realisation { + DrvOutput id; + StorePath outPath; + + std::string to_string() const; + static Realisation parse(const std::string & s, const std::string & whence); +}; + +typedef std::map DrvOutputs; + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index be29f8e6f..f1f4d0516 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -609,6 +609,27 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; } +void RemoteStore::registerDrvOutput(const Realisation & info) +{ + auto conn(getConnection()); + conn->to << wopRegisterDrvOutput; + conn->to << info.id.to_string(); + conn->to << std::string(info.outPath.to_string()); + conn.processStderr(); +} + +std::optional RemoteStore::queryRealisation(const DrvOutput & id) +{ + auto conn(getConnection()); + conn->to << wopQueryRealisation; + conn->to << id.to_string(); + conn.processStderr(); + auto outPaths = worker_proto::read(*this, conn->from, Phantom>{}); + if (outPaths.empty()) + return std::nullopt; + return {Realisation{.id = id, .outPath = *outPaths.begin()}}; +} + void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9f78fcb02..fdd53e6ed 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -81,6 +81,10 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; + void registerDrvOutput(const Realisation & info) override; + + std::optional queryRealisation(const DrvOutput &) override; + void buildPaths(const std::vector & paths, BuildMode buildMode) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6b9331495..7cdadc1f3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,6 @@ #pragma once +#include "realisation.hh" #include "path.hh" #include "hash.hh" #include "content-address.hh" @@ -396,6 +397,8 @@ protected: public: + virtual std::optional queryRealisation(const DrvOutput &) = 0; + /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) @@ -468,6 +471,18 @@ public: virtual StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair = NoRepair) = 0; + /** + * Add a mapping indicating that `deriver!outputName` maps to the output path + * `output`. + * + * This is redundant for known-input-addressed and fixed-output derivations + * as this information is already present in the drv file, but necessary for + * floating-ca derivations and their dependencies as there's no way to + * retrieve this information otherwise. + */ + virtual void registerDrvOutput(const Realisation & output) + { unsupported("registerDrvOutput"); } + /* Write a NAR dump of a store path. */ virtual void narFromPath(const StorePath & path, Sink & sink) = 0; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 63bd6ea49..f2cdc7ca3 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,5 +1,8 @@ #pragma once +#include "store-api.hh" +#include "serialise.hh" + namespace nix { @@ -50,6 +53,8 @@ typedef enum { wopAddToStoreNar = 39, wopQueryMissing = 40, wopQueryDerivationOutputMap = 41, + wopRegisterDrvOutput = 42, + wopQueryRealisation = 43, } WorkerOp; -- cgit v1.2.3 From 3ac9d74eb1de0f696bb0384132f7ecc7d057f9d6 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 20 Oct 2020 15:14:02 +0200 Subject: Rework the db schema for derivation outputs Add a new table for tracking the derivation output mappings. We used to hijack the `DerivationOutputs` table for that, but (despite its name), it isn't a really good fit: - Its entries depend on the drv being a valid path, making it play badly with garbage collection and preventing us to copy a drv output without copying the whole drv closure too; - It dosen't guaranty that the output path exists; By using a different table, we can experiment with a different schema better suited for tracking the output mappings of CA derivations. (incidentally, this also fixes #4138) --- src/libstore/build/derivation-goal.cc | 16 +-- src/libstore/ca-specific-schema.sql | 11 ++ src/libstore/local-store.cc | 217 +++++++++++++++++++++++----------- src/libstore/local-store.hh | 4 +- src/libstore/local.mk | 4 +- 5 files changed, 172 insertions(+), 80 deletions(-) create mode 100644 src/libstore/ca-specific-schema.sql (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index a0f10c33d..b7bf866eb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -493,8 +493,9 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast(drv.get()); - if ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) - || fullDrv.type() == DerivationType::DeferredInputAddressed) { + if (settings.isExperimentalFeatureEnabled("ca-derivations") && + ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) + || fullDrv.type() == DerivationType::DeferredInputAddressed)) { /* We are be able to resolve this derivation based on the now-known results of dependencies. If so, we become a stub goal aliasing that resolved derivation goal */ @@ -3405,12 +3406,11 @@ void DerivationGoal::registerOutputs() drvPathResolved = writeDerivation(worker.store, drv2); } - if (useDerivation || isCaFloating) - for (auto & [outputName, newInfo] : infos) - worker.store.registerDrvOutput( - DrvOutputId{drvPathResolved, outputName}, - DrvOutputInfo{.outPath = newInfo.path, - .resolvedDrv = drvPathResolved}); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) + for (auto& [outputName, newInfo] : infos) + worker.store.registerDrvOutput(Realisation{ + .id = DrvOutput{drvPathResolved, outputName}, + .outPath = newInfo.path}); } diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql new file mode 100644 index 000000000..93c442826 --- /dev/null +++ b/src/libstore/ca-specific-schema.sql @@ -0,0 +1,11 @@ +-- Extension of the sql schema for content-addressed derivations. +-- Won't be loaded unless the experimental feature `ca-derivations` +-- is enabled + +create table if not exists Realisations ( + drvPath text not null, + outputName text not null, -- symbolic output id, usually "out" + outputPath integer not null, + primary key (drvPath, outputName), + foreign key (outputPath) references ValidPaths(id) on delete cascade +); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 418b3ab9c..69ab821d9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -52,12 +52,52 @@ struct LocalStore::State::Stmts { SQLiteStmt QueryReferrers; SQLiteStmt InvalidatePath; SQLiteStmt AddDerivationOutput; + SQLiteStmt RegisterRealisedOutput; SQLiteStmt QueryValidDerivers; SQLiteStmt QueryDerivationOutputs; + SQLiteStmt QueryRealisedOutput; + SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; }; +int getSchema(Path schemaPath) +{ + int curSchema = 0; + if (pathExists(schemaPath)) { + string s = readFile(schemaPath); + if (!string2Int(s, curSchema)) + throw Error("'%1%' is corrupt", schemaPath); + } + return curSchema; +} + +void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) +{ + const int nixCASchemaVersion = 1; + int curCASchema = getSchema(schemaPath); + if (curCASchema != nixCASchemaVersion) { + if (curCASchema > nixCASchemaVersion) { + throw Error("current Nix store ca-schema is version %1%, but I only support %2%", + curCASchema, nixCASchemaVersion); + } + + if (!lockFile(lockFd.get(), ltWrite, false)) { + printInfo("waiting for exclusive access to the Nix store for ca drvs..."); + lockFile(lockFd.get(), ltWrite, true); + } + + if (curCASchema == 0) { + static const char schema[] = + #include "ca-specific-schema.sql.gen.hh" + ; + db.exec(schema); + } + writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); + lockFile(lockFd.get(), ltRead, true); + } +} + LocalStore::LocalStore(const Params & params) : StoreConfig(params) , Store(params) @@ -238,6 +278,10 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + } + /* Prepare SQL statements. */ state->stmts->RegisterValidPath.create(state->db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); @@ -264,6 +308,28 @@ LocalStore::LocalStore(const Params & params) state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + state->stmts->RegisterRealisedOutput.create(state->db, + R"( + insert or replace into Realisations (drvPath, outputName, outputPath) + values (?, ?, (select id from ValidPaths where path = ?)) + ; + )"); + state->stmts->QueryRealisedOutput.create(state->db, + R"( + select Output.path from Realisations + inner join ValidPaths as Output on Output.id = Realisations.outputPath + where drvPath = ? and outputName = ? + ; + )"); + state->stmts->QueryAllRealisedOutputs.create(state->db, + R"( + select outputName, Output.path from Realisations + inner join ValidPaths as Output on Output.id = Realisations.outputPath + where drvPath = ? + ; + )"); + } } @@ -301,16 +367,7 @@ std::string LocalStore::getUri() int LocalStore::getSchema() -{ - int curSchema = 0; - if (pathExists(schemaPath)) { - string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) - throw Error("'%1%' is corrupt", schemaPath); - } - return curSchema; -} - +{ return nix::getSchema(schemaPath); } void LocalStore::openDB(State & state, bool create) { @@ -600,13 +657,16 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat void LocalStore::registerDrvOutput(const Realisation & info) { auto state(_state.lock()); - // XXX: This ignores the references of the output because we can - // recompute them later from the drv and the references of the associated - // store path, but doing so is both inefficient and fragile. - return registerDrvOutput_(*state, queryValidPathId(*state, id.drvPath), id.outputName, info.outPath); + retrySQLite([&]() { + state->stmts->RegisterRealisedOutput.use() + (info.id.drvPath.to_string()) + (info.id.outputName) + (printStorePath(info.outPath)) + .exec(); + }); } -void LocalStore::registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output) +void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output) { retrySQLite([&]() { state.stmts->AddDerivationOutput.use() @@ -656,7 +716,7 @@ uint64_t LocalStore::addValidPath(State & state, /* Floating CA derivations have indeterminate output paths until they are built, so don't register anything in that case */ if (i.second.second) - registerDrvOutput_(state, id, i.first, *i.second.second); + cacheDrvOutputMapping(state, id, i.first, *i.second.second); } } @@ -817,70 +877,85 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) }); } - -std::map> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) +// Try to resolve the derivation at path `original`, with a caching layer +// to make it more efficient +std::optional cachedResolve( + LocalStore & store, + const StorePath & original) { - auto path = path_; - std::map> outputs; - Derivation drv = readDerivation(path); - for (auto & [outName, _] : drv.outputs) { - outputs.insert_or_assign(outName, std::nullopt); - } - bool haveCached = false; { auto resolutions = drvPathResolutions.lock(); - auto resolvedPathOptIter = resolutions->find(path); + auto resolvedPathOptIter = resolutions->find(original); if (resolvedPathOptIter != resolutions->end()) { auto & [_, resolvedPathOpt] = *resolvedPathOptIter; if (resolvedPathOpt) - path = *resolvedPathOpt; - haveCached = true; + return resolvedPathOpt; } } - /* can't just use else-if instead of `!haveCached` because we need to unlock - `drvPathResolutions` before it is locked in `Derivation::resolve`. */ - if (!haveCached && (drv.type() == DerivationType::CAFloating || drv.type() == DerivationType::DeferredInputAddressed)) { - /* Try resolve drv and use that path instead. */ - auto attempt = drv.tryResolve(*this); - if (!attempt) - /* If we cannot resolve the derivation, we cannot have any path - assigned so we return the map of all std::nullopts. */ - return outputs; - /* Just compute store path */ - auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true); - /* Store in memo table. */ - /* FIXME: memo logic should not be local-store specific, should have - wrapper-method instead. */ - drvPathResolutions.lock()->insert_or_assign(path, pathResolved); - path = std::move(pathResolved); - } - return retrySQLite>>([&]() { - auto state(_state.lock()); + /* Try resolve drv and use that path instead. */ + auto drv = store.readDerivation(original); + auto attempt = drv.tryResolve(store); + if (!attempt) + return std::nullopt; + /* Just compute store path */ + auto pathResolved = + writeDerivation(store, *std::move(attempt), NoRepair, true); + /* Store in memo table. */ + drvPathResolutions.lock()->insert_or_assign(original, pathResolved); + return pathResolved; +} + +std::map> +LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) +{ + auto path = path_; + auto outputs = retrySQLite>>([&]() { + auto state(_state.lock()); + std::map> outputs; uint64_t drvId; try { drvId = queryValidPathId(*state, path); - } catch (InvalidPath &) { - /* FIXME? if the derivation doesn't exist, we cannot have a mapping - for it. */ - return outputs; + } catch (InvalidPath&) { + // Ignore non-existing drvs as they might still have an output map + // defined if ca-derivations is enabled } + auto use(state->stmts->QueryDerivationOutputs.use()(drvId)); + while (use.next()) + outputs.insert_or_assign( + use.getStr(0), parseStorePath(use.getStr(1))); - auto useQueryDerivationOutputs { - state->stmts->QueryDerivationOutputs.use() - (drvId) - }; + return outputs; + }); + + if (!settings.isExperimentalFeatureEnabled("ca-derivations")) + return outputs; + + auto drv = readDerivation(path); + + for (auto & output : drv.outputsAndOptPaths(*this)) { + outputs.emplace(output.first, std::nullopt); + } + + auto resolvedDrv = cachedResolve(*this, path); + + if (!resolvedDrv) + return outputs; + + retrySQLite([&]() { + auto state(_state.lock()); + path = *resolvedDrv; + auto useQueryDerivationOutputs{ + state->stmts->QueryAllRealisedOutputs.use()(path.to_string())}; while (useQueryDerivationOutputs.next()) outputs.insert_or_assign( useQueryDerivationOutputs.getStr(0), - parseStorePath(useQueryDerivationOutputs.getStr(1)) - ); - - return outputs; + parseStorePath(useQueryDerivationOutputs.getStr(1))); }); -} + return outputs; +} std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) { @@ -1615,13 +1690,19 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } } -std::optional LocalStore::queryDrvOutputInfo(const DrvOutputId& id) { - auto outputPath = queryOutputPathOf(id.drvPath, id.outputName); - if (!(outputPath && isValidPath(*outputPath))) - return std::nullopt; - else - return {DrvOutputInfo{ - .outPath = *outputPath, - }}; +std::optional LocalStore::queryRealisation( + const DrvOutput& id) { + typedef std::optional Ret; + return retrySQLite([&]() -> Ret { + auto state(_state.lock()); + auto use(state->stmts->QueryRealisedOutput.use()(id.drvPath.to_string())( + id.outputName)); + if (!use.next()) + return std::nullopt; + auto outputPath = parseStorePath(use.getStr(0)); + auto resolvedDrv = StorePath(use.getStr(1)); + return Ret{ + Realisation{.id = id, .outPath = outputPath}}; + }); } } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 440411f01..69559e346 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -210,8 +210,8 @@ public: /* Register the store path 'output' as the output named 'outputName' of derivation 'deriver'. */ - void registerDrvOutput(const DrvOutputId & outputId, const DrvOutputInfo & info) override; - void registerDrvOutput_(State & state, uint64_t deriver, const string & outputName, const StorePath & output); + void registerDrvOutput(const Realisation & info) override; + void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); std::optional queryRealisation(const DrvOutput&) override; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index dfe1e2cc4..03c4351ac 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -48,7 +48,7 @@ ifneq ($(sandbox_shell),) libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" endif -$(d)/local-store.cc: $(d)/schema.sql.gen.hh +$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh $(d)/build.cc: @@ -58,7 +58,7 @@ $(d)/build.cc: @echo ')foo"' >> $@.tmp @mv $@.tmp $@ -clean-files += $(d)/schema.sql.gen.hh +clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) -- cgit v1.2.3 From 8914e01e37ad072d940e2000fede7c2e0f4b194c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Dec 2020 21:07:52 +0100 Subject: Store the realisations as JSON in the binary cache Fix #4332 --- src/libstore/binary-cache-store.cc | 5 ++-- src/libstore/realisation.cc | 61 ++++++++++++-------------------------- src/libstore/realisation.hh | 5 ++-- 3 files changed, 25 insertions(+), 46 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 085dc7ba1..5b081c1ae 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -449,7 +449,8 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut auto rawOutputInfo = getFile(outputInfoFilePath); if (rawOutputInfo) { - return { Realisation::parse(*rawOutputInfo, outputInfoFilePath) }; + return {Realisation::fromJSON( + nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)}; } else { return std::nullopt; } @@ -457,7 +458,7 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut void BinaryCacheStore::registerDrvOutput(const Realisation& info) { auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; - upsertFile(filePath, info.to_string(), "text/x-nix-derivertopath"); + upsertFile(filePath, info.toJSON(), "application/json"); } ref BinaryCacheStore::getFSAccessor() diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index fcc1a3825..47db1ec9f 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,5 +1,6 @@ #include "realisation.hh" #include "store-api.hh" +#include namespace nix { @@ -20,52 +21,28 @@ std::string DrvOutput::to_string() const { return std::string(drvPath.to_string()) + "!" + outputName; } -std::string Realisation::to_string() const { - std::string res; - - res += "Id: " + id.to_string() + '\n'; - res += "OutPath: " + std::string(outPath.to_string()) + '\n'; - - return res; +nlohmann::json Realisation::toJSON() const { + return nlohmann::json{ + {"id", id.to_string()}, + {"outPath", outPath.to_string()}, + }; } -Realisation Realisation::parse(const std::string & s, const std::string & whence) -{ - // XXX: Copy-pasted from NarInfo::NarInfo. Should be factored out - auto corrupt = [&]() { - return Error("Drv output info file '%1%' is corrupt", whence); +Realisation Realisation::fromJSON( + const nlohmann::json& json, + const std::string& whence) { + auto getField = [&](std::string fieldName) -> std::string { + auto fieldIterator = json.find(fieldName); + if (fieldIterator == json.end()) + throw Error( + "Drv output info file '%1%' is corrupt, missing field %2%", + whence, fieldName); + return *fieldIterator; }; - std::optional id; - std::optional outPath; - - size_t pos = 0; - while (pos < s.size()) { - - size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt(); - - std::string name(s, pos, colon - pos); - - size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt(); - - std::string value(s, colon + 2, eol - colon - 2); - - if (name == "Id") - id = DrvOutput::parse(value); - - if (name == "OutPath") - outPath = StorePath(value); - - pos = eol + 1; - } - - if (!outPath) corrupt(); - if (!id) corrupt(); - return Realisation { - .id = *id, - .outPath = *outPath, + return Realisation{ + .id = DrvOutput::parse(getField("id")), + .outPath = StorePath(getField("outPath")), }; } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index c573e1bb4..08579b739 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,6 +1,7 @@ #pragma once #include "path.hh" +#include namespace nix { @@ -25,8 +26,8 @@ struct Realisation { DrvOutput id; StorePath outPath; - std::string to_string() const; - static Realisation parse(const std::string & s, const std::string & whence); + nlohmann::json toJSON() const; + static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); }; typedef std::map DrvOutputs; -- cgit v1.2.3 From bab1cda0e6c30e25460b5a9c809589d3948f35df Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Dec 2020 16:56:56 +0100 Subject: Use the hash modulo in the derivation outputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than storing the derivation outputs as `drvPath!outputName` internally, store them as `drvHashModulo!outputName` (or `outputHash!outputName` for fixed-output derivations). This makes the storage slightly more opaque, but enables an earlier cutoff in cases where a fixed-output dependency changes (but keeps the same output hash) − same as what we already do for input-addressed derivations. --- src/libstore/build/derivation-goal.cc | 28 +++++----------- src/libstore/derivations.cc | 49 ++++++++++++++++++++-------- src/libstore/derivations.hh | 22 +++++-------- src/libstore/local-store.cc | 61 ++++++++++++++++------------------- src/libstore/realisation.cc | 10 +++--- src/libstore/realisation.hh | 10 ++++-- 6 files changed, 92 insertions(+), 88 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b7bf866eb..54b37553a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -504,9 +504,6 @@ void DerivationGoal::inputsRealised() Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); - /* Add to memotable to speed up downstream goal's queries with the - original derivation. */ - drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved); auto msg = fmt("Resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), @@ -2097,15 +2094,15 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf void registerDrvOutput(const Realisation & info) override { - if (!goal.isAllowed(info.id.drvPath)) - throw InvalidPath("cannot register unknown drv output '%s' in recursive Nix", printStorePath(info.id.drvPath)); + // XXX: Should we check for something here? Probably, but I'm not sure + // how next->registerDrvOutput(info); } std::optional queryRealisation(const DrvOutput & id) override { - if (!goal.isAllowed(id.drvPath)) - throw InvalidPath("cannot query the output info for unknown derivation '%s' in recursive Nix", printStorePath(id.drvPath)); + // XXX: Should we check for something here? Probably, but I'm not sure + // how return next->queryRealisation(id); } @@ -3394,23 +3391,14 @@ void DerivationGoal::registerOutputs() means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ - bool isCaFloating = drv->type() == DerivationType::CAFloating; - auto drvPathResolved = drvPath; - if (!useDerivation && isCaFloating) { - /* Once a floating CA derivations reaches this point, it - must already be resolved, so we don't bother trying to - downcast drv to get would would just be an empty - inputDrvs field. */ - Derivation drv2 { *drv }; - drvPathResolved = writeDerivation(worker.store, drv2); - } - - if (settings.isExperimentalFeatureEnabled("ca-derivations")) + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto outputHashes = staticOutputHashes(worker.store, *drv); for (auto& [outputName, newInfo] : infos) worker.store.registerDrvOutput(Realisation{ - .id = DrvOutput{drvPathResolved, outputName}, + .id = DrvOutput{outputHashes.at(outputName), outputName}, .outPath = newInfo.path}); + } } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 231ca26c2..5bcc7f012 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -496,10 +496,9 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & */ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { + bool isDeferred = false; /* Return a fixed hash for fixed-output derivations. */ switch (drv.type()) { - case DerivationType::CAFloating: - return UnknownHashes {}; case DerivationType::CAFixed: { std::map outputHashes; for (const auto & i : drv.outputs) { @@ -512,6 +511,9 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } return outputHashes; } + case DerivationType::CAFloating: + isDeferred = true; + break; case DerivationType::InputAddressed: break; case DerivationType::DeferredInputAddressed: @@ -522,13 +524,16 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m calls to this function. */ std::map inputs2; for (auto & i : drv.inputDrvs) { - bool hasUnknownHash = false; const auto & res = pathDerivationModulo(store, i.first); std::visit(overloaded { // Regular non-CA derivation, replace derivation [&](Hash drvHash) { inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second); }, + [&](DeferredHash deferredHash) { + isDeferred = true; + inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second); + }, // CA derivation's output hashes [&](CaOutputHashes outputHashes) { std::set justOut = { "out" }; @@ -540,16 +545,37 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m justOut); } }, - [&](UnknownHashes) { - hasUnknownHash = true; - }, }, res); - if (hasUnknownHash) { - return UnknownHashes {}; - } } - return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + + if (isDeferred) + return DeferredHash { hash }; + else + return hash; +} + + +std::map staticOutputHashes(Store& store, const Derivation& drv) +{ + std::map res; + std::visit(overloaded { + [&](Hash drvHash) { + for (auto & outputName : drv.outputNames()) { + res.insert({outputName, drvHash}); + } + }, + [&](DeferredHash deferredHash) { + for (auto & outputName : drv.outputNames()) { + res.insert({outputName, deferredHash.hash}); + } + }, + [&](CaOutputHashes outputHashes) { + res = outputHashes; + }, + }, hashDerivationModulo(store, drv, true)); + return res; } @@ -719,9 +745,6 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } - -Sync drvPathResolutions; - std::optional Derivation::tryResolve(Store & store) { BasicDerivation resolved { *this }; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b966d6d90..4e5985fab 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -18,8 +18,6 @@ namespace nix { /* The traditional non-fixed-output derivation type. */ struct DerivationOutputInputAddressed { - /* Will need to become `std::optional` once input-addressed - derivations are allowed to depend on cont-addressed derivations */ StorePath path; }; @@ -174,12 +172,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName // whose output hashes are always known since they are fixed up-front. typedef std::map CaOutputHashes; -struct UnknownHashes {}; +struct DeferredHash { Hash hash; }; typedef std::variant< Hash, // regular DRV normalized hash CaOutputHashes, // Fixed-output derivation hashes - UnknownHashes // Deferred hashes for floating outputs drvs and their dependencies + DeferredHash // Deferred hashes for floating outputs drvs and their dependencies > DrvHashModulo; /* Returns hashes with the details of fixed-output subderivations @@ -207,22 +205,18 @@ typedef std::variant< */ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +/* + Return a map associating each output to a hash that uniquely identifies its + derivation (modulo the self-references). + */ +std::map staticOutputHashes(Store& store, const Derivation& drv); + /* Memoisation of hashDerivationModulo(). */ typedef std::map DrvHashes; // FIXME: global, though at least thread-safe. extern Sync drvHashes; -/* Memoisation of `readDerivation(..).resove()`. */ -typedef std::map< - StorePath, - std::optional -> DrvPathResolutions; - -// FIXME: global, though at least thread-safe. -// FIXME: arguably overlaps with hashDerivationModulo memo table. -extern Sync drvPathResolutions; - bool wantOutput(const string & output, const std::set & wanted); struct Source; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 69ab821d9..1539c94e2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -659,7 +659,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) auto state(_state.lock()); retrySQLite([&]() { state->stmts->RegisterRealisedOutput.use() - (info.id.drvPath.to_string()) + (info.id.strHash()) (info.id.outputName) (printStorePath(info.outPath)) .exec(); @@ -879,17 +879,18 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) // Try to resolve the derivation at path `original`, with a caching layer // to make it more efficient -std::optional cachedResolve( - LocalStore & store, - const StorePath & original) +std::optional cachedResolve( + LocalStore& store, + const StorePath& original) { + // This is quite dirty and leaky, but will disappear once #4340 is merged + static Sync>> resolutionsCache; { - auto resolutions = drvPathResolutions.lock(); - auto resolvedPathOptIter = resolutions->find(original); - if (resolvedPathOptIter != resolutions->end()) { - auto & [_, resolvedPathOpt] = *resolvedPathOptIter; - if (resolvedPathOpt) - return resolvedPathOpt; + auto resolutions = resolutionsCache.lock(); + auto resolvedDrvIter = resolutions->find(original); + if (resolvedDrvIter != resolutions->end()) { + auto & [_, resolvedDrv] = *resolvedDrvIter; + return *resolvedDrv; } } @@ -898,12 +899,9 @@ std::optional cachedResolve( auto attempt = drv.tryResolve(store); if (!attempt) return std::nullopt; - /* Just compute store path */ - auto pathResolved = - writeDerivation(store, *std::move(attempt), NoRepair, true); /* Store in memo table. */ - drvPathResolutions.lock()->insert_or_assign(original, pathResolved); - return pathResolved; + resolutionsCache.lock()->insert_or_assign(original, *attempt); + return *attempt; } std::map> @@ -933,26 +931,24 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) auto drv = readDerivation(path); - for (auto & output : drv.outputsAndOptPaths(*this)) { - outputs.emplace(output.first, std::nullopt); - } - auto resolvedDrv = cachedResolve(*this, path); - if (!resolvedDrv) + if (!resolvedDrv) { + for (auto& [outputName, _] : drv.outputsAndOptPaths(*this)) { + if (!outputs.count(outputName)) + outputs.emplace(outputName, std::nullopt); + } return outputs; + } - retrySQLite([&]() { - auto state(_state.lock()); - path = *resolvedDrv; - auto useQueryDerivationOutputs{ - state->stmts->QueryAllRealisedOutputs.use()(path.to_string())}; - - while (useQueryDerivationOutputs.next()) - outputs.insert_or_assign( - useQueryDerivationOutputs.getStr(0), - parseStorePath(useQueryDerivationOutputs.getStr(1))); - }); + auto resolvedDrvHashes = staticOutputHashes(*this, *resolvedDrv); + for (auto& [outputName, hash] : resolvedDrvHashes) { + auto realisation = queryRealisation(DrvOutput{hash, outputName}); + if (realisation) + outputs.insert_or_assign(outputName, realisation->outPath); + else + outputs.insert_or_assign(outputName, std::nullopt); + } return outputs; } @@ -1695,12 +1691,11 @@ std::optional LocalStore::queryRealisation( typedef std::optional Ret; return retrySQLite([&]() -> Ret { auto state(_state.lock()); - auto use(state->stmts->QueryRealisedOutput.use()(id.drvPath.to_string())( + auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())( id.outputName)); if (!use.next()) return std::nullopt; auto outputPath = parseStorePath(use.getStr(0)); - auto resolvedDrv = StorePath(use.getStr(1)); return Ret{ Realisation{.id = id, .outPath = outputPath}}; }); diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 47db1ec9f..47ad90eee 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -7,18 +7,18 @@ namespace nix { MakeError(InvalidDerivationOutputId, Error); DrvOutput DrvOutput::parse(const std::string &strRep) { - const auto &[rawPath, outputs] = parsePathWithOutputs(strRep); - if (outputs.size() != 1) + size_t n = strRep.find("!"); + if (n == strRep.npos) throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep); return DrvOutput{ - .drvPath = StorePath(rawPath), - .outputName = *outputs.begin(), + .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)), + .outputName = strRep.substr(n+1), }; } std::string DrvOutput::to_string() const { - return std::string(drvPath.to_string()) + "!" + outputName; + return strHash() + "!" + outputName; } nlohmann::json Realisation::toJSON() const { diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 08579b739..4b8ead3c5 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -6,11 +6,15 @@ namespace nix { struct DrvOutput { - StorePath drvPath; + // The hash modulo of the derivation + Hash drvHash; std::string outputName; std::string to_string() const; + std::string strHash() const + { return drvHash.to_string(Base16, true); } + static DrvOutput parse(const std::string &); bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); } @@ -18,8 +22,8 @@ struct DrvOutput { private: // Just to make comparison operators easier to write - std::pair to_pair() const - { return std::make_pair(drvPath, outputName); } + std::pair to_pair() const + { return std::make_pair(drvHash, outputName); } }; struct Realisation { -- cgit v1.2.3 From e9b39f6004ec68f062230514534b08033cf133c7 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 11 Dec 2020 21:12:53 +0100 Subject: Restrict the operations on drv outputs in recursive Nix There's currently no way to properly filter them, so disallow them altogether instead. --- src/libstore/build/derivation-goal.cc | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 54b37553a..f494545fb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -2093,18 +2093,14 @@ struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConf } void registerDrvOutput(const Realisation & info) override - { - // XXX: Should we check for something here? Probably, but I'm not sure - // how - next->registerDrvOutput(info); - } + // XXX: This should probably be allowed as a no-op if the realisation + // corresponds to an allowed derivation + { throw Error("registerDrvOutput"); } std::optional queryRealisation(const DrvOutput & id) override - { - // XXX: Should we check for something here? Probably, but I'm not sure - // how - return next->queryRealisation(id); - } + // XXX: This should probably be allowed if the realisation corresponds to + // an allowed derivation + { throw Error("queryRealisation"); } void buildPaths(const std::vector & paths, BuildMode buildMode) override { -- cgit v1.2.3 From 44c3fbc6e03ec518f6174c2b7c21b603973beb91 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 20 Oct 2020 15:03:54 +0200 Subject: Fix `addTextToStore` for binary caches Because of a too eager refactoring, `addTextToStore` used to throw an error because the input wasn't a valid nar. Partially revert that refactoring to wrap the text into a proper nar (using `dumpString`) to make this method work again --- src/libstore/binary-cache-store.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5b081c1ae..94c11355f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -433,7 +433,9 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s if (!repair && isValidPath(path)) return path; - auto source = StringSource { s }; + StringSink sink; + dumpString(s, sink); + auto source = StringSource { *sink.s }; return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { path, nar.first }; info.narSize = nar.second; -- cgit v1.2.3 From 7080321618e29033a8b5dc2f9fc938dcf2df270d Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Dec 2020 10:54:24 +0100 Subject: Use the fs accessor for readInvalidDerivation Extend `FSAccessor::readFile` to allow not checking that the path is a valid one, and rewrite `readInvalidDerivation` using this extended `readFile`. Several places in the code use `readInvalidDerivation`, either because they need to read a derivation that has been written in the store but not registered yet, or more generally to prevent a deadlock because `readDerivation` tries to lock the state, so can't be called from a place where the lock is already held. However, `readInvalidDerivation` implicitely assumes that the store is a `LocalFSStore`, which isn't always the case. The concrete motivation for this is that it's required for `nix copy --from someBinaryCache` to work, which is tremendously useful for the tests. --- src/libstore/fs-accessor.hh | 9 ++++++++- src/libstore/local-fs-store.cc | 8 ++++---- src/libstore/nar-accessor.cc | 2 +- src/libstore/remote-fs-accessor.cc | 8 ++++---- src/libstore/remote-fs-accessor.hh | 4 ++-- src/libstore/store-api.cc | 21 +++++++++------------ 6 files changed, 28 insertions(+), 24 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 64780a6da..c825e84f2 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -25,7 +25,14 @@ public: virtual StringSet readDirectory(const Path & path) = 0; - virtual std::string readFile(const Path & path) = 0; + /** + * Read a file inside the store. + * + * If `requireValidPath` is set to `true` (the default), the path must be + * inside a valid store path, otherwise it just needs to be physically + * present (but not necessarily properly registered) + */ + virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; virtual std::string readLink(const Path & path) = 0; }; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index e7c3dae92..6de13c73a 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -19,10 +19,10 @@ struct LocalStoreAccessor : public FSAccessor LocalStoreAccessor(ref store) : store(store) { } - Path toRealPath(const Path & path) + Path toRealPath(const Path & path, bool requireValidPath = true) { auto storePath = store->toStorePath(path).first; - if (!store->isValidPath(storePath)) + if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -61,9 +61,9 @@ struct LocalStoreAccessor : public FSAccessor return res; } - std::string readFile(const Path & path) override + std::string readFile(const Path & path, bool requireValidPath = true) override { - return nix::readFile(toRealPath(path)); + return nix::readFile(toRealPath(path, requireValidPath)); } std::string readLink(const Path & path) override diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 1427a0f98..784ebb719 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -203,7 +203,7 @@ struct NarAccessor : public FSAccessor return res; } - std::string readFile(const Path & path) override + std::string readFile(const Path & path, bool requireValidPath = true) override { auto i = get(path); if (i.type != FSAccessor::Type::tRegular) diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 63bde92de..f43456f0b 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -43,13 +43,13 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & } } -std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) +std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) { auto path = canonPath(path_); auto [storePath, restPath] = store->toStorePath(path); - if (!store->isValidPath(storePath)) + if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); auto i = nars.find(std::string(storePath.hashPart())); @@ -113,9 +113,9 @@ StringSet RemoteFSAccessor::readDirectory(const Path & path) return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path) +std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) { - auto res = fetch(path); + auto res = fetch(path, requireValidPath); return res.first->readFile(res.second); } diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 347cf5764..594852d0e 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -14,7 +14,7 @@ class RemoteFSAccessor : public FSAccessor Path cacheDir; - std::pair, Path> fetch(const Path & path_); + std::pair, Path> fetch(const Path & path_, bool requireValidPath = true); friend class BinaryCacheStore; @@ -32,7 +32,7 @@ public: StringSet readDirectory(const Path & path) override; - std::string readFile(const Path & path) override; + std::string readFile(const Path & path, bool requireValidPath = true) override; std::string readLink(const Path & path) override; }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7bf9235b2..25e28cffa 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1018,26 +1018,23 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) return readDerivation(drvPath); } - -Derivation Store::readDerivation(const StorePath & drvPath) +Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) { - auto accessor = getFSAccessor(); + auto accessor = store.getFSAccessor(); try { - return parseDerivation(*this, - accessor->readFile(printStorePath(drvPath)), + return parseDerivation(store, + accessor->readFile(store.printStorePath(drvPath), requireValidPath), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { - throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); + throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); } } +Derivation Store::readDerivation(const StorePath & drvPath) +{ return readDerivationCommon(*this, drvPath, true); } + Derivation Store::readInvalidDerivation(const StorePath & drvPath) -{ - return parseDerivation( - *this, - readFile(Store::toRealPath(drvPath)), - Derivation::nameFromPath(drvPath)); -} +{ return readDerivationCommon(*this, drvPath, false); } } -- cgit v1.2.3 From 6e899278d305da904fb766937f56344841c022b3 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 6 Nov 2020 09:51:17 +0100 Subject: Better detect when `buildPaths` would be a no-op MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `buildPaths` can be called even for stores where it's not defined in case it's bound to be a no-op. The “no-op detection” mechanism was only detecting the case wher `buildPaths` was called on a set of (non-drv) paths that were already present on the store. This commit extends this mechanism to also detect the case where `buildPaths` is called on a set of derivation outputs which are already built on the store. This only works with the ca-derivations flag. It could be possible to extend this to also work without it, but it would add quite a bit of complexity, and it's not used without it anyways. --- src/libstore/store-api.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7bf9235b2..50905bb33 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -729,9 +729,17 @@ void Store::buildPaths(const std::vector & paths, BuildMod StorePathSet paths2; for (auto & path : paths) { - if (path.path.isDerivation()) - unsupported("buildPaths"); - paths2.insert(path.path); + if (path.path.isDerivation()) { + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto & outputName : path.outputs) { + if (!queryRealisation({path.path, outputName})) + unsupported("buildPaths"); + } + } else + unsupported("buildPaths"); + + } else + paths2.insert(path.path); } if (queryValidPaths(paths2).size() != paths2.size()) -- cgit v1.2.3 From 962b82ef25069893779ed56d31e44814793f9273 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 15 Dec 2020 09:34:45 +0100 Subject: Fix BinaryCacheStore::registerDrvOutput Was crashing because coercing a json document into a string is only valid if the json is a string, otherwise we need to call `.dump()` --- src/libstore/binary-cache-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 94c11355f..4f5f8607d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -460,7 +460,7 @@ std::optional BinaryCacheStore::queryRealisation(const DrvOut void BinaryCacheStore::registerDrvOutput(const Realisation& info) { auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; - upsertFile(filePath, info.toJSON(), "application/json"); + upsertFile(filePath, info.toJSON().dump(), "application/json"); } ref BinaryCacheStore::getFSAccessor() -- cgit v1.2.3 From cac8d5b742ec0cb80ad7232e20f63c74a217e545 Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 16 Dec 2020 13:36:17 +0100 Subject: Don't ignore an absent drv file in queryPartialDrvOutputMap This ignore was here because `queryPartialDrvOutputMap` was used both 1. as a cache to avoid having to re-read the derivation (when gc-ing for example), and 2. as the source of truth for ca realisations The use-case 2. required it to be able to work even when the derivation wasn't there anymore (see https://github.com/NixOS/nix/issues/4138). However, this use-case is now handled by `queryRealisation`, meaning that we can safely error out if the derivation isn't there anymore --- src/libstore/local-store.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1539c94e2..20bbc73cf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -912,12 +912,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) auto state(_state.lock()); std::map> outputs; uint64_t drvId; - try { - drvId = queryValidPathId(*state, path); - } catch (InvalidPath&) { - // Ignore non-existing drvs as they might still have an output map - // defined if ca-derivations is enabled - } + drvId = queryValidPathId(*state, path); auto use(state->stmts->QueryDerivationOutputs.use()(drvId)); while (use.next()) outputs.insert_or_assign( -- cgit v1.2.3 From 4d458394991f3086c3c9c306d000e6c0058c4fa7 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 17 Dec 2020 11:35:24 +0100 Subject: Fix the detection of already built drv outputs PRs #4370 and #4348 had a bad interaction in that the second broke the fist one in a not trivial way. The issue was that since #4348 the logic for detecting whether a derivation output is already built requires some logic that was specific to the `LocalStore`. It happens though that most of this logic could be upstreamed to any `Store`, which is what this commit does. --- src/libstore/derivations.cc | 32 +++++++++++++++++++++++++++++++- src/libstore/derivations.hh | 4 ++++ src/libstore/local-store.cc | 45 ++++----------------------------------------- src/libstore/local-store.hh | 2 +- src/libstore/store-api.cc | 39 +++++++++++++++++++++++++++++++-------- src/libstore/store-api.hh | 9 +++++++-- 6 files changed, 78 insertions(+), 53 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 5bcc7f012..7466c7d41 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -745,7 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolve(Store & store) { +std::optional Derivation::tryResolveUncached(Store & store) { BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation @@ -771,4 +771,34 @@ std::optional Derivation::tryResolve(Store & store) { return resolved; } +std::optional Derivation::tryResolve(Store& store) +{ + auto drvPath = writeDerivation(store, *this, NoRepair, false); + return Derivation::tryResolve(store, drvPath); +} + +std::optional Derivation::tryResolve(Store& store, const StorePath& drvPath) +{ + // This is quite dirty and leaky, but will disappear once #4340 is merged + static Sync>> resolutionsCache; + + { + auto resolutions = resolutionsCache.lock(); + auto resolvedDrvIter = resolutions->find(drvPath); + if (resolvedDrvIter != resolutions->end()) { + auto & [_, resolvedDrv] = *resolvedDrvIter; + return *resolvedDrv; + } + } + + /* Try resolve drv and use that path instead. */ + auto drv = store.readDerivation(drvPath); + auto attempt = drv.tryResolveUncached(store); + if (!attempt) + return std::nullopt; + /* Store in memo table. */ + resolutionsCache.lock()->insert_or_assign(drvPath, *attempt); + return *attempt; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 4e5985fab..3d8f19aef 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -138,10 +138,14 @@ struct Derivation : BasicDerivation 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); + static std::optional tryResolve(Store & store, const StorePath & drvPath); Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } + +private: + std::optional tryResolveUncached(Store & store); }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 20bbc73cf..e9f9bde4d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -877,35 +877,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) }); } -// Try to resolve the derivation at path `original`, with a caching layer -// to make it more efficient -std::optional cachedResolve( - LocalStore& store, - const StorePath& original) -{ - // This is quite dirty and leaky, but will disappear once #4340 is merged - static Sync>> resolutionsCache; - { - auto resolutions = resolutionsCache.lock(); - auto resolvedDrvIter = resolutions->find(original); - if (resolvedDrvIter != resolutions->end()) { - auto & [_, resolvedDrv] = *resolvedDrvIter; - return *resolvedDrv; - } - } - - /* Try resolve drv and use that path instead. */ - auto drv = store.readDerivation(original); - auto attempt = drv.tryResolve(store); - if (!attempt) - return std::nullopt; - /* Store in memo table. */ - resolutionsCache.lock()->insert_or_assign(original, *attempt); - return *attempt; -} std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) +LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) { auto path = path_; auto outputs = retrySQLite>>([&]() { @@ -924,20 +898,9 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) if (!settings.isExperimentalFeatureEnabled("ca-derivations")) return outputs; - auto drv = readDerivation(path); - - auto resolvedDrv = cachedResolve(*this, path); - - if (!resolvedDrv) { - for (auto& [outputName, _] : drv.outputsAndOptPaths(*this)) { - if (!outputs.count(outputName)) - outputs.emplace(outputName, std::nullopt); - } - return outputs; - } - - auto resolvedDrvHashes = staticOutputHashes(*this, *resolvedDrv); - for (auto& [outputName, hash] : resolvedDrvHashes) { + auto drv = readInvalidDerivation(path); + auto drvHashes = staticOutputHashes(*this, drv); + for (auto& [outputName, hash] : drvHashes) { auto realisation = queryRealisation(DrvOutput{hash, outputName}); if (realisation) outputs.insert_or_assign(outputName, realisation->outPath); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 69559e346..877dba742 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryDerivationOutputMapNoResolve(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 50905bb33..2cd39ab11 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -366,6 +366,29 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } +std::map> Store::queryDerivationOutputMapNoResolve(const StorePath & path) +{ + std::map> outputs; + auto drv = readInvalidDerivation(path); + for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) { + outputs.emplace(outputName, output.second); + } + return outputs; +} + +std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) +{ + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto resolvedDrv = Derivation::tryResolve(*this, path); + if (resolvedDrv) { + auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true); + if (isValidPath(resolvedDrvPath)) + return queryDerivationOutputMapNoResolve(resolvedDrvPath); + } + } + return queryDerivationOutputMapNoResolve(path); +} + OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; @@ -730,14 +753,14 @@ void Store::buildPaths(const std::vector & paths, BuildMod for (auto & path : paths) { if (path.path.isDerivation()) { - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - for (auto & outputName : path.outputs) { - if (!queryRealisation({path.path, outputName})) - unsupported("buildPaths"); - } - } else - unsupported("buildPaths"); - + auto outPaths = queryPartialDerivationOutputMap(path.path); + for (auto & outputName : path.outputs) { + auto currentOutputPathIter = outPaths.find(outputName); + if (currentOutputPathIter == outPaths.end() || + !currentOutputPathIter->second || + !isValidPath(*currentOutputPathIter->second)) + unsupported("buildPaths"); + } } else paths2.insert(path.path); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7cdadc1f3..ce95b78b1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -416,8 +416,13 @@ public: /* Query the mapping outputName => outputPath for the given derivation. All outputs are mentioned so ones mising the mapping are mapped to `std::nullopt`. */ - virtual std::map> queryPartialDerivationOutputMap(const StorePath & path) - { unsupported("queryPartialDerivationOutputMap"); } + virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); + + /* + * Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve + * the derivation + */ + virtual std::map> queryDerivationOutputMapNoResolve(const StorePath & path); /* Query the mapping outputName=>outputPath for the given derivation. Assume every output has a mapping and throw an exception otherwise. */ -- cgit v1.2.3 From 1a1af75338cb9ed28dc00de2e696d8efc5d37287 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 15:33:12 +0000 Subject: Overhaul store subclassing We embrace virtual the rest of the way, and get rid of the `assert(false)` 0-param constructors. We also list config base classes first, so the constructor order is always: 1. all the configs 2. all the stores Each in the same order --- src/libstore/binary-cache-store.hh | 2 +- src/libstore/build/derivation-goal.cc | 11 ++++++++--- src/libstore/dummy-store.cc | 3 ++- src/libstore/http-binary-cache-store.cc | 5 ++++- src/libstore/legacy-ssh-store.cc | 3 ++- src/libstore/local-binary-cache-store.cc | 5 ++++- src/libstore/local-fs-store.hh | 2 +- src/libstore/local-store.cc | 2 ++ src/libstore/local-store.hh | 2 +- src/libstore/remote-store.cc | 4 ++-- src/libstore/remote-store.hh | 2 +- src/libstore/s3-binary-cache-store.cc | 11 ++++++++++- src/libstore/s3-binary-cache-store.hh | 6 ++---- src/libstore/ssh-store.cc | 4 +++- src/libstore/store-api.hh | 20 +------------------- src/libstore/uds-remote-store.cc | 3 +++ src/libstore/uds-remote-store.hh | 7 +------ 17 files changed, 48 insertions(+), 44 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 07a8b2beb..443a53cac 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -24,7 +24,7 @@ struct BinaryCacheStoreConfig : virtual StoreConfig "enable multi-threading compression, available for xz only currently"}; }; -class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig +class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store { private: diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index f494545fb..47d11dc53 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1985,7 +1985,7 @@ void DerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } -struct RestrictedStoreConfig : LocalFSStoreConfig +struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; const std::string name() { return "Restricted Store"; } @@ -1994,14 +1994,19 @@ struct RestrictedStoreConfig : LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore { ref next; DerivationGoal & goal; RestrictedStore(const Params & params, ref next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RestrictedStoreConfig(params) + , Store(params) + , LocalFSStore(params) + , next(next), goal(goal) { } Path getRealStoreDir() override diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 91fc178db..3c7caf8f2 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -9,7 +9,7 @@ struct DummyStoreConfig : virtual StoreConfig { const std::string name() override { return "Dummy Store"; } }; -struct DummyStore : public Store, public virtual DummyStoreConfig +struct DummyStore : public virtual DummyStoreConfig, public virtual Store { DummyStore(const std::string scheme, const std::string uri, const Params & params) : DummyStore(params) @@ -17,6 +17,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig DummyStore(const Params & params) : StoreConfig(params) + , DummyStoreConfig(params) , Store(params) { } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 9d2a89f96..0a3afcd51 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -15,7 +15,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "Http Binary Cache Store"; } }; -class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig +class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: @@ -36,6 +36,9 @@ public: const Path & _cacheUri, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , HttpBinaryCacheStoreConfig(params) + , Store(params) , BinaryCacheStore(params) , cacheUri(scheme + "://" + _cacheUri) { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index ad1779aea..253c0033e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -22,7 +22,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig const std::string name() override { return "Legacy SSH Store"; } }; -struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig +struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store { // Hack for getting remote build log output. // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in @@ -48,6 +48,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig LegacySSHStore(const string & scheme, const string & host, const Params & params) : StoreConfig(params) + , LegacySSHStoreConfig(params) , Store(params) , host(host) , connections(make_ref>( diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index bb7464989..a58b7733f 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -11,7 +11,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "Local Binary Cache Store"; } }; -class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig +class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: @@ -24,6 +24,9 @@ public: const Path & binaryCacheDir, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , LocalBinaryCacheStoreConfig(params) + , Store(params) , BinaryCacheStore(params) , binaryCacheDir(binaryCacheDir) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 8eccd8236..55941b771 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -20,7 +20,7 @@ struct LocalFSStoreConfig : virtual StoreConfig "log", "directory where Nix will store state"}; }; -class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store { public: diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e9f9bde4d..c52d4b62a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -100,6 +100,8 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) LocalStore::LocalStore(const Params & params) : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) , Store(params) , LocalFSStore(params) , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 877dba742..ae9497b2e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -43,7 +43,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig }; -class LocalStore : public LocalFSStore, public virtual LocalStoreConfig +class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore { private: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f1f4d0516..be07f02dc 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -77,8 +77,8 @@ void write(const Store & store, Sink & out, const std::optional /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) - : Store(params) - , RemoteStoreConfig(params) + : RemoteStoreConfig(params) + , Store(params) , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index fdd53e6ed..b3a9910a3 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,7 +29,7 @@ struct RemoteStoreConfig : virtual StoreConfig /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public virtual Store, public virtual RemoteStoreConfig +class RemoteStore : public virtual RemoteStoreConfig, public virtual Store { public: diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d6edafd7e..6bfbee044 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -177,6 +177,11 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } +S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) + : BinaryCacheStoreConfig(params) + , BinaryCacheStore(params) +{ } + struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; @@ -195,7 +200,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "S3 Binary Cache Store"; } }; -struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig +struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { std::string bucketName; @@ -208,6 +213,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache const std::string & bucketName, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , S3BinaryCacheStoreConfig(params) + , Store(params) + , BinaryCacheStore(params) , S3BinaryCacheStore(params) , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 4d43fe4d2..bce828b11 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -6,13 +6,11 @@ namespace nix { -class S3BinaryCacheStore : public BinaryCacheStore +class S3BinaryCacheStore : public virtual BinaryCacheStore { protected: - S3BinaryCacheStore(const Params & params) - : BinaryCacheStore(params) - { } + S3BinaryCacheStore(const Params & params); public: diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 08d0bd565..17c258201 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -20,12 +20,14 @@ struct SSHStoreConfig : virtual RemoteStoreConfig const std::string name() override { return "SSH Store"; } }; -class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig +class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { public: SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) + , RemoteStoreConfig(params) + , SSHStoreConfig(params) , Store(params) , RemoteStore(params) , host(host) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ce95b78b1..9bcff08eb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -175,25 +175,7 @@ struct StoreConfig : public Config { using Config::Config; - /** - * When constructing a store implementation, we pass in a map `params` of - * parameters that's supposed to initialize the associated config. - * To do that, we must use the `StoreConfig(StringMap & params)` - * constructor, so we'd like to `delete` its default constructor to enforce - * it. - * - * However, actually deleting it means that all the subclasses of - * `StoreConfig` will have their default constructor deleted (because it's - * supposed to call the deleted default constructor of `StoreConfig`). But - * because we're always using virtual inheritance, the constructors of - * child classes will never implicitely call this one, so deleting it will - * be more painful than anything else. - * - * So we `assert(false)` here to ensure at runtime that the right - * constructor is always called without having to redefine a custom - * constructor for each `*Config` class. - */ - StoreConfig() { assert(false); } + StoreConfig() = delete; virtual ~StoreConfig() { } diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 24f3e9c6d..cac4fa036 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -15,6 +15,9 @@ namespace nix { UDSRemoteStore::UDSRemoteStore(const Params & params) : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + , UDSRemoteStoreConfig(params) , Store(params) , LocalFSStore(params) , RemoteStore(params) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index e5de104c9..ddc7716cd 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -14,15 +14,10 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon { } - UDSRemoteStoreConfig() - : UDSRemoteStoreConfig(Store::Params({})) - { - } - const std::string name() override { return "Local Daemon Store"; } }; -class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig +class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore { public: -- cgit v1.2.3 From 57062179ce36e35715284d2ef570f8cb0b90198d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 16:05:09 +0000 Subject: Move some PKI stuff from LocalStore to Store --- src/libstore/local-store.cc | 9 --------- src/libstore/local-store.hh | 12 ------------ src/libstore/misc.cc | 9 +++++++++ src/libstore/store-api.hh | 13 +++++++++++++ 4 files changed, 22 insertions(+), 21 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c52d4b62a..1eb2dec75 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1092,15 +1092,6 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) } -const PublicKeys & LocalStore::getPublicKeys() -{ - auto state(_state.lock()); - if (!state->publicKeys) - state->publicKeys = std::make_unique(getDefaultPublicKeys()); - return *state->publicKeys; -} - - void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ae9497b2e..d97645058 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -35,10 +35,6 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - Setting requireSigs{(StoreConfig*) this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; - const std::string name() override { return "Local Store"; } }; @@ -75,8 +71,6 @@ private: minFree but not much below availAfterGC, then there is no point in starting a new GC. */ uint64_t availAfterGC = std::numeric_limits::max(); - - std::unique_ptr publicKeys; }; Sync _state; @@ -94,12 +88,6 @@ public: const Path tempRootsDir; const Path fnTempRoots; -private: - - const PublicKeys & getPublicKeys(); - -public: - // Hack for build-remote.cc. PathSet locksHeld; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ad4dccef9..0d4190a56 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -282,4 +282,13 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) } +const PublicKeys & Store::getPublicKeys() +{ + auto cryptoState(_cryptoState.lock()); + if (!cryptoState->publicKeys) + cryptoState->publicKeys = std::make_unique(getDefaultPublicKeys()); + return *cryptoState->publicKeys; +} + + } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9bcff08eb..e3de6db17 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -189,6 +189,10 @@ struct StoreConfig : public Config const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + Setting requireSigs{this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; @@ -710,11 +714,20 @@ public: return toRealPath(printStorePath(storePath)); } + const PublicKeys & getPublicKeys(); + virtual void createUser(const std::string & userName, uid_t userId) { } protected: + struct CryptoState + { + std::unique_ptr publicKeys; + }; + + Sync _cryptoState; + Stats stats; /* Unsupported methods. */ -- cgit v1.2.3 From 450c3500f1e3fb619636c0a29d65300020f99d7d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 17:36:52 +0000 Subject: Crudely make worker only provide a Store, not LocalStore We downcast in a few places, this will be refactored to be better later. --- src/libstore/build/derivation-goal.cc | 66 +++++++++++++++++++++++------------ src/libstore/build/worker.cc | 6 ++-- src/libstore/build/worker.hh | 9 +++-- 3 files changed, 54 insertions(+), 27 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 47d11dc53..de32f60db 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -848,14 +848,16 @@ void DerivationGoal::buildDone() So instead, check if the disk is (nearly) full now. If so, we don't mark this build as a permanent failure. */ #if HAVE_STATVFS - uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable - struct statvfs st; - if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; - if (statvfs(tmpDir.c_str(), &st) == 0 && - (uint64_t) st.f_bavail * st.f_bsize < required) - diskFull = true; + if (auto localStore = dynamic_cast(&worker.store)) { + uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(localStore->realStoreDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (uint64_t) st.f_bavail * st.f_bsize < required) + diskFull = true; + } #endif deleteTmpDir(false); @@ -1215,12 +1217,15 @@ void DerivationGoal::startBuilder() useChroot = !(derivationIsImpure(derivationType)) && !noChroot; } - if (worker.store.storeDir != worker.store.realStoreDir) { - #if __linux__ - useChroot = true; - #else - throw Error("building using a diverted store is not supported on this platform"); - #endif + if (auto localStoreP = dynamic_cast(&worker.store)) { + auto & localStore = *localStoreP; + if (localStore.storeDir != localStore.realStoreDir) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } } /* Create a temporary directory where the build will take @@ -2182,7 +2187,8 @@ void DerivationGoal::startDaemon() Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; - params["root"] = worker.store.rootDir; + if (auto localStore = dynamic_cast(&worker.store)) + params["root"] = localStore->rootDir; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; auto store = make_ref(params, @@ -3246,7 +3252,13 @@ void DerivationGoal::registerOutputs() } } + auto localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + Unsupported("Can only register outputs with local store"); + auto & localStore = *localStoreP; + if (buildMode == bmCheck) { + if (!worker.store.isValidPath(newInfo.path)) continue; ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); if (newInfo.narHash != oldInfo.narHash) { @@ -3271,8 +3283,8 @@ void DerivationGoal::registerOutputs() /* Since we verified the build, it's now ultimately trusted. */ if (!oldInfo.ultimate) { oldInfo.ultimate = true; - worker.store.signPathInfo(oldInfo); - worker.store.registerValidPaths({{oldInfo.path, oldInfo}}); + localStore.signPathInfo(oldInfo); + localStore.registerValidPaths({{oldInfo.path, oldInfo}}); } continue; @@ -3288,13 +3300,13 @@ void DerivationGoal::registerOutputs() } if (curRound == nrRounds) { - worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() + localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences() worker.markContentsGood(newInfo.path); } newInfo.deriver = drvPath; newInfo.ultimate = true; - worker.store.signPathInfo(newInfo); + localStore.signPathInfo(newInfo); finish(newInfo.path); @@ -3302,7 +3314,7 @@ void DerivationGoal::registerOutputs() isn't statically known so that we can safely unlock the path before the next iteration */ if (newInfo.ca) - worker.store.registerValidPaths({{newInfo.path, newInfo}}); + localStore.registerValidPaths({{newInfo.path, newInfo}}); infos.emplace(outputName, std::move(newInfo)); } @@ -3375,11 +3387,16 @@ void DerivationGoal::registerOutputs() paths referenced by each of them. If there are cycles in the outputs, this will fail. */ { + auto localStoreP = dynamic_cast(&worker.store); + if (!localStoreP) + Unsupported("Can only register outputs with local store"); + auto & localStore = *localStoreP; + ValidPathInfos infos2; for (auto & [outputName, newInfo] : infos) { infos2.insert_or_assign(newInfo.path, newInfo); } - worker.store.registerValidPaths(infos2); + localStore.registerValidPaths(infos2); } /* In case of a fixed-output derivation hash mismatch, throw an @@ -3577,7 +3594,12 @@ Path DerivationGoal::openLogFile() auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); /* Create a log file. */ - Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); + Path logDir; + if (auto localStore = dynamic_cast(&worker.store)) + logDir = localStore->logDir; + else + logDir = settings.nixLogDir; + Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2)); createDirs(dir); Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 6c96a93bd..a9575fb0f 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -8,7 +8,7 @@ namespace nix { -Worker::Worker(LocalStore & store) +Worker::Worker(Store & store) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) , actSubstitutions(*logger, actCopyPaths) @@ -229,7 +229,9 @@ void Worker::run(const Goals & _topGoals) checkInterrupt(); - store.autoGC(false); + // TODO GC interface? + if (auto localStore = dynamic_cast(&store)) + localStore->autoGC(false); /* Call every wake goal (in the ordering established by CompareGoalPtrs). */ diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index bf8cc4586..82e711191 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -2,9 +2,12 @@ #include "types.hh" #include "lock.hh" -#include "local-store.hh" +#include "store-api.hh" #include "goal.hh" +#include +#include + namespace nix { /* Forward definition. */ @@ -102,7 +105,7 @@ public: /* Set if at least one derivation is not deterministic in check mode. */ bool checkMismatch; - LocalStore & store; + Store & store; std::unique_ptr hook; @@ -124,7 +127,7 @@ public: it answers with "decline-permanently", we don't try again. */ bool tryBuildHook = true; - Worker(LocalStore & store); + Worker(Store & store); ~Worker(); /* Make a goal (with caching). */ -- cgit v1.2.3 From 85f2e9e8fa4f7452a05cfffc901d118a7c861d0a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 17:54:57 +0000 Subject: Expose schedule entrypoints to all stores Remote stores still override so the other end schedules. --- src/libstore/binary-cache-store.hh | 7 --- src/libstore/build/entry-points.cc | 108 ++++++++++++++++++++++++++++++++ src/libstore/build/local-store-build.cc | 108 -------------------------------- src/libstore/dummy-store.cc | 7 --- src/libstore/local-store.hh | 9 --- src/libstore/store-api.cc | 23 ------- src/libstore/store-api.hh | 6 +- 7 files changed, 111 insertions(+), 157 deletions(-) create mode 100644 src/libstore/build/entry-points.cc delete mode 100644 src/libstore/build/local-store-build.cc (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 443a53cac..c2163166c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -108,13 +108,6 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { unsupported("buildDerivation"); } - - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } - ref getFSAccessor() override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc new file mode 100644 index 000000000..9f97d40ba --- /dev/null +++ b/src/libstore/build/entry-points.cc @@ -0,0 +1,108 @@ +#include "machines.hh" +#include "worker.hh" +#include "substitution-goal.hh" +#include "derivation-goal.hh" + +namespace nix { + +void Store::buildPaths(const std::vector & drvPaths, BuildMode buildMode) +{ + Worker worker(*this); + + Goals goals; + for (auto & path : drvPaths) { + if (path.path.isDerivation()) + goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); + else + goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); + } + + worker.run(goals); + + StorePathSet failed; + std::optional ex; + for (auto & i : goals) { + if (i->ex) { + if (ex) + logError(i->ex->info()); + else + ex = i->ex; + } + if (i->exitCode != Goal::ecSuccess) { + if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); + else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); + } + } + + if (failed.size() == 1 && ex) { + ex->status = worker.exitStatus(); + throw *ex; + } else if (!failed.empty()) { + if (ex) logError(ex->info()); + throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + } +} + +BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + Worker worker(*this); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); + + BuildResult result; + + try { + worker.run(Goals{goal}); + result = goal->getResult(); + } catch (Error & e) { + result.status = BuildResult::MiscFailure; + result.errorMsg = e.msg(); + } + + return result; +} + + +void Store::ensurePath(const StorePath & path) +{ + /* If the path is already valid, we're done. */ + if (isValidPath(path)) return; + + Worker worker(*this); + GoalPtr goal = worker.makeSubstitutionGoal(path); + Goals goals = {goal}; + + worker.run(goals); + + if (goal->exitCode != Goal::ecSuccess) { + if (goal->ex) { + goal->ex->status = worker.exitStatus(); + throw *goal->ex; + } else + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + } +} + + +void LocalStore::repairPath(const StorePath & path) +{ + Worker worker(*this); + GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); + Goals goals = {goal}; + + worker.run(goals); + + if (goal->exitCode != Goal::ecSuccess) { + /* Since substituting the path didn't work, if we have a valid + deriver, then rebuild the deriver. */ + auto info = queryPathInfo(path); + if (info->deriver && isValidPath(*info->deriver)) { + goals.clear(); + goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); + worker.run(goals); + } else + throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); + } +} + +} diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/local-store-build.cc deleted file mode 100644 index c91cda2fd..000000000 --- a/src/libstore/build/local-store-build.cc +++ /dev/null @@ -1,108 +0,0 @@ -#include "machines.hh" -#include "worker.hh" -#include "substitution-goal.hh" -#include "derivation-goal.hh" - -namespace nix { - -void LocalStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode) -{ - Worker worker(*this); - - Goals goals; - for (auto & path : drvPaths) { - if (path.path.isDerivation()) - goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); - else - goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); - } - - worker.run(goals); - - StorePathSet failed; - std::optional ex; - for (auto & i : goals) { - if (i->ex) { - if (ex) - logError(i->ex->info()); - else - ex = i->ex; - } - if (i->exitCode != Goal::ecSuccess) { - if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); - else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); - } - } - - if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); - throw *ex; - } else if (!failed.empty()) { - if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); - } -} - -BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - - BuildResult result; - - try { - worker.run(Goals{goal}); - result = goal->getResult(); - } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - - return result; -} - - -void LocalStore::ensurePath(const StorePath & path) -{ - /* If the path is already valid, we're done. */ - if (isValidPath(path)) return; - - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->status = worker.exitStatus(); - throw *goal->ex; - } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); - } -} - - -void LocalStore::repairPath(const StorePath & path) -{ - Worker worker(*this); - GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); - Goals goals = {goal}; - - worker.run(goals); - - if (goal->exitCode != Goal::ecSuccess) { - /* Since substituting the path didn't work, if we have a valid - deriver, then rebuild the deriver. */ - auto info = queryPathInfo(path); - if (info->deriver && isValidPath(*info->deriver)) { - goals.clear(); - goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); - worker.run(goals); - } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); - } -} - -} diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 3c7caf8f2..8f26af685 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -55,13 +55,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { unsupported("buildDerivation"); } - std::optional queryRealisation(const DrvOutput&) override { unsupported("queryRealisation"); } }; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d97645058..aa5de31f0 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -133,15 +133,6 @@ public: StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; - void buildPaths( - const std::vector & paths, - BuildMode buildMode) override; - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override; - - void ensurePath(const StorePath & path) override; - void addTempRoot(const StorePath & path) override; void addIndirectRoot(const Path & path) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7aca22bde..f12a564a1 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -747,29 +747,6 @@ const Store::Stats & Store::getStats() } -void Store::buildPaths(const std::vector & paths, BuildMode buildMode) -{ - StorePathSet paths2; - - for (auto & path : paths) { - if (path.path.isDerivation()) { - auto outPaths = queryPartialDerivationOutputMap(path.path); - for (auto & outputName : path.outputs) { - auto currentOutputPathIter = outPaths.find(outputName); - if (currentOutputPathIter == outPaths.end() || - !currentOutputPathIter->second || - !isValidPath(*currentOutputPathIter->second)) - unsupported("buildPaths"); - } - } else - paths2.insert(path.path); - } - - if (queryValidPaths(paths2).size() != paths2.size()) - unsupported("buildPaths"); -} - - void copyStorePath(ref srcStore, ref dstStore, const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e3de6db17..4db980fe9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -523,17 +523,17 @@ public: explicitly choosing to allow it). */ virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) = 0; + BuildMode buildMode = bmNormal); /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the path). */ - virtual void ensurePath(const StorePath & path) = 0; + virtual void ensurePath(const StorePath & path); /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ virtual void addTempRoot(const StorePath & path) - { unsupported("addTempRoot"); } + { warn("not creating temp root, store doesn't support GC"); } /* Add an indirect root, which is merely a symlink to `path' from /nix/var/nix/gcroots/auto/. `path' is supposed -- cgit v1.2.3 From fed123724679de89d3f56a4c01b5c4c96f93e584 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 20 Dec 2020 19:55:21 +0000 Subject: Test nix-build with non-local-store --store Just a few small things needed fixing! --- src/libstore/build/derivation-goal.cc | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index de32f60db..17f39a86e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -592,9 +592,17 @@ void DerivationGoal::tryToBuild() PathSet lockFiles; /* FIXME: Should lock something like the drv itself so we don't build same CA drv concurrently */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); + if (dynamic_cast(&worker.store)) + /* If we aren't a local store, we might need to use the local store as + a build remote, but that would cause a deadlock. */ + /* FIXME: Make it so we can use ourselves as a build remote even if we + are the local store (separate locking for building vs scheduling? */ + /* FIXME: find some way to lock for scheduling for the other stores so + a forking daemon with --store still won't farm out redundant builds. + */ + for (auto & i : drv->outputsAndOptPaths(worker.store)) + if (i.second.second) + lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); if (!outputLocks.lockPaths(lockFiles, "", false)) { if (!actLock) @@ -680,6 +688,12 @@ void DerivationGoal::tryLocalBuild() { /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ + if (!dynamic_cast(&worker.store)) { + throw Error( + "unable to build with a primary store that isn't a local store; " + "either pass a different '--store' or enable remote builds." + "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + } unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { worker.waitForBuildSlot(shared_from_this()); -- cgit v1.2.3 From 6262a703636000e525d5c1b877ac28d604a493f0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 28 Dec 2020 17:21:19 +0100 Subject: scanForReferences: Remove misleading comment References have always been determined only by the hash part, not the name or the store prefix. Fixes #4396. --- src/libstore/references.cc | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/references.cc b/src/libstore/references.cc index eb117b5ba..39c4970c6 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -88,9 +88,6 @@ PathSet scanForReferences(Sink & toTee, TeeSink sink { refsSink, toTee }; std::map backMap; - /* For efficiency (and a higher hit rate), just search for the - hash part of the file name. (This assumes that all references - have the form `HASH-bla'). */ for (auto & i : refs) { auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); -- cgit v1.2.3 From 8af4f886e212346afdd1d40789f96f1321da96c5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Jan 2021 11:47:29 +0100 Subject: Fix deadlock in LocalStore::addSignatures() Fixes #4367. --- src/libstore/local-store.cc | 69 ++++++++++++++++++++++++--------------------- src/libstore/local-store.hh | 2 ++ 2 files changed, 39 insertions(+), 32 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c52d4b62a..702e7b136 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -736,54 +736,59 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { try { - callback(retrySQLite>([&]() { + callback(retrySQLite>([&]() { auto state(_state.lock()); + return queryPathInfoInternal(*state, path); + })); - /* Get the path info. */ - auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path))); + } catch (...) { callback.rethrow(); } +} - if (!useQueryPathInfo.next()) - return std::shared_ptr(); - auto id = useQueryPathInfo.getInt(0); +std::shared_ptr LocalStore::queryPathInfoInternal(State & state, const StorePath & path) +{ + /* Get the path info. */ + auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path))); - auto narHash = Hash::dummy; - try { - narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); - } catch (BadHash & e) { - throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); - } + if (!useQueryPathInfo.next()) + return std::shared_ptr(); - auto info = std::make_shared(path, narHash); + auto id = useQueryPathInfo.getInt(0); - info->id = id; + auto narHash = Hash::dummy; + try { + narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); + } catch (BadHash & e) { + throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); + } - info->registrationTime = useQueryPathInfo.getInt(2); + auto info = std::make_shared(path, narHash); - auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3); - if (s) info->deriver = parseStorePath(s); + info->id = id; - /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); + info->registrationTime = useQueryPathInfo.getInt(2); - info->ultimate = useQueryPathInfo.getInt(5) == 1; + auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); + if (s) info->deriver = parseStorePath(s); - s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6); - if (s) info->sigs = tokenizeString(s, " "); + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); - s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + info->ultimate = useQueryPathInfo.getInt(5) == 1; - /* Get the references. */ - auto useQueryReferences(state->stmts->QueryReferences.use()(info->id)); + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6); + if (s) info->sigs = tokenizeString(s, " "); - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); + if (s) info->ca = parseContentAddressOpt(s); - return info; - })); + /* Get the references. */ + auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); - } catch (...) { callback.rethrow(); } + while (useQueryReferences.next()) + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + + return info; } @@ -1608,7 +1613,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si SQLiteTxn txn(state->db); - auto info = std::const_pointer_cast(std::shared_ptr(queryPathInfo(storePath))); + auto info = std::const_pointer_cast(queryPathInfoInternal(*state, storePath)); info->sigs.insert(sigs.begin(), sigs.end()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ae9497b2e..6d29c5960 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -235,6 +235,8 @@ private: void verifyPath(const Path & path, const StringSet & store, PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); + void updatePathInfo(State & state, const ValidPathInfo & info); void upgradeStore6(); -- cgit v1.2.3 From 555152ffe8494190ca42dd481991c9b54759f686 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:04:46 +0100 Subject: crypto.cc: API cleanup and add generate() / to_string() methods --- src/libstore/crypto.cc | 33 ++++++++++++++++++++++++++------- src/libstore/crypto.hh | 24 ++++++++++++++++-------- 2 files changed, 42 insertions(+), 15 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 9ec8abd22..135ced277 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -8,15 +8,15 @@ namespace nix { -static std::pair split(const string & s) +static std::pair split(std::string_view s) { size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) return {"", ""}; - return {std::string(s, 0, colon), std::string(s, colon + 1)}; + return {s.substr(0, colon), s.substr(colon + 1)}; } -Key::Key(const string & s) +Key::Key(std::string_view s) { auto ss = split(s); @@ -29,7 +29,12 @@ Key::Key(const string & s) key = base64Decode(key); } -SecretKey::SecretKey(const string & s) +std::string Key::to_string() const +{ + return name + ":" + base64Encode(key); +} + +SecretKey::SecretKey(std::string_view s) : Key(s) { #if HAVE_SODIUM @@ -45,7 +50,7 @@ SecretKey::SecretKey(const string & s) } #endif -std::string SecretKey::signDetached(const std::string & data) const +std::string SecretKey::signDetached(std::string_view data) const { #if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; @@ -69,7 +74,21 @@ PublicKey SecretKey::toPublicKey() const #endif } -PublicKey::PublicKey(const string & s) +SecretKey SecretKey::generate(std::string_view name) +{ +#if HAVE_SODIUM + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + unsigned char sk[crypto_sign_SECRETKEYBYTES]; + if (crypto_sign_keypair(pk, sk) != 0) + throw Error("key generation failed"); + + return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); +#else + noSodium(); +#endif +} + +PublicKey::PublicKey(std::string_view s) : Key(s) { #if HAVE_SODIUM @@ -84,7 +103,7 @@ bool verifyDetached(const std::string & data, const std::string & sig, #if HAVE_SODIUM auto ss = split(sig); - auto key = publicKeys.find(ss.first); + auto key = publicKeys.find(std::string(ss.first)); if (key == publicKeys.end()) return false; auto sig2 = base64Decode(ss.second); diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 9110af3aa..03f85c103 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -13,32 +13,40 @@ struct Key /* Construct Key from a string in the format ‘:’. */ - Key(const std::string & s); + Key(std::string_view s); + + std::string to_string() const; protected: - Key(const std::string & name, const std::string & key) - : name(name), key(key) { } + Key(std::string_view name, std::string && key) + : name(name), key(std::move(key)) { } }; struct PublicKey; struct SecretKey : Key { - SecretKey(const std::string & s); + SecretKey(std::string_view s); /* Return a detached signature of the given string. */ - std::string signDetached(const std::string & s) const; + std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; + + static SecretKey generate(std::string_view name); + +private: + SecretKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } }; struct PublicKey : Key { - PublicKey(const std::string & data); + PublicKey(std::string_view data); private: - PublicKey(const std::string & name, const std::string & key) - : Key(name, key) { } + PublicKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } friend struct SecretKey; }; -- cgit v1.2.3 From 0df69d96e02ce4c9e17bd33333c5d78313341dd3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Jan 2021 17:56:53 +0100 Subject: Make sodium a required dependency --- src/libstore/crypto.cc | 29 ----------------------------- 1 file changed, 29 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 135ced277..1027469c9 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -2,9 +2,7 @@ #include "util.hh" #include "globals.hh" -#if HAVE_SODIUM #include -#endif namespace nix { @@ -37,70 +35,46 @@ std::string Key::to_string() const SecretKey::SecretKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_SECRETKEYBYTES) throw Error("secret key is not valid"); -#endif } -#if !HAVE_SODIUM -[[noreturn]] static void noSodium() -{ - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -} -#endif - std::string SecretKey::signDetached(std::string_view data) const { -#if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; unsigned long long sigLen; crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), (unsigned char *) key.data()); return name + ":" + base64Encode(std::string((char *) sig, sigLen)); -#else - noSodium(); -#endif } PublicKey SecretKey::toPublicKey() const { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); -#else - noSodium(); -#endif } SecretKey SecretKey::generate(std::string_view name) { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; unsigned char sk[crypto_sign_SECRETKEYBYTES]; if (crypto_sign_keypair(pk, sk) != 0) throw Error("key generation failed"); return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); -#else - noSodium(); -#endif } PublicKey::PublicKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_PUBLICKEYBYTES) throw Error("public key is not valid"); -#endif } bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys) { -#if HAVE_SODIUM auto ss = split(sig); auto key = publicKeys.find(std::string(ss.first)); @@ -113,9 +87,6 @@ bool verifyDetached(const std::string & data, const std::string & sig, return crypto_sign_verify_detached((unsigned char *) sig2.data(), (unsigned char *) data.data(), data.size(), (unsigned char *) key->second.key.data()) == 0; -#else - noSodium(); -#endif } PublicKeys getDefaultPublicKeys() -- cgit v1.2.3 From 08133503494d023b646b3107acf159a5274466ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Jan 2021 21:51:46 +0100 Subject: Add 'nix store prefetch-{file,tarball}' These replace nix-prefetch-url and nix-prefetch-url --unpack, respectively. --- src/libstore/filetransfer.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index afc7e7aa6..45d9ccf89 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -63,7 +63,7 @@ struct FileTransferRequest std::string mimeType; std::function dataCallback; - FileTransferRequest(const std::string & uri) + FileTransferRequest(std::string_view uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() -- cgit v1.2.3 From 6548b89cc4eb214cb4632fd4332c610f2d1f0a9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Jan 2021 12:22:21 +0100 Subject: string2Int(): Return std::optional --- src/libstore/build/derivation-goal.cc | 4 +--- src/libstore/globals.cc | 8 ++++++-- src/libstore/local-store.cc | 4 +++- src/libstore/names.cc | 12 ++++++------ src/libstore/nar-info.cc | 8 ++++++-- src/libstore/profiles.cc | 11 +++++------ src/libstore/store-api.cc | 13 +++++++------ 7 files changed, 34 insertions(+), 26 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index af3ab87a9..35f365795 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1699,12 +1699,10 @@ void DerivationGoal::startBuilder() userNamespaceSync.writeSide = -1; }); - pid_t tmp; auto ss = tokenizeString>(readLine(builderOut.readSide.get())); assert(ss.size() == 2); usingUserNamespace = ss[0] == "1"; - if (!string2Int(ss[1], tmp)) abort(); - pid = tmp; + pid = string2Int(ss[1]).value(); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ad66ef8a8..0531aad9f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -228,8 +228,12 @@ template<> void BaseSetting::convertToArg(Args & args, const std::s void MaxBuildJobsSetting::set(const std::string & str, bool append) { if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); - else if (!string2Int(str, value)) - throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + else { + if (auto n = string2Int(str)) + value = *n; + else + throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 702e7b136..c61f34275 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -66,8 +66,10 @@ int getSchema(Path schemaPath) int curSchema = 0; if (pathExists(schemaPath)) { string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) + auto n = string2Int(s); + if (!n) throw Error("'%1%' is corrupt", schemaPath); + curSchema = *n; } return curSchema; } diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 41e28dc99..ce808accc 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -80,16 +80,16 @@ string nextComponent(string::const_iterator & p, static bool componentsLT(const string & c1, const string & c2) { - int n1, n2; - bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); + auto n1 = string2Int(c1); + auto n2 = string2Int(c2); - if (c1Num && c2Num) return n1 < n2; - else if (c1 == "" && c2Num) return true; + if (n1 && n2) return *n1 < *n2; + else if (c1 == "" && n2) return true; else if (c1 == "pre" && c2 != "pre") return true; else if (c2 == "pre") return false; /* Assume that `2.3a' < `2.3.1'. */ - else if (c2Num) return true; - else if (c1Num) return false; + else if (n2) return true; + else if (n1) return false; else return c1 < c2; } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 3454f34bb..49079388a 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -46,14 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) throw corrupt(); + auto n = string2Int(value); + if (!n) throw corrupt(); + fileSize = *n; } else if (name == "NarHash") { narHash = parseHashField(value); haveNarHash = true; } else if (name == "NarSize") { - if (!string2Int(value, narSize)) throw corrupt(); + auto n = string2Int(value); + if (!n) throw corrupt(); + narSize = *n; } else if (name == "References") { auto refs = tokenizeString(value, " "); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ed10dd519..5d1723886 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -21,9 +21,8 @@ static std::optional parseName(const string & profileName, con string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link"); if (p == string::npos) return {}; - unsigned int n; - if (string2Int(string(s, 0, p), n) && n >= 0) - return n; + if (auto n = string2Int(s.substr(0, p))) + return *n; else return {}; } @@ -214,12 +213,12 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b { time_t curTime = time(0); string strDays = string(timeSpec, 0, timeSpec.size() - 1); - int days; + auto days = string2Int(strDays); - if (!string2Int(strDays, days) || days < 1) + if (!days || *days < 1) throw Error("invalid number of days specifier '%1%'", timeSpec); - time_t oldTime = curTime - days * 24 * 3600; + time_t oldTime = curTime - *days * 24 * 3600; deleteGenerationsOlderThan(profile, oldTime, dryRun); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7aca22bde..01e2fcc7b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -932,19 +932,20 @@ std::optional decodeValidPathInfo(const Store & store, std::istre getline(str, s); auto narHash = Hash::parseAny(s, htSHA256); getline(str, s); - uint64_t narSize; - if (!string2Int(s, narSize)) throw Error("number expected"); - hashGiven = { narHash, narSize }; + auto narSize = string2Int(s); + if (!narSize) throw Error("number expected"); + hashGiven = { narHash, *narSize }; } ValidPathInfo info(store.parseStorePath(path), hashGiven->first); info.narSize = hashGiven->second; std::string deriver; getline(str, deriver); if (deriver != "") info.deriver = store.parseStorePath(deriver); - string s; int n; + string s; getline(str, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { + auto n = string2Int(s); + if (!n) throw Error("number expected"); + while ((*n)--) { getline(str, s); info.references.insert(store.parseStorePath(s)); } -- cgit v1.2.3 From 44fd7a05b655315fa0e6156ac33a1c5624460968 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Tue, 12 Jan 2021 01:28:00 +0100 Subject: Don't let 'preferLocalBuild' override 'max-jobs=0' This resolves #3810 by changing the behavior of `max-jobs = 0`, so that specifying the option also avoids local building of derivations with the attribute `preferLocalBuild = true`. --- src/libstore/parsed-derivations.cc | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index e7b7202d4..c5c3ae3dc 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -101,6 +101,10 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const && !drv.isBuiltin()) return false; + if (settings.maxBuildJobs.get() == 0 + && !drv.isBuiltin()) + return false; + for (auto & feature : getRequiredSystemFeatures()) if (!localStore.systemFeatures.get().count(feature)) return false; -- cgit v1.2.3 From f69820417fa65dbfea88a5f4dd0ccb5376015a6b Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Mon, 11 Jan 2021 22:05:32 -0600 Subject: Set kern.curproc_arch_affinity=0 to escape Rosetta By default, once you enter x86_64 Rosetta 2, macOS will try to run everything in x86_64. So an x86_64 Nix will still try to use x86_64 even when system = aarch64-darwin. To avoid this we can set kern.curproc_arch_affinity sysctl. With kern.curproc_arch_affinity=0, we ignore this preference. This is based on how https://opensource.apple.com/source/system_cmds/system_cmds-880.40.5/arch.tproj/arch.c.auto.html works. Completely undocumented, but seems to work! Note, you can verify this works with this impure Nix expression: ``` { a = derivation { name = "a"; system = "aarch64-darwin"; builder = "/bin/sh"; args = [ "-e" (builtins.toFile "builder" '' [ "$(/usr/bin/arch)" = arm64 ] [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; b = derivation { name = "b"; system = "x86_64-darwin"; builder = "/bin/sh"; args = [ "-e" (builtins.toFile "builder" '' [ "$(/usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch x86_64 /bin/sh -c /usr/bin/arch)" = i386 ] [ "$(/usr/bin/arch -arch arm64 /bin/sh -c /usr/bin/arch)" = arm64 ] /usr/bin/touch $out '') ]; }; } ``` --- src/libstore/build/derivation-goal.cc | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 35f365795..a02ddb950 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -52,6 +52,7 @@ #if __APPLE__ #include +#include #endif #include @@ -2869,6 +2870,10 @@ void DerivationGoal::runChild() throw SysError("failed to initialize builder"); if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + cpu_type_t cpu = CPU_TYPE_ARM64; posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); } else if (drv->platform == "x86_64-darwin") { -- cgit v1.2.3 From 0ca1a5013269060919393afaa708640f574ab350 Mon Sep 17 00:00:00 2001 From: Rickard Nilsson Date: Wed, 13 Jan 2021 10:13:51 +0100 Subject: Remove a redundant condition in DerivationGoal::tryLocalBuild() --- src/libstore/build/derivation-goal.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 35f365795..415a55d37 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -679,13 +679,9 @@ void DerivationGoal::tryToBuild() } void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ + /* Make sure that we are allowed to start a build. */ unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + if (curBuilds >= settings.maxBuildJobs) { worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; -- cgit v1.2.3 From 61216d32e1c0973424d549c9f3065426b51015c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 23:27:39 +0100 Subject: Add 'nix store repair' command --- src/libstore/local-store.hh | 4 +--- src/libstore/store-api.hh | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6d29c5960..6c7ebac1e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -198,9 +198,7 @@ public: void vacuumDB(); - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ - void repairPath(const StorePath & path); + void repairPath(const StorePath & path) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9bcff08eb..d1b83933a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -604,6 +604,11 @@ public: virtual ref getFSAccessor() { unsupported("getFSAccessor"); } + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + virtual void repairPath(const StorePath & path) + { unsupported("repairPath"); } + /* Add signatures to the specified store path. The signatures are not verified. */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) -- cgit v1.2.3 From 7af743470c09b835f910d2e25786c080ccfe52c1 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 15 Jan 2021 16:37:41 +0000 Subject: Make public keys and `requireSigs` local-store specific again Thanks @regnat and @edolstra for catching this and comming up with the solution. They way I had generalized those is wrong, because local settings for non-local stores is confusing default. And due to the nature of C++ inheritance, fixing the defaults is more annoying than it should be. Additionally, I thought we might just drop the check in the substitution logic since `Store::addToStore` is now streaming, but @regnat rightfully pointed out that as it downloads dependencies first, that would still be too late, and also waste effort on possibly unneeded/unwanted dependencies. The simple and correct thing to do is just make a store method for the boolean logic, keeping all the setting and key stuff the way it was before. That new method is both used by `LocalStore::addToStore` and the substitution goal check. Perhaps we might eventually make it fancier, e.g. sending the ValidPathInfo to remote stores for them to validate, but this is good enough for now. --- src/libstore/build/substitution-goal.cc | 4 +--- src/libstore/local-store.cc | 14 +++++++++++++- src/libstore/local-store.hh | 14 ++++++++++++++ src/libstore/misc.cc | 9 --------- src/libstore/store-api.hh | 28 +++++++++++++++------------- 5 files changed, 43 insertions(+), 26 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d16584f65..f3c9040bc 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -142,9 +142,7 @@ void SubstitutionGoal::tryNext() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (worker.store.requireSigs - && !sub->isTrusted - && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) + if (!sub->isTrusted && worker.store.pathInfoIsTrusted(*info)) { logWarning({ .name = "Invalid path signature", diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4f48522c6..d6d74a0b0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1098,11 +1098,23 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) } } +const PublicKeys & LocalStore::getPublicKeys() +{ + auto state(_state.lock()); + if (!state->publicKeys) + state->publicKeys = std::make_unique(getDefaultPublicKeys()); + return *state->publicKeys; +} + +bool LocalStore::pathInfoIsTrusted(const ValidPathInfo & info) +{ + return requireSigs && !info.checkSignatures(*this, getPublicKeys()); +} void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) + if (checkSigs && pathInfoIsTrusted(info)) throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); addTempRoot(info.path); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 69704d266..9d235ba0a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -35,6 +35,10 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; + Setting requireSigs{(StoreConfig*) this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + const std::string name() override { return "Local Store"; } }; @@ -71,6 +75,8 @@ private: minFree but not much below availAfterGC, then there is no point in starting a new GC. */ uint64_t availAfterGC = std::numeric_limits::max(); + + std::unique_ptr publicKeys; }; Sync _state; @@ -88,6 +94,12 @@ public: const Path tempRootsDir; const Path fnTempRoots; +private: + + const PublicKeys & getPublicKeys(); + +public: + // Hack for build-remote.cc. PathSet locksHeld; @@ -124,6 +136,8 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; + bool pathInfoIsTrusted(const ValidPathInfo &) override; + void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 0d4190a56..ad4dccef9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -282,13 +282,4 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) } -const PublicKeys & Store::getPublicKeys() -{ - auto cryptoState(_cryptoState.lock()); - if (!cryptoState->publicKeys) - cryptoState->publicKeys = std::make_unique(getDefaultPublicKeys()); - return *cryptoState->publicKeys; -} - - } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e6a14afc3..3221cf249 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -189,10 +189,6 @@ struct StoreConfig : public Config const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; - Setting requireSigs{this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; - Setting priority{this, 0, "priority", "priority of this substituter (lower value means higher priority)"}; Setting wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; @@ -376,6 +372,21 @@ public: void queryPathInfo(const StorePath & path, Callback> callback) noexcept; + /* Check whether the given valid path info is sufficiently well-formed + (e.g. hash content-address or signature) in order to be included in the + given store. + + These same checks would be performed in addToStore, but this allows an + earlier failure in the case where dependencies need to be added too, but + the addToStore wouldn't fail until those dependencies are added. Also, + we don't really want to add the dependencies listed in a nar info we + don't trust anyyways. + */ + virtual bool pathInfoIsTrusted(const ValidPathInfo &) + { + return true; + } + protected: virtual void queryPathInfoUncached(const StorePath & path, @@ -719,20 +730,11 @@ public: return toRealPath(printStorePath(storePath)); } - const PublicKeys & getPublicKeys(); - virtual void createUser(const std::string & userName, uid_t userId) { } protected: - struct CryptoState - { - std::unique_ptr publicKeys; - }; - - Sync _cryptoState; - Stats stats; /* Unsupported methods. */ -- cgit v1.2.3 From 9432c170e736a6b506d9b35ced5eccff6422ec50 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 21 Dec 2020 17:12:58 +0100 Subject: Fix the drv output map for non ca derivations With the `ca-derivation` experimental features, non-ca derivations used to have their output paths returned as unknown as long as they weren't built (because of a mistake in the code that systematically erased the previous value) --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c61f34275..ab78f1435 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -914,7 +914,7 @@ LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) if (realisation) outputs.insert_or_assign(outputName, realisation->outPath); else - outputs.insert_or_assign(outputName, std::nullopt); + outputs.insert({outputName, std::nullopt}); } return outputs; -- cgit v1.2.3 From ea756b3654931f23839aee9f461a8c891c6ffe43 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 18 Jan 2021 14:38:31 +0100 Subject: --refresh: Imply setting .narinfo disk cache TTL to 0 --- src/libstore/nar-info-disk-cache.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 8541cc51f..1d8d2d57e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -109,8 +109,10 @@ public: SQLiteStmt(state->db, "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") .use() - (now - settings.ttlNegativeNarInfoCache) - (now - settings.ttlPositiveNarInfoCache) + // Use a minimum TTL to prevent --refresh from + // nuking the entire disk cache. + (now - std::max(settings.ttlNegativeNarInfoCache.get(), 3600U)) + (now - std::max(settings.ttlPositiveNarInfoCache.get(), 30 * 24 * 3600U)) .exec(); debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); -- cgit v1.2.3 From 144cad906991015e997a6b3e7cc69412eb2b8ab1 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Mon, 18 Jan 2021 18:13:07 +0100 Subject: narinfo: Change NAR URLs to be addressed on the NAR hash instead of the compressed hash This change is to simplify [Trustix](https://github.com/tweag/trustix) indexing and makes it possible to reconstruct this URL regardless of the compression used. In particular this means that https://github.com/tweag/trustix/blob/7c2e9ca597de233846e0b265fb081626ca6c59d8/contrib/nix/nar/nar.go#L61-L71 can be removed and only the bits that are required to establish trust needs to be published in the Trustix build logs. --- src/libstore/binary-cache-store.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4f5f8607d..15163ead5 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -176,11 +176,7 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "br" ? ".br" : - ""); + narInfo->url = "nar/" + info.narHash.to_string(Base32, false) + ".nar"; auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", -- cgit v1.2.3 From 8d4268d1901452164b3e666f2eb6bd6bf516493b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 21 Jan 2021 00:27:36 +0100 Subject: Improve error formatting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes: * The divider lines are gone. These were in practice a bit confusing, in particular with --show-trace or --keep-going, since then there were multiple lines, suggesting a start/end which wasn't the case. * Instead, multi-line error messages are now indented to align with the prefix (e.g. "error: "). * The 'description' field is gone since we weren't really using it. * 'hint' is renamed to 'msg' since it really wasn't a hint. * The error is now printed *before* the location info. * The 'name' field is no longer printed since most of the time it wasn't very useful since it was just the name of the exception (like EvalError). Ideally in the future this would be a unique, easily googleable error ID (like rustc). * "trace:" is now just "…". This assumes error contexts start with something like "while doing X". Example before: error: --- AssertionError ---------------------------------------------------------------------------------------- nix at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| assertion 'false' failed ----------------------------------------------------- show-trace ----------------------------------------------------- trace: while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) { Example after: error: assertion 'false' failed at: (7:7) in file: /home/eelco/Dev/nixpkgs/pkgs/applications/misc/hello/default.nix 6| 7| x = assert false; 1; | ^ 8| … while evaluating the attribute 'x' of the derivation 'hello-2.10' at: (192:11) in file: /home/eelco/Dev/nixpkgs/pkgs/stdenv/generic/make-derivation.nix 191| // (lib.optionalAttrs (!(attrs ? name) && attrs ? pname && attrs ? version)) { 192| name = "${attrs.pname}-${attrs.version}"; | ^ 193| } // (lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix && (attrs ? name || (attrs ? pname && attrs ? version)))) { --- src/libstore/build/derivation-goal.cc | 41 +++++++++++++-------------------- src/libstore/build/substitution-goal.cc | 7 ++---- src/libstore/build/worker.cc | 5 +--- src/libstore/builtins/buildenv.cc | 10 ++------ src/libstore/filetransfer.cc | 15 ++++-------- src/libstore/local-store.cc | 32 ++++++------------------- src/libstore/optimise-store.cc | 17 ++++---------- src/libstore/sqlite.cc | 2 +- 8 files changed, 38 insertions(+), 91 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2e74cfd6c..c733ccf08 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -87,8 +87,8 @@ void handleDiffHook( printError(chomp(diffRes.second)); } catch (Error & error) { ErrorInfo ei = error.info(); - ei.hint = hintfmt("diff hook execution failed: %s", - (error.info().hint.has_value() ? error.info().hint->str() : "")); + // FIXME: wrap errors. + ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str()); logError(ei); } } @@ -439,12 +439,9 @@ void DerivationGoal::repairClosure() /* Check each path (slow!). */ for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - logError({ - .name = "Corrupt path in closure", - .hint = hintfmt( - "found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) - }); + printError( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair))); @@ -877,9 +874,12 @@ void DerivationGoal::buildDone() statusToString(status)); if (!logger->isVerbose() && !logTail.empty()) { - msg += (format("; last %d log lines:") % logTail.size()).str(); - for (auto & line : logTail) - msg += "\n " + line; + msg += fmt(";\nlast %d log lines:\n", logTail.size()); + for (auto & line : logTail) { + msg += "> "; + msg += line; + msg += "\n"; + } } if (diskFull) @@ -1055,12 +1055,9 @@ HookReply DerivationGoal::tryBuildHook() } catch (SysError & e) { if (e.errNo == EPIPE) { - logError({ - .name = "Build hook died", - .hint = hintfmt( - "build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))) - }); + printError( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))); worker.hook = 0; return rpDecline; } else @@ -3068,10 +3065,7 @@ void DerivationGoal::registerOutputs() auto rewriteOutput = [&]() { /* Apply hash rewriting if necessary. */ if (!outputRewrites.empty()) { - logWarning({ - .name = "Rewriting hashes", - .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath), - }); + warn("rewriting hashes in '%1%'; cross fingers", actualPath); /* FIXME: this is in-memory. */ StringSink sink; @@ -3359,10 +3353,7 @@ void DerivationGoal::registerOutputs() if (settings.enforceDeterminism) throw NotDeterministic(hint); - logError({ - .name = "Output determinism error", - .hint = hint - }); + printError(hint); curRound = nrRounds; // we know enough, bail out early } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d16584f65..760fd8ab8 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -146,11 +146,8 @@ void SubstitutionGoal::tryNext() && !sub->isTrusted && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { - logWarning({ - .name = "Invalid path signature", - .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)) - }); + warn("substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), worker.store.printStorePath(storePath)); tryNext(); return; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 6c96a93bd..880a93b15 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -454,10 +454,7 @@ bool Worker::pathContentsGood(const StorePath & path) } pathContentsGoodCache.insert_or_assign(path, res); if (!res) - logError({ - .name = "Corrupted path", - .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) - }); + printError("path '%s' is corrupted or missing!", store.printStorePath(path)); return res; } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 802fb87bc..e88fc687a 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -22,10 +22,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, srcFiles = readDirectory(srcDir); } catch (SysError & e) { if (e.errNo == ENOTDIR) { - logWarning({ - .name = "Create links - directory", - .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) - }); + warn("not including '%s' in the user environment because it's not a directory", srcDir); return; } throw; @@ -44,10 +41,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw SysError("getting status of '%1%'", srcFile); } catch (SysError & e) { if (e.errNo == ENOENT || e.errNo == ENOTDIR) { - logWarning({ - .name = "Create links - skipping symlink", - .hint = hintfmt("skipping dangling symlink '%s'", dstFile) - }); + warn("skipping dangling symlink '%s'", dstFile); continue; } throw; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 31b4215a9..677ad44cc 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -632,11 +632,7 @@ struct curlFileTransfer : public FileTransfer workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - logError({ - .name = "File transfer", - .hint = hintfmt("unexpected error in download thread: %s", - e.what()) - }); + printError("unexpected error in download thread: %s", e.what()); } { @@ -852,11 +848,10 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr< // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how // to print different messages for different verbosity levels. For now // we add some heuristics for detecting when we want to show the response. - if (response && (response->size() < 1024 || response->find("") != string::npos)) { - err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); - } else { - err.hint = hf; - } + if (response && (response->size() < 1024 || response->find("") != string::npos)) + err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + else + err.msg = hf; } bool isUri(const string & s) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ab78f1435..f306d8505 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -150,12 +150,7 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - logError({ - .name = "'build-users-group' not found", - .hint = hintfmt( - "warning: the group '%1%' specified in 'build-users-group' does not exist", - settings.buildUsersGroup) - }); + printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); else { struct stat st; if (stat(realStoreDir.c_str(), &st)) @@ -1403,12 +1398,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) Path linkPath = linksDir + "/" + link.name; string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); if (hash != link.name) { - logError({ - .name = "Invalid hash", - .hint = hintfmt( - "link '%s' was modified! expected hash '%s', got '%s'", - linkPath, link.name, hash) - }); + printError("link '%s' was modified! expected hash '%s', got '%s'", + linkPath, link.name, hash); if (repair) { if (unlink(linkPath.c_str()) == 0) printInfo("removed link '%s'", linkPath); @@ -1441,11 +1432,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - logError({ - .name = "Invalid hash - path modified", - .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) - }); + printError("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); if (repair) repairPath(i); else errors = true; } else { @@ -1496,10 +1484,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, if (!done.insert(pathS).second) return; if (!isStorePath(pathS)) { - logError({ - .name = "Nix path not found", - .hint = hintfmt("path '%s' is not in the Nix store", pathS) - }); + printError("path '%s' is not in the Nix store", pathS); return; } @@ -1522,10 +1507,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, auto state(_state.lock()); invalidatePath(*state, path); } else { - logError({ - .name = "Missing path with referrers", - .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) - }); + printError("path '%s' disappeared, but it still has valid referrers!", pathS); if (repair) try { repairPath(path); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index a0d482ddf..78d587139 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -126,16 +126,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - logWarning({ - .name = "Suspicious file", - .hint = hintfmt("skipping suspicious writable file '%1%'", path) - }); + warn("skipping suspicious writable file '%1%'", path); return; } /* This can still happen on top-level files. */ if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { - debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); + debug("'%s' is already linked, with %d other file(s)", path, st.st_nlink - 2); return; } @@ -191,10 +188,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } if (st.st_size != stLink.st_size) { - logWarning({ - .name = "Corrupted link", - .hint = hintfmt("removing corrupted link '%1%'", linkPath) - }); + warn("removing corrupted link '%s'", linkPath); unlink(linkPath.c_str()); goto retry; } @@ -229,10 +223,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - logError({ - .name = "Unlink error", - .hint = hintfmt("unable to unlink '%1%'", tempLink) - }); + printError("unable to unlink '%1%'", tempLink); if (errno == EMLINK) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index f5935ee5c..447b4179b 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -211,7 +211,7 @@ void handleSQLiteBusy(const SQLiteBusy & e) lastWarned = now; logWarning({ .name = "Sqlite busy", - .hint = hintfmt(e.what()) + .msg = hintfmt(e.what()) }); } -- cgit v1.2.3 From 8c07ed1ddad6595cd679181b0b8d78e09fc6d152 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Jan 2021 15:27:55 +0000 Subject: Improve documentation and test and requested --- src/libstore/store-api.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3221cf249..9e98eb8f9 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -372,9 +372,9 @@ public: void queryPathInfo(const StorePath & path, Callback> callback) noexcept; - /* Check whether the given valid path info is sufficiently well-formed - (e.g. hash content-address or signature) in order to be included in the - given store. + /* Check whether the given valid path info is sufficiently attested, by + either being signed by a trusted public key or content-addressed, in + order to be included in the given store. These same checks would be performed in addToStore, but this allows an earlier failure in the case where dependencies need to be added too, but -- cgit v1.2.3 From 53a709535b42197a9abd3fe46406bb186ad6c751 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Jan 2021 10:21:12 -0500 Subject: Apply suggestions from code review Thanks! Co-authored-by: Eelco Dolstra --- src/libstore/build/derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 953e241d8..fa8b99118 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3291,7 +3291,7 @@ void DerivationGoal::registerOutputs() auto localStoreP = dynamic_cast(&worker.store); if (!localStoreP) - Unsupported("Can only register outputs with local store"); + throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); auto & localStore = *localStoreP; if (buildMode == bmCheck) { @@ -3426,7 +3426,7 @@ void DerivationGoal::registerOutputs() { auto localStoreP = dynamic_cast(&worker.store); if (!localStoreP) - Unsupported("Can only register outputs with local store"); + throw Unsupported("can only register outputs with local store, but this is %s", worker.store.getUri()); auto & localStore = *localStoreP; ValidPathInfos infos2; -- cgit v1.2.3 From a76682466062ef2c972d19f259feeef1c46a44a3 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 22 Jan 2021 14:46:40 -0600 Subject: Handle missing etag in 304 Not Modified response GitHub now omits the etag, but 304 implies it matches the one we provided. Just use that one to avoid having an etag-less resource. Fixes #4469 --- src/libstore/filetransfer.cc | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 31b4215a9..1b7eae3ec 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -375,6 +375,13 @@ struct curlFileTransfer : public FileTransfer else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) { result.cached = httpStatus == 304; + + // In 2021, GitHub responds to If-None-Match with 304, + // but omits ETag. We just use the If-None-Match etag + // since 304 implies they are the same. + if (httpStatus == 304 && result.etag == "") + result.etag = request.expectedETag; + act.progress(result.bodySize, result.bodySize); done = true; callback(std::move(result)); -- cgit v1.2.3 From 3ba98ba8f08523e60310cf75ec809bd21d0ce977 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 25 Jan 2021 17:15:38 +0100 Subject: Tell user to run 'nix log' to get full build logs --- src/libstore/build/derivation-goal.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 36bbe46d4..179a010d4 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -896,6 +896,8 @@ void DerivationGoal::buildDone() msg += line; msg += "\n"; } + msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + worker.store.printStorePath(drvPath)); } if (diskFull) -- cgit v1.2.3 From d3c428413367a87ab2d27abe9c7f3c379eb12e1c Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 5 Jan 2021 10:01:22 +0100 Subject: Make the error message for missing outputs more useful Don't only show the name of the output, but also the derivation to which this output belongs (as otherwise it's very hard to track back what went wrong) --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 01e2fcc7b..9da415c42 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -394,7 +394,7 @@ OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw Error("output '%s' has no store path mapped to it", outName); + throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path)); result.insert_or_assign(outName, *optOutPath); } return result; -- cgit v1.2.3 From 9da11bac5797c34b7bb2ee99275befe9c9fb6dd9 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 7 Jan 2021 11:21:43 +0100 Subject: Fix the error message when a dep is missing Fix a mismatch in the errors thrown when a needed output was missing from an input derivation that was leading to a wrong and quite misleading error message --- src/libstore/build/derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2e74cfd6c..656f92cee 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -539,12 +539,12 @@ void DerivationGoal::inputsRealised() if (!optRealizedInput) throw Error( "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); worker.store.computeFSClosure(*optRealizedInput, inputPaths); } else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath)); + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); } } } -- cgit v1.2.3 From 965dc6070a1b7dc582d90039c670d436f4a2e9f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 27 Jan 2021 14:04:49 +0100 Subject: Drop trailing whitespace --- src/libstore/filetransfer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 563f49170..8ea5cdc9d 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -856,7 +856,7 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr< // to print different messages for different verbosity levels. For now // we add some heuristics for detecting when we want to show the response. if (response && (response->size() < 1024 || response->find("") != string::npos)) - err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response); + err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response)); else err.msg = hf; } -- cgit v1.2.3 From 9355ecd54301372b6a919a2205340f904c7a51c6 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 14 Dec 2020 17:24:30 +0100 Subject: Add a new Cmd type working on RealisedPaths Where a `RealisedPath` is a store path with its history, meaning either an opaque path for stuff that has been directly added to the store, or a `Realisation` for stuff that has been built by a derivation This is a low-level refactoring that doesn't bring anything by itself (except a few dozen extra lines of code :/ ), but raising the abstraction level a bit is important on a number of levels: - Commands like `nix build` have to query for the realisations after the build is finished which is fragile (see 27905f12e4a7207450abe37c9ed78e31603b67e1 for example). Having them oprate directly at the realisation level would avoid that - Others like `nix copy` currently operate directly on (built) store paths, but need a bit more information as they will need to register the realisations on the remote side --- src/libstore/realisation.cc | 31 ++++++++++++++++ src/libstore/realisation.hh | 90 +++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 113 insertions(+), 8 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 47ad90eee..c9b66186f 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -46,4 +46,35 @@ Realisation Realisation::fromJSON( }; } +StorePath RealisedPath::path() const { + return visit([](auto && arg) { return arg.getPath(); }); +} + +void RealisedPath::closure( + Store& store, + const RealisedPath::Set& startPaths, + RealisedPath::Set& ret) +{ + // FIXME: This only builds the store-path closure, not the real realisation + // closure + StorePathSet initialStorePaths, pathsClosure; + for (auto& path : startPaths) + initialStorePaths.insert(path.path()); + store.computeFSClosure(initialStorePaths, pathsClosure); + ret.insert(startPaths.begin(), startPaths.end()); + ret.insert(pathsClosure.begin(), pathsClosure.end()); +} + +void RealisedPath::closure(Store& store, RealisedPath::Set& ret) const +{ + RealisedPath::closure(store, {*this}, ret); +} + +RealisedPath::Set RealisedPath::closure(Store& store) const +{ + RealisedPath::Set ret; + closure(store, ret); + return ret; +} + } // namespace nix diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 4b8ead3c5..1ecddc4d1 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -3,6 +3,34 @@ #include "path.hh" #include + +/* Awfull hacky generation of the comparison operators by doing a lexicographic + * comparison between the choosen fields + * ``` + * GENERATE_CMP(ClassName, my->field1, my->field2, ...) + * ``` + * + * will generate comparison operators semantically equivalent to: + * ``` + * bool operator<(const ClassName& other) { + * return field1 < other.field1 && field2 < other.field2 && ...; + * } + * ``` + */ +#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ + bool operator COMPARATOR(const MY_TYPE& other) const { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple( FIELDS ); \ + me = &other; \ + auto fields2 = std::make_tuple( FIELDS ); \ + return fields1 COMPARATOR fields2; \ + } +#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) +#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) +#define GENERATE_CMP(args...) \ + GENERATE_EQUAL(args) \ + GENERATE_LEQ(args) + namespace nix { struct DrvOutput { @@ -17,13 +45,7 @@ struct DrvOutput { static DrvOutput parse(const std::string &); - bool operator<(const DrvOutput& other) const { return to_pair() < other.to_pair(); } - bool operator==(const DrvOutput& other) const { return to_pair() == other.to_pair(); } - -private: - // Just to make comparison operators easier to write - std::pair to_pair() const - { return std::make_pair(drvHash, outputName); } + GENERATE_CMP(DrvOutput, me->drvHash, me->outputName); }; struct Realisation { @@ -32,8 +54,60 @@ struct Realisation { nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); + + StorePath getPath() const { return outPath; } + + GENERATE_CMP(Realisation, me->id, me->outPath); +}; + +struct OpaquePath { + StorePath path; + + StorePath getPath() const { return path; } + + GENERATE_CMP(OpaquePath, me->path); }; -typedef std::map DrvOutputs; + +/** + * A store path with all the history of how it went into the store + */ +struct RealisedPath { + /* + * A path is either the result of the realisation of a derivation or + * an opaque blob that has been directly added to the store + */ + using Raw = std::variant; + Raw raw; + + using Set = std::set; + + RealisedPath(StorePath path) : raw(OpaquePath{path}) {} + RealisedPath(Realisation r) : raw(r) {} + + /** + * Syntactic sugar to run `std::visit` on the raw value: + * path.visit(blah) == std::visit(blah, path.raw) + */ + template + constexpr decltype(auto) visit(Visitor && vis) { + return std::visit(vis, raw); + } + template + constexpr decltype(auto) visit(Visitor && vis) const { + return std::visit(vis, raw); + } + + /** + * Get the raw store path associated to this + */ + StorePath path() const; + + void closure(Store& store, Set& ret) const; + static void closure(Store& store, const Set& startPaths, Set& ret); + Set closure(Store& store) const; + + GENERATE_CMP(RealisedPath, me->raw); +}; } -- cgit v1.2.3 From 43d409f6690b79b5d4e1ab5e9780de93eb0f677a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 14:47:56 +0100 Subject: Fix a whitespace issue Co-authored-by: Eelco Dolstra --- src/libstore/realisation.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index c9b66186f..e4276c040 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -65,7 +65,7 @@ void RealisedPath::closure( ret.insert(pathsClosure.begin(), pathsClosure.end()); } -void RealisedPath::closure(Store& store, RealisedPath::Set& ret) const +void RealisedPath::closure(Store& store, RealisedPath::Set & ret) const { RealisedPath::closure(store, {*this}, ret); } -- cgit v1.2.3 From d2091af231ab97b729c2486b55e520c565e59dd3 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 4 Feb 2021 15:11:05 +0100 Subject: Move the GENERATE_CMP macro to its own file Despite being an ugly hack, it can probably be useful in a couple extra places --- src/libstore/realisation.hh | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 1ecddc4d1..557f54362 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -2,34 +2,7 @@ #include "path.hh" #include - - -/* Awfull hacky generation of the comparison operators by doing a lexicographic - * comparison between the choosen fields - * ``` - * GENERATE_CMP(ClassName, my->field1, my->field2, ...) - * ``` - * - * will generate comparison operators semantically equivalent to: - * ``` - * bool operator<(const ClassName& other) { - * return field1 < other.field1 && field2 < other.field2 && ...; - * } - * ``` - */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \ - bool operator COMPARATOR(const MY_TYPE& other) const { \ - const MY_TYPE* me = this; \ - auto fields1 = std::make_tuple( FIELDS ); \ - me = &other; \ - auto fields2 = std::make_tuple( FIELDS ); \ - return fields1 COMPARATOR fields2; \ - } -#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) -#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) -#define GENERATE_CMP(args...) \ - GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) +#include "comparator.hh" namespace nix { -- cgit v1.2.3 From e69cfdebb090b3aabbff69a44504883d5b6fb866 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 4 Feb 2021 15:15:22 +0100 Subject: Remove the `visit` machinery in `RealisedPath` In addition to being some ugly template trickery, it was also totally useless as it was used in only one place where I could replace it by just a few extra characters --- src/libstore/realisation.cc | 2 +- src/libstore/realisation.hh | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index e4276c040..cd74af4ee 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -47,7 +47,7 @@ Realisation Realisation::fromJSON( } StorePath RealisedPath::path() const { - return visit([](auto && arg) { return arg.getPath(); }); + return std::visit([](auto && arg) { return arg.getPath(); }, raw); } void RealisedPath::closure( diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 557f54362..7c91d802a 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -58,19 +58,6 @@ struct RealisedPath { RealisedPath(StorePath path) : raw(OpaquePath{path}) {} RealisedPath(Realisation r) : raw(r) {} - /** - * Syntactic sugar to run `std::visit` on the raw value: - * path.visit(blah) == std::visit(blah, path.raw) - */ - template - constexpr decltype(auto) visit(Visitor && vis) { - return std::visit(vis, raw); - } - template - constexpr decltype(auto) visit(Visitor && vis) const { - return std::visit(vis, raw); - } - /** * Get the raw store path associated to this */ -- cgit v1.2.3 From 0187838e2e7ff01f1b480e3e85d9e96da0b4b78e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Feb 2021 12:11:50 +0100 Subject: Add a trace to readLine() failures Hopefully this helps to diagnose 'error: unexpected EOF reading a line' on macOS. --- src/libstore/build/derivation-goal.cc | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 8717499c0..190adf31c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1044,7 +1044,14 @@ HookReply DerivationGoal::tryBuildHook() whether the hook wishes to perform the build. */ string reply; while (true) { - string s = readLine(worker.hook->fromHook.readSide.get()); + auto s = [&]() { + try { + return readLine(worker.hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the response from the build hook"); + throw e; + } + }(); if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) ; else if (string(s, 0, 2) == "# ") { @@ -1084,7 +1091,12 @@ HookReply DerivationGoal::tryBuildHook() hook = std::move(worker.hook); - machineName = readLine(hook->fromHook.readSide.get()); + try { + machineName = readLine(hook->fromHook.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the machine name from the build hook"); + throw e; + } /* Tell the hook all the inputs that have to be copied to the remote system. */ @@ -1773,7 +1785,14 @@ void DerivationGoal::startBuilder() /* Check if setting up the build environment failed. */ while (true) { - string msg = readLine(builderOut.readSide.get()); + string msg = [&]() { + try { + return readLine(builderOut.readSide.get()); + } catch (Error & e) { + e.addTrace({}, "while reading the response of setting up the build environment"); + throw e; + } + }(); if (string(msg, 0, 1) == "\2") break; if (string(msg, 0, 1) == "\1") { FdSource source(builderOut.readSide.get()); -- cgit v1.2.3 From 480426a364f09e7992230b32f2941a09fb52d729 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Feb 2021 15:57:33 +0100 Subject: Add more instrumentation for #4270 --- src/libstore/build/derivation-goal.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 190adf31c..eeaec4f2c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1784,12 +1784,14 @@ void DerivationGoal::startBuilder() worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); /* Check if setting up the build environment failed. */ + std::vector msgs; while (true) { string msg = [&]() { try { return readLine(builderOut.readSide.get()); } catch (Error & e) { - e.addTrace({}, "while reading the response of setting up the build environment"); + e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)", + concatStringsSep("|", msgs)); throw e; } }(); @@ -1801,6 +1803,7 @@ void DerivationGoal::startBuilder() throw ex; } debug("sandbox setup: " + msg); + msgs.push_back(std::move(msg)); } } -- cgit v1.2.3 From f2245091d033a8037aeb29ae701d20611500af6d Mon Sep 17 00:00:00 2001 From: Graham Christensen Date: Tue, 9 Feb 2021 12:26:41 -0500 Subject: Revert "narinfo: Change NAR URLs to be addressed on the NAR hash instead of the compressed hash" --- src/libstore/binary-cache-store.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 15163ead5..4f5f8607d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -176,7 +176,11 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + info.narHash.to_string(Base32, false) + ".nar"; + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : + ""); auto duration = std::chrono::duration_cast(now2 - now1).count(); printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", -- cgit v1.2.3 From 87c8d3d702123528ac068bb703232e24431c535e Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 27 Jan 2021 10:03:05 +0100 Subject: Register the realisations for unresolved drvs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once a build is done, get back to the original derivation, and register all the newly built outputs for this derivation. This allows Nix to work properly with derivations that don't have all their build inputs available − thus allowing garbage collection and (once it's implemented) binary substitution --- src/libstore/build/derivation-goal.cc | 54 ++++++++++++++++++++++++++++++++++- src/libstore/build/derivation-goal.hh | 3 ++ src/libstore/derivations.cc | 9 +++++- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 2 +- src/libstore/store-api.cc | 15 +--------- src/libstore/store-api.hh | 6 ---- 7 files changed, 67 insertions(+), 24 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index eeaec4f2c..315cf3f0a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -506,6 +506,7 @@ void DerivationGoal::inputsRealised() Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); + resolvedDrv = drvResolved; auto msg = fmt("Resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), @@ -1019,7 +1020,45 @@ void DerivationGoal::buildDone() } void DerivationGoal::resolvedFinished() { - done(BuildResult::Built); + assert(resolvedDrv); + + // If the derivation was originally a full `Derivation` (and not just + // a `BasicDerivation`, we must retrieve it because the `staticOutputHashes` + // will be wrong otherwise + Derivation fullDrv = *drv; + if (auto upcasted = dynamic_cast(drv.get())) + fullDrv = *upcasted; + + auto originalHashes = staticOutputHashes(worker.store, fullDrv); + auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); + + // `wantedOutputs` might be empty, which means “all the outputs” + auto realWantedOutputs = wantedOutputs; + if (realWantedOutputs.empty()) + realWantedOutputs = resolvedDrv->outputNames(); + + for (auto & wantedOutput : realWantedOutputs) { + assert(originalHashes.count(wantedOutput) != 0); + assert(resolvedHashes.count(wantedOutput) != 0); + auto realisation = worker.store.queryRealisation( + DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} + ); + // We've just built it, but maybe the build failed, in which case the + // realisation won't be there + if (realisation) { + auto newRealisation = *realisation; + newRealisation.id = DrvOutput{originalHashes.at(wantedOutput), wantedOutput}; + worker.store.registerDrvOutput(newRealisation); + } else { + // If we don't have a realisation, then it must mean that something + // failed when building the resolved drv + assert(!result.success()); + } + } + + // This is potentially a bit fishy in terms of error reporting. Not sure + // how to do it in a cleaner way + amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); } HookReply DerivationGoal::tryBuildHook() @@ -3804,6 +3843,19 @@ void DerivationGoal::checkPathValidity() : PathStatus::Corrupt, }; } + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + Derivation fullDrv = *drv; + if (auto upcasted = dynamic_cast(drv.get())) + fullDrv = *upcasted; + auto outputHashes = staticOutputHashes(worker.store, fullDrv); + if (auto real = worker.store.queryRealisation( + DrvOutput{outputHashes.at(i.first), i.first})) { + info.known = { + .path = real->outPath, + .status = PathStatus::Valid, + }; + } + } initialOutputs.insert_or_assign(i.first, info); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 8ee0be9e1..b7b85c85d 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -48,6 +48,9 @@ struct DerivationGoal : public Goal /* The path of the derivation. */ StorePath drvPath; + /* The path of the corresponding resolved derivation */ + std::optional resolvedDrv; + /* The specific outputs that we need to build. Empty means all of them. */ StringSet wantedOutputs; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7466c7d41..4b774c42a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -756,8 +756,13 @@ std::optional Derivation::tryResolveUncached(Store & store) { StringSet newOutputNames; for (auto & outputName : input.second) { auto actualPathOpt = inputDrvOutputs.at(outputName); - if (!actualPathOpt) + if (!actualPathOpt) { + warn("Input %s!%s missing, aborting the resolving", + store.printStorePath(input.first), + outputName + ); return std::nullopt; + } auto actualPath = *actualPathOpt; inputRewrites.emplace( downstreamPlaceholder(store, input.first, outputName), @@ -782,6 +787,8 @@ std::optional Derivation::tryResolve(Store& store, const StoreP // This is quite dirty and leaky, but will disappear once #4340 is merged static Sync>> resolutionsCache; + debug("Trying to resolve %s", store.printStorePath(drvPath)); + { auto resolutions = resolutionsCache.lock(); auto resolvedDrvIter = resolutions->find(drvPath); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f45af2bac..e06c47cde 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -883,7 +883,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) +LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) { auto path = path_; auto outputs = retrySQLite>>([&]() { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9d235ba0a..780cc0f07 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -127,7 +127,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryDerivationOutputMapNoResolve(const StorePath & path) override; + std::map> queryPartialDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 37c11fe86..2658f7617 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -366,7 +366,7 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -std::map> Store::queryDerivationOutputMapNoResolve(const StorePath & path) +std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) { std::map> outputs; auto drv = readInvalidDerivation(path); @@ -376,19 +376,6 @@ std::map> Store::queryDerivationOutputMapN return outputs; } -std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) -{ - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - auto resolvedDrv = Derivation::tryResolve(*this, path); - if (resolvedDrv) { - auto resolvedDrvPath = writeDerivation(*this, *resolvedDrv, NoRepair, true); - if (isValidPath(resolvedDrvPath)) - return queryDerivationOutputMapNoResolve(resolvedDrvPath); - } - } - return queryDerivationOutputMapNoResolve(path); -} - OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9e98eb8f9..6dcd43ed1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -415,12 +415,6 @@ public: `std::nullopt`. */ virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); - /* - * Similar to `queryPartialDerivationOutputMap`, but doesn't try to resolve - * the derivation - */ - virtual std::map> queryDerivationOutputMapNoResolve(const StorePath & path); - /* Query the mapping outputName=>outputPath for the given derivation. Assume every output has a mapping and throw an exception otherwise. */ OutputPathMap queryDerivationOutputMap(const StorePath & path); -- cgit v1.2.3 From 93d9eb78a0733c5adcbc6ee7b8a257605ae4a32f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 11:12:24 +0100 Subject: Syntactic fixes Co-authored-by: Eelco Dolstra --- src/libstore/derivations.cc | 2 +- src/libstore/local-store.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 4b774c42a..36993ffc2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -787,7 +787,7 @@ std::optional Derivation::tryResolve(Store& store, const StoreP // This is quite dirty and leaky, but will disappear once #4340 is merged static Sync>> resolutionsCache; - debug("Trying to resolve %s", store.printStorePath(drvPath)); + debug("trying to resolve %s", store.printStorePath(drvPath)); { auto resolutions = resolutionsCache.lock(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e06c47cde..0962418dd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -883,7 +883,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath& path_) +LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) { auto path = path_; auto outputs = retrySQLite>>([&]() { -- cgit v1.2.3 From 0bfbd043699908bcaff1493c733ab4798b642b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 4 Feb 2021 11:13:38 +0100 Subject: Don't expose the "bang" drvoutput syntax It's not fixed nor useful atm, so better keep it hidden Co-authored-by: Eelco Dolstra --- src/libstore/derivations.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 36993ffc2..7807089ca 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -757,9 +757,9 @@ std::optional Derivation::tryResolveUncached(Store & store) { for (auto & outputName : input.second) { auto actualPathOpt = inputDrvOutputs.at(outputName); if (!actualPathOpt) { - warn("Input %s!%s missing, aborting the resolving", - store.printStorePath(input.first), - outputName + warn("output %s of input %s missing, aborting the resolving", + outputName, + store.printStorePath(input.first) ); return std::nullopt; } -- cgit v1.2.3 From 4bc28c44f258f4f8c8a3935d1acf746f6abe3d8f Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 4 Feb 2021 14:41:49 +0100 Subject: Store the output hashes in the initialOutputs of the drv goal That way we 1. Don't have to recompute them several times 2. Can compute them in a place where we know the type of the parent derivation, meaning that we don't need the casting dance we had before --- src/libstore/build/derivation-goal.cc | 49 +++++++++++++++++++++-------------- src/libstore/build/derivation-goal.hh | 1 + 2 files changed, 30 insertions(+), 20 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 315cf3f0a..d8a89a2d0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -124,6 +124,17 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation , buildMode(buildMode) { this->drv = std::make_unique(BasicDerivation(drv)); + + auto outputHashes = staticOutputHashes(worker.store, drv); + for (auto &[outputName, outputHash] : outputHashes) + initialOutputs.insert({ + outputName, + InitialOutput{ + .wanted = true, // Will be refined later + .outputHash = outputHash + } + }); + state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", @@ -258,8 +269,20 @@ void DerivationGoal::loadDerivation() assert(worker.store.isValidPath(drvPath)); + auto fullDrv = new Derivation(worker.store.derivationFromPath(drvPath)); + + auto outputHashes = staticOutputHashes(worker.store, *fullDrv); + for (auto &[outputName, outputHash] : outputHashes) + initialOutputs.insert({ + outputName, + InitialOutput{ + .wanted = true, // Will be refined later + .outputHash = outputHash + } + }); + /* Get the derivation. */ - drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); + drv = std::unique_ptr(fullDrv); haveDerivation(); } @@ -1022,14 +1045,6 @@ void DerivationGoal::buildDone() void DerivationGoal::resolvedFinished() { assert(resolvedDrv); - // If the derivation was originally a full `Derivation` (and not just - // a `BasicDerivation`, we must retrieve it because the `staticOutputHashes` - // will be wrong otherwise - Derivation fullDrv = *drv; - if (auto upcasted = dynamic_cast(drv.get())) - fullDrv = *upcasted; - - auto originalHashes = staticOutputHashes(worker.store, fullDrv); auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); // `wantedOutputs` might be empty, which means “all the outputs” @@ -1038,7 +1053,7 @@ void DerivationGoal::resolvedFinished() { realWantedOutputs = resolvedDrv->outputNames(); for (auto & wantedOutput : realWantedOutputs) { - assert(originalHashes.count(wantedOutput) != 0); + assert(initialOutputs.count(wantedOutput) != 0); assert(resolvedHashes.count(wantedOutput) != 0); auto realisation = worker.store.queryRealisation( DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} @@ -1047,7 +1062,7 @@ void DerivationGoal::resolvedFinished() { // realisation won't be there if (realisation) { auto newRealisation = *realisation; - newRealisation.id = DrvOutput{originalHashes.at(wantedOutput), wantedOutput}; + newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; worker.store.registerDrvOutput(newRealisation); } else { // If we don't have a realisation, then it must mean that something @@ -3829,9 +3844,8 @@ void DerivationGoal::checkPathValidity() { bool checkHash = buildMode == bmRepair; for (auto & i : queryPartialDerivationOutputMap()) { - InitialOutput info { - .wanted = wantOutput(i.first, wantedOutputs), - }; + InitialOutput & info = initialOutputs.at(i.first); + info.wanted = wantOutput(i.first, wantedOutputs); if (i.second) { auto outputPath = *i.second; info.known = { @@ -3844,19 +3858,14 @@ void DerivationGoal::checkPathValidity() }; } if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - Derivation fullDrv = *drv; - if (auto upcasted = dynamic_cast(drv.get())) - fullDrv = *upcasted; - auto outputHashes = staticOutputHashes(worker.store, fullDrv); if (auto real = worker.store.queryRealisation( - DrvOutput{outputHashes.at(i.first), i.first})) { + DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) { info.known = { .path = real->outPath, .status = PathStatus::Valid, }; } } - initialOutputs.insert_or_assign(i.first, info); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index b7b85c85d..761100d3a 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -37,6 +37,7 @@ struct InitialOutputStatus { struct InitialOutput { bool wanted; + Hash outputHash; std::optional known; }; -- cgit v1.2.3 From f483b623e98a0feb2568e5be076b533c5838ba32 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 16 Feb 2021 08:16:12 +0100 Subject: Remove the drv resolution caching mechanism It isn't needed anymore now that don't need to eagerly resolve everything like we used to do. So we can safely get rid of it --- src/libstore/derivations.cc | 34 +--------------------------------- src/libstore/derivations.hh | 4 ---- 2 files changed, 1 insertion(+), 37 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7807089ca..6d0742b4f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -745,7 +745,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolveUncached(Store & store) { +std::optional Derivation::tryResolve(Store & store) { BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation @@ -776,36 +776,4 @@ std::optional Derivation::tryResolveUncached(Store & store) { return resolved; } -std::optional Derivation::tryResolve(Store& store) -{ - auto drvPath = writeDerivation(store, *this, NoRepair, false); - return Derivation::tryResolve(store, drvPath); -} - -std::optional Derivation::tryResolve(Store& store, const StorePath& drvPath) -{ - // This is quite dirty and leaky, but will disappear once #4340 is merged - static Sync>> resolutionsCache; - - debug("trying to resolve %s", store.printStorePath(drvPath)); - - { - auto resolutions = resolutionsCache.lock(); - auto resolvedDrvIter = resolutions->find(drvPath); - if (resolvedDrvIter != resolutions->end()) { - auto & [_, resolvedDrv] = *resolvedDrvIter; - return *resolvedDrv; - } - } - - /* Try resolve drv and use that path instead. */ - auto drv = store.readDerivation(drvPath); - auto attempt = drv.tryResolveUncached(store); - if (!attempt) - return std::nullopt; - /* Store in memo table. */ - resolutionsCache.lock()->insert_or_assign(drvPath, *attempt); - return *attempt; -} - } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 3d8f19aef..4e5985fab 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -138,14 +138,10 @@ struct Derivation : BasicDerivation 2. Input placeholders are replaced with realized input store paths. */ std::optional tryResolve(Store & store); - static std::optional tryResolve(Store & store, const StorePath & drvPath); Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } - -private: - std::optional tryResolveUncached(Store & store); }; -- cgit v1.2.3 From 2de232d2b301b2f0854b9fa715ab085612c85e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Tue, 16 Feb 2021 14:32:12 +0100 Subject: Add x86_64 compute levels as additional system types When performing distributed builds of machine learning packages, it would be nice if builders without the required SIMD instructions can be excluded as build nodes. Since x86_64 has accumulated a large number of different instruction set extensions, listing all possible extensions would be unwieldy. AMD, Intel, Red Hat, and SUSE have recently defined four different microarchitecture levels that are now part of the x86-64 psABI supplement and will be used in glibc 2.33: https://gitlab.com/x86-psABIs/x86-64-ABI https://lwn.net/Articles/844831/ This change uses libcpuid to detect CPU features and then uses them to add the supported x86_64 levels to the additional system types. For example on a Ryzen 3700X: $ ~/aps/bin/nix -vv --version | grep "Additional system" Additional system types: i686-linux, x86_64-v1-linux, x86_64-v2-linux, x86_64-v3-linux --- src/libstore/globals.cc | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 0531aad9f..df07aee9b 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -3,6 +3,7 @@ #include "archive.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "compute-levels.hh" #include #include @@ -133,24 +134,29 @@ StringSet Settings::getDefaultSystemFeatures() StringSet Settings::getDefaultExtraPlatforms() { + StringSet extraPlatforms; + if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1()) - return StringSet{"i686-linux"}; -#if __APPLE__ + extraPlatforms.insert("i686-linux"); + +#if __linux__ + StringSet levels = computeLevels(); + for (auto iter = levels.begin(); iter != levels.end(); ++iter) + extraPlatforms.insert(*iter + "-linux"); +#elif __APPLE__ // Rosetta 2 emulation layer can run x86_64 binaries on aarch64 // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { + if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") - return StringSet{"aarch64-darwin"}; + extraPlatforms.insert("aarch64-darwin"); else if (std::string{SYSTEM} == "aarch64-darwin") - return StringSet{"x86_64-darwin"}; - else - return StringSet{}; + extraPlatforms.insert("x86_64-darwin"); } #endif - else - return StringSet{}; + + return extraPlatforms; } bool Settings::isExperimentalFeatureEnabled(const std::string & name) -- cgit v1.2.3 From 35205e2e922952fc0654260a07fc3191c5afc2cc Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Mon, 22 Feb 2021 17:10:55 -0500 Subject: Warn about instability of plugin API --- src/libstore/globals.hh | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/libstore') diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1d968ef3e..1254698ca 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -831,6 +831,9 @@ public: command, and RegisterSetting to add new nix config settings. See the constructors for those types for more details. + Warning! These APIs are inherently unstable and may change from + release to release. + Since these files are loaded into the same address space as Nix itself, they must be DSOs compatible with the instance of Nix running at the time (i.e. compiled against the same headers, not -- cgit v1.2.3 From ec3497c1d63f4c0547d0402d92015f846f56aac7 Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 28 Jan 2021 07:37:04 -0500 Subject: Bail if plugin-files is set after plugins have been loaded. We know the flag will be ignored but the user wants it to take effect. --- src/libstore/globals.cc | 11 +++++++++++ src/libstore/globals.hh | 19 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index df07aee9b..03294b7fe 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -243,6 +243,14 @@ void MaxBuildJobsSetting::set(const std::string & str, bool append) } +void PluginFilesSetting::set(const std::string & str, bool append) +{ + if (pluginsLoaded) + throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); + BaseSetting::set(str, append); +} + + void initPlugins() { for (const auto & pluginFile : settings.pluginFiles.get()) { @@ -270,6 +278,9 @@ void initPlugins() unknown settings. */ globalConfig.reapplyUnknownSettings(); globalConfig.warnUnknownSettings(); + + /* Tell the user if they try to set plugin-files after we've already loaded */ + settings.pluginFiles.pluginsLoaded = true; } } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1254698ca..df61d6417 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -28,6 +28,23 @@ struct MaxBuildJobsSetting : public BaseSetting void set(const std::string & str, bool append = false) override; }; +struct PluginFilesSetting : public BaseSetting +{ + bool pluginsLoaded = false; + + PluginFilesSetting(Config * options, + const Paths & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, name, description, aliases) + { + options->addSetting(this); + } + + void set(const std::string & str, bool append = false) override; +}; + class Settings : public Config { unsigned int getDefaultCores(); @@ -819,7 +836,7 @@ public: Setting minFreeCheckInterval{this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - Setting pluginFiles{ + PluginFilesSetting pluginFiles{ this, {}, "plugin-files", R"( A list of plugin files to be loaded by Nix. Each of these files will -- cgit v1.2.3 From 98d1b64400cc7b75216fc885859883c707c18bef Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 28 Jan 2021 09:37:43 -0500 Subject: Initialize plugins after handling initial command line flags This is technically a breaking change, since attempting to set plugin files after the first non-flag argument will now throw an error. This is acceptable given the relative lack of stability in a plugin interface and the need to tie the knot somewhere once plugins can actually define new subcommands. --- src/libstore/globals.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'src/libstore') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 03294b7fe..2780e0bf5 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -253,6 +253,7 @@ void PluginFilesSetting::set(const std::string & str, bool append) void initPlugins() { + assert(!settings.pluginFiles.pluginsLoaded); for (const auto & pluginFile : settings.pluginFiles.get()) { Paths pluginFiles; try { -- cgit v1.2.3