diff options
Diffstat (limited to 'src')
43 files changed, 503 insertions, 261 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 0904f2ce9..0559aeaf4 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -270,7 +270,7 @@ connected: { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); - copyPaths(store, ref<Store>(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); + copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); } uploadLock = -1; @@ -321,7 +321,7 @@ connected: if (auto localStore = store.dynamic_pointer_cast<LocalStore>()) for (auto & path : missingPaths) localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ - copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); + copyPaths(*sshStore, *store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute); } // XXX: Should be done as part of `copyPaths` for (auto & realisation : missingRealisations) { diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 6777b23be..2daf43aa7 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -54,6 +54,30 @@ void StoreCommand::run() run(getStore()); } +EvalCommand::EvalCommand() +{ +} + +EvalCommand::~EvalCommand() +{ + if (evalState) + evalState->printStats(); +} + +ref<Store> EvalCommand::getEvalStore() +{ + if (!evalStore) + evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore(); + return ref<Store>(evalStore); +} + +ref<EvalState> EvalCommand::getEvalState() +{ + if (!evalState) + evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore()); + return ref<EvalState>(evalState); +} + BuiltPathsCommand::BuiltPathsCommand(bool recursive) : recursive(recursive) { @@ -91,7 +115,7 @@ void BuiltPathsCommand::run(ref<Store> store) for (auto & p : store->queryAllValidPaths()) paths.push_back(BuiltPath::Opaque{p}); } else { - paths = toBuiltPaths(store, realiseMode, operateOn, installables); + paths = toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); if (recursive) { // XXX: This only computes the store path closure, ignoring // intermediate realisations diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 35b3a384b..f3625ed0d 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -45,11 +45,18 @@ private: struct EvalCommand : virtual StoreCommand, MixEvalArgs { + EvalCommand(); + + ~EvalCommand(); + + ref<Store> getEvalStore(); + ref<EvalState> getEvalState(); - std::shared_ptr<EvalState> evalState; +private: + std::shared_ptr<Store> evalStore; - ~EvalCommand(); + std::shared_ptr<EvalState> evalState; }; struct MixFlakeOptions : virtual Args, EvalCommand @@ -216,15 +223,21 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name) return RegisterCommand(std::move(name), [](){ return make_ref<T>(); }); } -BuiltPaths build(ref<Store> store, Realise mode, +BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode, std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal); -std::set<StorePath> toStorePaths(ref<Store> store, - Realise mode, OperateOn operateOn, +std::set<StorePath> toStorePaths( + ref<Store> evalStore, + ref<Store> store, + Realise mode, + OperateOn operateOn, std::vector<std::shared_ptr<Installable>> installables); -StorePath toStorePath(ref<Store> store, - Realise mode, OperateOn operateOn, +StorePath toStorePath( + ref<Store> evalStore, + ref<Store> store, + Realise mode, + OperateOn operateOn, std::shared_ptr<Installable> installable); std::set<StorePath> toDerivations(ref<Store> store, @@ -232,6 +245,7 @@ std::set<StorePath> toDerivations(ref<Store> store, bool useDeriver = false); BuiltPaths toBuiltPaths( + ref<Store> evalStore, ref<Store> store, Realise mode, OperateOn operateOn, diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 95327f958..e3ce564b0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -289,19 +289,6 @@ void completeFlakeRefWithFragment( completeFlakeRef(evalState->store, prefix); } -ref<EvalState> EvalCommand::getEvalState() -{ - if (!evalState) - evalState = std::make_shared<EvalState>(searchPath, getStore()); - return ref<EvalState>(evalState); -} - -EvalCommand::~EvalCommand() -{ - if (evalState) - evalState->printStats(); -} - void completeFlakeRef(ref<Store> store, std::string_view prefix) { if (prefix == "") @@ -391,6 +378,7 @@ DerivedPaths InstallableValue::toDerivedPaths() DerivedPaths res; std::map<StorePath, std::set<std::string>> drvsToOutputs; + RealisedPath::Set drvsToCopy; // Group by derivation, helps with .all in particular for (auto & drv : toDerivations()) { @@ -398,6 +386,7 @@ DerivedPaths InstallableValue::toDerivedPaths() if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath)); drvsToOutputs[drv.drvPath].insert(outputName); + drvsToCopy.insert(drv.drvPath); } for (auto & i : drvsToOutputs) @@ -712,10 +701,10 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable( return installables.front(); } -BuiltPaths getBuiltPaths(ref<Store> store, DerivedPaths hopefullyBuiltPaths) +BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths) { BuiltPaths res; - for (auto& b : hopefullyBuiltPaths) + for (auto & b : hopefullyBuiltPaths) std::visit( overloaded{ [&](DerivedPath::Opaque bo) { @@ -723,14 +712,13 @@ BuiltPaths getBuiltPaths(ref<Store> store, DerivedPaths hopefullyBuiltPaths) }, [&](DerivedPath::Built bfd) { OutputPathMap outputs; - auto drv = store->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*store, drv); + auto drv = evalStore->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto drvOutputs = drv.outputsAndOptPaths(*store); - for (auto& output : bfd.outputs) { + for (auto & output : bfd.outputs) { if (!outputHashes.count(output)) throw Error( - "the derivation '%s' doesn't have an output " - "named '%s'", + "the derivation '%s' doesn't have an output named '%s'", store->printStorePath(bfd.drvPath), output); if (settings.isExperimentalFeatureEnabled( "ca-derivations")) { @@ -762,7 +750,7 @@ BuiltPaths getBuiltPaths(ref<Store> store, DerivedPaths hopefullyBuiltPaths) return res; } -BuiltPaths build(ref<Store> store, Realise mode, +BuiltPaths build(ref<Store> evalStore, ref<Store> store, Realise mode, std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode) { if (mode == Realise::Nothing) @@ -778,20 +766,21 @@ BuiltPaths build(ref<Store> store, Realise mode, if (mode == Realise::Nothing) printMissing(store, pathsToBuild, lvlError); else if (mode == Realise::Outputs) - store->buildPaths(pathsToBuild, bMode); + store->buildPaths(pathsToBuild, bMode, evalStore); - return getBuiltPaths(store, pathsToBuild); + return getBuiltPaths(evalStore, store, pathsToBuild); } BuiltPaths toBuiltPaths( + ref<Store> evalStore, ref<Store> store, Realise mode, OperateOn operateOn, std::vector<std::shared_ptr<Installable>> installables) { - if (operateOn == OperateOn::Output) { - return build(store, mode, installables); - } else { + if (operateOn == OperateOn::Output) + return build(evalStore, store, mode, installables); + else { if (mode == Realise::Nothing) settings.readOnlyMode = true; @@ -802,23 +791,27 @@ BuiltPaths toBuiltPaths( } } -StorePathSet toStorePaths(ref<Store> store, +StorePathSet toStorePaths( + ref<Store> evalStore, + ref<Store> store, Realise mode, OperateOn operateOn, std::vector<std::shared_ptr<Installable>> installables) { StorePathSet outPaths; - for (auto & path : toBuiltPaths(store, mode, operateOn, installables)) { + for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) { auto thisOutPaths = path.outPaths(); outPaths.insert(thisOutPaths.begin(), thisOutPaths.end()); } return outPaths; } -StorePath toStorePath(ref<Store> store, +StorePath toStorePath( + ref<Store> evalStore, + ref<Store> store, Realise mode, OperateOn operateOn, std::shared_ptr<Installable> installable) { - auto paths = toStorePaths(store, mode, operateOn, {installable}); + auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable}); if (paths.size() != 1) throw Error("argument '%s' should evaluate to one store path", installable->what()); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 298fd48f8..79931ad3e 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -26,7 +26,7 @@ struct App struct UnresolvedApp { App unresolved; - App resolve(ref<Store>); + App resolve(ref<Store> evalStore, ref<Store> store); }; struct Installable diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index aa14bf79b..fb0932c00 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -61,6 +61,14 @@ MixEvalArgs::MixEvalArgs() fetchers::overrideRegistry(from.input, to.input, extraAttrs); }} }); + + addFlag({ + .longName = "eval-store", + .description = "The Nix store to use for evaluations.", + .category = category, + .labels = {"store-url"}, + .handler = {&evalStoreUrl}, + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh index be7fda783..0e113fff1 100644 --- a/src/libexpr/common-eval-args.hh +++ b/src/libexpr/common-eval-args.hh @@ -16,8 +16,9 @@ struct MixEvalArgs : virtual Args Strings searchPath; -private: + std::optional<std::string> evalStoreUrl; +private: std::map<std::string, std::string> autoArgs; }; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c3206a577..327f7e974 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -378,7 +378,10 @@ static Strings parseNixPath(const string & s) } -EvalState::EvalState(const Strings & _searchPath, ref<Store> store) +EvalState::EvalState( + const Strings & _searchPath, + ref<Store> store, + std::shared_ptr<Store> buildStore) : sWith(symbols.create("<with>")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) @@ -411,6 +414,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sEpsilon(symbols.create("")) , repair(NoRepair) , store(store) + , buildStore(buildStore ? buildStore : store) , regexCache(makeRegexCache()) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) @@ -776,18 +780,10 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) } -std::atomic<uint64_t> nrValuesFreed{0}; - -void finalizeValue(void * obj, void * data) -{ - nrValuesFreed++; -} - Value * EvalState::allocValue() { nrValues++; auto v = (Value *) allocBytes(sizeof(Value)); - //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr); return v; } @@ -854,39 +850,37 @@ Value * Expr::maybeThunk(EvalState & state, Env & env) } -unsigned long nrAvoided = 0; - Value * ExprVar::maybeThunk(EvalState & state, Env & env) { Value * v = state.lookupVar(&env, *this, true); /* The value might not be initialised in the environment yet. In that case, ignore it. */ - if (v) { nrAvoided++; return v; } + if (v) { state.nrAvoided++; return v; } return Expr::maybeThunk(state, env); } Value * ExprString::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } Value * ExprInt::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } Value * ExprFloat::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } Value * ExprPath::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } @@ -1141,8 +1135,6 @@ static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPa } -unsigned long nrLookups = 0; - void ExprSelect::eval(EvalState & state, Env & env, Value & v) { Value vTmp; @@ -1154,7 +1146,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) try { for (auto & i : attrPath) { - nrLookups++; + state.nrLookups++; Bindings::iterator j; Symbol name = getName(i, state, env); if (def) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e3eaed6d3..6f3474854 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -94,8 +94,12 @@ public: Value vEmptySet; + /* Store used to materialise .drv files. */ const ref<Store> store; + /* Store used to build stuff. */ + const ref<Store> buildStore; + private: SrcToStore srcToStore; @@ -128,7 +132,10 @@ private: public: - EvalState(const Strings & _searchPath, ref<Store> store); + EvalState( + const Strings & _searchPath, + ref<Store> store, + std::shared_ptr<Store> buildStore = nullptr); ~EvalState(); void addToSearchPath(const string & s); @@ -316,8 +323,10 @@ private: unsigned long nrValuesInEnvs = 0; unsigned long nrValues = 0; unsigned long nrListElems = 0; + unsigned long nrLookups = 0; unsigned long nrAttrsets = 0; unsigned long nrAttrsInAttrsets = 0; + unsigned long nrAvoided = 0; unsigned long nrOpUpdates = 0; unsigned long nrOpUpdateValuesCopied = 0; unsigned long nrListConcats = 0; @@ -339,6 +348,11 @@ private: friend struct ExprOpUpdate; friend struct ExprOpConcatLists; + friend struct ExprVar; + friend struct ExprString; + friend struct ExprInt; + friend struct ExprFloat; + friend struct ExprPath; friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 86930c2e3..3a9529c4d 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -238,7 +238,7 @@ LegacyArgs::LegacyArgs(const std::string & programName, addFlag({ .longName = "no-gc-warning", .description = "Disable warnings about not using `--add-root`.", - .handler = {&gcWarning, true}, + .handler = {&gcWarning, false}, }); addFlag({ diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 7df597400..876b8def0 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -165,7 +165,7 @@ void DerivationGoal::getDerivation() /* The first thing to do is to make sure that the derivation exists. If it doesn't, it may be created through a substitute. */ - if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { + if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { loadDerivation(); return; } @@ -188,12 +188,12 @@ void DerivationGoal::loadDerivation() /* `drvPath' should already be a root, but let's be on the safe side: if the user forgot to make it a root, we wouldn't want things being garbage collected while we're busy. */ - worker.store.addTempRoot(drvPath); + worker.evalStore.addTempRoot(drvPath); - assert(worker.store.isValidPath(drvPath)); + assert(worker.evalStore.isValidPath(drvPath)); /* Get the derivation. */ - drv = std::make_unique<Derivation>(worker.store.derivationFromPath(drvPath)); + drv = std::make_unique<Derivation>(worker.evalStore.derivationFromPath(drvPath)); haveDerivation(); } @@ -212,8 +212,8 @@ void DerivationGoal::haveDerivation() if (i.second.second) worker.store.addTempRoot(*i.second.second); - auto outputHashes = staticOutputHashes(worker.store, *drv); - for (auto &[outputName, outputHash] : outputHashes) + auto outputHashes = staticOutputHashes(worker.evalStore, *drv); + for (auto & [outputName, outputHash] : outputHashes) initialOutputs.insert({ outputName, InitialOutput{ @@ -337,6 +337,15 @@ void DerivationGoal::gaveUpOnSubstitution() for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + /* Copy the input sources from the eval store to the build + store. */ + if (&worker.evalStore != &worker.store) { + RealisedPath::Set inputSrcs; + for (auto & i : drv->inputSrcs) + inputSrcs.insert(i); + copyClosure(worker.evalStore, worker.store, inputSrcs); + } + for (auto & i : drv->inputSrcs) { if (worker.store.isValidPath(i)) continue; if (!settings.useSubstitutes) @@ -478,8 +487,8 @@ void DerivationGoal::inputsRealised() /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - assert(worker.store.isValidPath(drvPath)); - auto outputs = worker.store.queryPartialDerivationOutputMap(depDrvPath); + assert(worker.evalStore.isValidPath(drvPath)); + auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath); for (auto & j : wantedDepOutputs) { if (outputs.count(j) > 0) { auto optRealizedInput = outputs.at(j); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 732d4785d..96deb81d1 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,9 +6,9 @@ namespace nix { -void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode) +void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore) { - Worker worker(*this); + Worker worker(*this, evalStore ? *evalStore : *this); Goals goals; for (auto & br : reqs) { @@ -51,7 +51,7 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - Worker worker(*this); + Worker worker(*this, *this); auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); BuildResult result; @@ -93,7 +93,7 @@ void Store::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - Worker worker(*this); + Worker worker(*this, *this); GoalPtr goal = worker.makePathSubstitutionGoal(path); Goals goals = {goal}; @@ -111,7 +111,7 @@ void Store::ensurePath(const StorePath & path) void LocalStore::repairPath(const StorePath & path) { - Worker worker(*this); + Worker worker(*this, *this); GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); Goals goals = {goal}; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 11c99d9f0..990ff60b7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1254,8 +1254,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo return next->queryRealisation(id); } - void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override + void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override { + assert(!evalStore); + if (buildMode != bmNormal) throw Error("unsupported build mode"); StorePathSet newPaths; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index e56cfadbe..29a8cfb87 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -204,7 +204,7 @@ void PathSubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); - copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), + copyStorePath(*sub, worker.store, subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 0f2ade348..a7a6b92a6 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -9,11 +9,12 @@ namespace nix { -Worker::Worker(Store & store) +Worker::Worker(Store & store, Store & evalStore) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) , actSubstitutions(*logger, actCopyPaths) , store(store) + , evalStore(evalStore) { /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 918de35f6..6a3b99c02 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -110,6 +110,7 @@ public: bool checkMismatch; Store & store; + Store & evalStore; std::unique_ptr<HookInstance> hook; @@ -131,7 +132,7 @@ public: it answers with "decline-permanently", we don't try again. */ bool tryBuildHook = true; - Worker(Store & store); + Worker(Store & store, Store & evalStore); ~Worker(); /* Make a goal (with caching). */ diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e06fb9ce2..d68ff64d7 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -243,23 +243,6 @@ struct ClientSettings } }; -static void writeValidPathInfo( - ref<Store> store, - unsigned int clientVersion, - Sink & to, - std::shared_ptr<const ValidPathInfo> info) -{ - to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); - worker_proto::write(*store, to, info->references); - to << info->registrationTime << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { - to << info->ultimate - << info->sigs - << renderContentAddress(info->ca); - } -} - static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, Source & from) { std::vector<DerivedPath> reqs; @@ -422,9 +405,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, }(); logger->stopWork(); - to << store->printStorePath(pathInfo->path); - writeValidPathInfo(store, clientVersion, to, pathInfo); - + pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion)); } else { HashType hashAlgo; std::string baseName; @@ -471,6 +452,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } + case wopAddMultipleToStore: { + bool repair, dontCheckSigs; + from >> repair >> dontCheckSigs; + if (!trusted && dontCheckSigs) + dontCheckSigs = false; + + logger->startWork(); + FramedSource source(from); + store->addMultipleToStore(source, + RepairFlag{repair}, + dontCheckSigs ? NoCheckSigs : CheckSigs); + logger->stopWork(); + break; + } + case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); @@ -770,7 +766,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - writeValidPathInfo(store, clientVersion, to, info); + info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index f6defd98f..899475860 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -568,7 +568,7 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m } -std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv) +std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv) { std::map<std::string, Hash> res; std::visit(overloaded { diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 8f26af685..36c6e725c 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -43,11 +43,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store RepairFlag repair, CheckSigsFlag checkSigs) override { unsupported("addToStore"); } - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override - { unsupported("addToStore"); } - StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override { unsupported("addTextToStore"); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index edaf75136..45eed5707 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -267,8 +267,11 @@ public: return status; } - void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode) override + void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override { + if (evalStore && evalStore.get() != this) + throw Error("building on an SSH store is incompatible with '--eval-store'"); + auto conn(connections->get()); conn->to << cmdBuildPaths; diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc new file mode 100644 index 000000000..fda55b2b6 --- /dev/null +++ b/src/libstore/path-info.cc @@ -0,0 +1,46 @@ +#include "path-info.hh" +#include "worker-protocol.hh" + +namespace nix { + +ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) +{ + return read(source, store, format, store.parseStorePath(readString(source))); +} + +ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path) +{ + auto deriver = readString(source); + auto narHash = Hash::parseAny(readString(source), htSHA256); + ValidPathInfo info(path, narHash); + if (deriver != "") info.deriver = store.parseStorePath(deriver); + info.references = worker_proto::read(store, source, Phantom<StorePathSet> {}); + source >> info.registrationTime >> info.narSize; + if (format >= 16) { + source >> info.ultimate; + info.sigs = readStrings<StringSet>(source); + info.ca = parseContentAddressOpt(readString(source)); + } + return info; +} + +void ValidPathInfo::write( + Sink & sink, + const Store & store, + unsigned int format, + bool includePath) const +{ + if (includePath) + sink << store.printStorePath(path); + sink << (deriver ? store.printStorePath(*deriver) : "") + << narHash.to_string(Base16, false); + worker_proto::write(store, sink, references); + sink << registrationTime << narSize; + if (format >= 16) { + sink << ultimate + << sigs + << renderContentAddress(ca); + } +} + +} diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index de87f8b33..b4b54e593 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -105,6 +105,11 @@ struct ValidPathInfo ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; virtual ~ValidPathInfo() { } + + static ValidPathInfo read(Source & source, const Store & store, unsigned int format); + static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path); + + void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const; }; typedef std::map<StorePath, ValidPathInfo> ValidPathInfos; diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 05d2bc44f..9070a6ee2 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -45,7 +45,7 @@ struct Realisation { size_t checkSignatures(const PublicKeys & publicKeys) const; static std::set<Realisation> closure(Store &, const std::set<Realisation> &); - static void closure(Store &, const std::set<Realisation> &, std::set<Realisation>& res); + static void closure(Store &, const std::set<Realisation> &, std::set<Realisation> & res); bool isCompatibleWith(const Realisation & other) const; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index aec243637..140f39120 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -386,23 +386,6 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } -ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) -{ - auto deriver = readString(conn->from); - auto narHash = Hash::parseAny(readString(conn->from), htSHA256); - auto info = make_ref<ValidPathInfo>(path, narHash); - if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings<StringSet>(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); - } - return info; -} - - void RemoteStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { @@ -423,7 +406,8 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = readValidPathInfo(conn, path); + info = std::make_shared<ValidPathInfo>( + ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path})); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -525,8 +509,8 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( }); } - auto path = parseStorePath(readString(conn->from)); - return readValidPathInfo(conn, path); + return make_ref<ValidPathInfo>( + ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion))); } else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); @@ -642,6 +626,25 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } +void RemoteStore::addMultipleToStore( + Source & source, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) { + auto conn(getConnection()); + conn->to + << wopAddMultipleToStore + << repair + << !checkSigs; + conn.withFramedSink([&](Sink & sink) { + source.drainInto(sink); + }); + } else + Store::addMultipleToStore(source, repair, checkSigs); +} + + StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { @@ -705,8 +708,18 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons } } -void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) { + if (evalStore && evalStore.get() != this) { + /* The remote doesn't have a way to access evalStore, so copy + the .drvs. */ + RealisedPath::Set drvPaths2; + for (auto & i : drvPaths) + if (auto p = std::get_if<DerivedPath::Built>(&i)) + drvPaths2.insert(p->drvPath); + copyClosure(*evalStore, *this, drvPaths2); + } + auto conn(getConnection()); conn->to << wopBuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); @@ -1001,14 +1014,14 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun) +void ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun) { (*this)->to.flush(); std::exception_ptr ex; - /* Handle log messages / exceptions from the remote on a - separate thread. */ + /* Handle log messages / exceptions from the remote on a separate + thread. */ std::thread stderrThread([&]() { try { @@ -1041,7 +1054,6 @@ void ConnectionHandle::withFramedSink(std::function<void(Sink &sink)> fun) stderrThread.join(); if (ex) std::rethrow_exception(ex); - } } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 6cf76a46d..8901c79fc 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -78,6 +78,11 @@ public: void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; + void addMultipleToStore( + Source & source, + RepairFlag repair, + CheckSigsFlag checkSigs) override; + StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; @@ -85,7 +90,7 @@ public: std::optional<const Realisation> queryRealisation(const DrvOutput &) override; - void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override; + void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override; BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -151,8 +156,6 @@ protected: virtual void narFromPath(const StorePath & path, Sink & sink) override; - ref<const ValidPathInfo> readValidPathInfo(ConnectionHandle & conn, const StorePath & path); - private: std::atomic_bool failed{false}; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 40a406468..970bafd88 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -9,6 +9,7 @@ #include "url.hh" #include "archive.hh" #include "callback.hh" +#include "remote-store.hh" #include <regex> @@ -249,6 +250,20 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath, } +void Store::addMultipleToStore( + Source & source, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + auto expected = readNum<uint64_t>(source); + for (uint64_t i = 0; i < expected; ++i) { + auto info = ValidPathInfo::read(source, *this, 16); + info.ultimate = false; + addToStore(info, source, repair, checkSigs); + } +} + + /* The aim of this function is to compute in one pass the correct ValidPathInfo for the files that we are trying to add to the store. To accomplish that in one @@ -770,30 +785,43 @@ const Store::Stats & Store::getStats() } -void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) +static std::string makeCopyPathMessage( + std::string_view srcUri, + std::string_view dstUri, + std::string_view storePath) { - auto srcUri = srcStore->getUri(); - auto dstUri = dstStore->getUri(); + return srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri); +} + +void copyStorePath( + Store & srcStore, + Store & dstStore, + const StorePath & storePath, + RepairFlag repair, + CheckSigsFlag checkSigs) +{ + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(storePath); Activity act(*logger, lvlInfo, actCopyPath, - srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri) - : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri) - : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri), - {srcStore->printStorePath(storePath), srcUri, dstUri}); + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); PushActivity pact(act.id); - auto info = srcStore->queryPathInfo(storePath); + auto info = srcStore.queryPathInfo(storePath); uint64_t total = 0; // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref<ValidPathInfo>(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); - if (dstStore->storeDir == srcStore->storeDir) + info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca); + if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; } @@ -810,17 +838,22 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, act.progress(total, info->narSize); }); TeeSink tee { sink, progressSink }; - srcStore->narFromPath(storePath, tee); + srcStore.narFromPath(storePath, tee); }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(storePath), srcStore.getUri()); }); - dstStore->addToStore(*info, *source, repair, checkSigs); + dstStore.addToStore(*info, *source, repair, checkSigs); } -std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths, - RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) +std::map<StorePath, StorePath> copyPaths( + Store & srcStore, + Store & dstStore, + const RealisedPath::Set & paths, + RepairFlag repair, + CheckSigsFlag checkSigs, + SubstituteFlag substitute) { StorePathSet storePaths; std::set<Realisation> toplevelRealisations; @@ -838,11 +871,11 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor try { // Copy the realisation closure processGraph<Realisation>( - pool, Realisation::closure(*srcStore, toplevelRealisations), + pool, Realisation::closure(srcStore, toplevelRealisations), [&](const Realisation & current) -> std::set<Realisation> { std::set<Realisation> children; for (const auto & [drvOutput, _] : current.dependentRealisations) { - auto currentChild = srcStore->queryRealisation(drvOutput); + auto currentChild = srcStore.queryRealisation(drvOutput); if (!currentChild) throw Error( "incomplete realisation closure: '%s' is a " @@ -853,9 +886,9 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor return children; }, [&](const Realisation& current) -> void { - dstStore->registerDrvOutput(current, checkSigs); + dstStore.registerDrvOutput(current, checkSigs); }); - } catch (MissingExperimentalFeature& e) { + } catch (MissingExperimentalFeature & e) { // Don't fail if the remote doesn't support CA derivations is it might // not be within our control to change that, and we might still want // to at least copy the output paths. @@ -868,10 +901,15 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor return pathsMap; } -std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, - RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) +std::map<StorePath, StorePath> copyPaths( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths, + RepairFlag repair, + CheckSigsFlag checkSigs, + SubstituteFlag substitute) { - auto valid = dstStore->queryValidPaths(storePaths, substitute); + auto valid = dstStore.queryValidPaths(storePaths, substitute); StorePathSet missing; for (auto & path : storePaths) @@ -881,9 +919,31 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor for (auto & path : storePaths) pathsMap.insert_or_assign(path, path); - Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + auto sorted = srcStore.topoSortPaths(missing); + std::reverse(sorted.begin(), sorted.end()); + + auto source = sinkToSource([&](Sink & sink) { + sink << sorted.size(); + for (auto & storePath : sorted) { + auto srcUri = srcStore.getUri(); + auto dstUri = dstStore.getUri(); + auto storePathS = srcStore.printStorePath(storePath); + Activity act(*logger, lvlInfo, actCopyPath, + makeCopyPathMessage(srcUri, dstUri, storePathS), + {storePathS, srcUri, dstUri}); + PushActivity pact(act.id); + + auto info = srcStore.queryPathInfo(storePath); + info->write(sink, srcStore, 16); + srcStore.narFromPath(storePath, sink); + } + }); + + dstStore.addMultipleToStore(*source, repair, checkSigs); + + #if 0 std::atomic<size_t> nrDone{0}; std::atomic<size_t> nrFailed{0}; std::atomic<uint64_t> bytesExpected{0}; @@ -899,18 +959,21 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor StorePathSet(missing.begin(), missing.end()), [&](const StorePath & storePath) { - auto info = srcStore->queryPathInfo(storePath); + auto info = srcStore.queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore->storeDir == srcStore->storeDir) + storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); + if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + debug("replaced path '%s' to '%s' for substituter '%s'", + srcStore.printStorePath(storePath), + dstStore.printStorePath(storePathForDst), + dstStore.getUri()); } pathsMap.insert_or_assign(storePath, storePathForDst); - if (dstStore->isValidPath(storePath)) { + if (dstStore.isValidPath(storePath)) { nrDone++; showProgress(); return StorePathSet(); @@ -925,19 +988,22 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor [&](const StorePath & storePath) { checkInterrupt(); - auto info = srcStore->queryPathInfo(storePath); + auto info = srcStore.queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore->storeDir == srcStore->storeDir) + storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); + if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) - debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri()); + debug("replaced path '%s' to '%s' for substituter '%s'", + srcStore.printStorePath(storePath), + dstStore.printStorePath(storePathForDst), + dstStore.getUri()); } pathsMap.insert_or_assign(storePath, storePathForDst); - if (!dstStore->isValidPath(storePathForDst)) { + if (!dstStore.isValidPath(storePathForDst)) { MaintainCount<decltype(nrRunning)> mc(nrRunning); showProgress(); try { @@ -946,7 +1012,7 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what())); + logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what())); showProgress(); return; } @@ -955,9 +1021,27 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor nrDone++; showProgress(); }); + #endif + return pathsMap; } +void copyClosure( + Store & srcStore, + Store & dstStore, + const RealisedPath::Set & paths, + RepairFlag repair, + CheckSigsFlag checkSigs, + SubstituteFlag substitute) +{ + if (&srcStore == &dstStore) return; + + RealisedPath::Set closure; + RealisedPath::closure(srcStore, paths, closure); + + copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); +} + std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven) { std::string path; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 4faa65228..4fb6c40c7 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -440,6 +440,12 @@ public: virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; + /* Import multiple paths into the store. */ + virtual void addMultipleToStore( + Source & source, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs); + /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see @@ -497,7 +503,8 @@ public: not derivations, substitute them. */ virtual void buildPaths( const std::vector<DerivedPath> & paths, - BuildMode buildMode = bmNormal); + BuildMode buildMode = bmNormal, + std::shared_ptr<Store> evalStore = nullptr); /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). @@ -751,8 +758,12 @@ protected: /* Copy a path from one store to another. */ -void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); +void copyStorePath( + Store & srcStore, + Store & dstStore, + const StorePath & storePath, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs); /* Copy store paths from one store to another. The paths may be copied @@ -761,17 +772,27 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, of store paths is not automatically closed; use copyClosure() for that. Returns a map of what each path was copied to the dstStore as. */ -std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, +std::map<StorePath, StorePath> copyPaths( + Store & srcStore, Store & dstStore, const RealisedPath::Set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); -std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, + +std::map<StorePath, StorePath> copyPaths( + Store & srcStore, Store & dstStore, const StorePathSet & paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); +/* Copy the closure of `paths` from `srcStore` to `dstStore`. */ +void copyClosure( + Store & srcStore, Store & dstStore, + const RealisedPath::Set & paths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); /* Remove the temporary roots file for this process. Any temporary root becomes garbage after this point unless it has been registered diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index e89183d40..93cf546d2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 31) +#define PROTOCOL_VERSION (1 << 8 | 32) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -55,6 +55,7 @@ typedef enum { wopQueryDerivationOutputMap = 41, wopRegisterDrvOutput = 42, wopQueryRealisation = 43, + wopAddMultipleToStore = 44, } WorkerOp; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index b8334defc..d876315c8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -435,7 +435,7 @@ static void _deletePath(const Path & path, uint64_t & bytesFreed) if (dir == "") dir = "/"; - AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY)); + AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; if (!dirfd) { if (errno == ENOENT) return; throw SysError("opening directory '%1%'", path); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 29be46cf5..848b108dd 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -250,8 +250,9 @@ static void main_nix_build(int argc, char * * argv) throw UsageError("'-p' and '-E' are mutually exclusive"); auto store = openStore(); + auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + auto state = std::make_unique<EvalState>(myArgs.searchPath, evalStore, store); state->repair = repair; auto autoArgs = myArgs.getAutoArgs(*state); @@ -301,8 +302,8 @@ static void main_nix_build(int argc, char * * argv) absolute = canonPath(absPath(i), true); } catch (Error & e) {}; auto [path, outputNames] = parsePathWithOutputs(absolute); - if (store->isStorePath(path) && hasSuffix(path, ".drv")) - drvs.push_back(DrvInfo(*state, store, absolute)); + if (evalStore->isStorePath(path) && hasSuffix(path, ".drv")) + drvs.push_back(DrvInfo(*state, evalStore, absolute)); else /* If we're in a #! script, interpret filenames relative to the script. */ @@ -340,7 +341,7 @@ static void main_nix_build(int argc, char * * argv) printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, downloadSize, narSize); if (!dryRun) - store->buildPaths(paths, buildMode); + store->buildPaths(paths, buildMode, evalStore); }; if (runEnv) { @@ -348,9 +349,10 @@ static void main_nix_build(int argc, char * * argv) throw UsageError("nix-shell requires a single derivation"); auto & drvInfo = drvs.front(); - auto drv = store->derivationFromPath(store->parseStorePath(drvInfo.queryDrvPath())); + auto drv = evalStore->derivationFromPath(evalStore->parseStorePath(drvInfo.queryDrvPath())); std::vector<StorePathWithOutputs> pathsToBuild; + RealisedPath::Set pathsToCopy; /* Figure out what bash shell to use. If $NIX_BUILD_SHELL is not set, then build bashInteractive from @@ -369,7 +371,9 @@ static void main_nix_build(int argc, char * * argv) if (!drv) throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation"); - pathsToBuild.push_back({store->parseStorePath(drv->queryDrvPath())}); + auto bashDrv = store->parseStorePath(drv->queryDrvPath()); + pathsToBuild.push_back({bashDrv}); + pathsToCopy.insert(bashDrv); shell = drv->queryOutPath() + "/bin/bash"; @@ -384,9 +388,14 @@ static void main_nix_build(int argc, char * * argv) for (const auto & input : drv.inputDrvs) if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(store->printStorePath(input.first), std::regex(exclude)); })) + { pathsToBuild.push_back({input.first, input.second}); - for (const auto & src : drv.inputSrcs) + pathsToCopy.insert(input.first); + } + for (const auto & src : drv.inputSrcs) { pathsToBuild.push_back({src}); + pathsToCopy.insert(src); + } buildPaths(pathsToBuild); @@ -438,7 +447,7 @@ static void main_nix_build(int argc, char * * argv) if (env.count("__json")) { StorePathSet inputs; for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) { - auto outputs = store->queryPartialDerivationOutputMap(depDrvPath); + auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath); for (auto & i : wantedDepOutputs) { auto o = outputs.at(i); store->computeFSClosure(*o, inputs); @@ -531,6 +540,7 @@ static void main_nix_build(int argc, char * * argv) std::vector<StorePathWithOutputs> pathsToBuild; std::vector<std::pair<StorePath, std::string>> pathsToBuildOrdered; + RealisedPath::Set drvsToCopy; std::map<StorePath, std::pair<size_t, StringSet>> drvMap; @@ -543,13 +553,13 @@ static void main_nix_build(int argc, char * * argv) pathsToBuild.push_back({drvPath, {outputName}}); pathsToBuildOrdered.push_back({drvPath, {outputName}}); + drvsToCopy.insert(drvPath); auto i = drvMap.find(drvPath); if (i != drvMap.end()) i->second.second.insert(outputName); - else { + else drvMap[drvPath] = {drvMap.size(), {outputName}}; - } } buildPaths(pathsToBuild); @@ -564,7 +574,7 @@ static void main_nix_build(int argc, char * * argv) if (counter) drvPrefix += fmt("-%d", counter + 1); - auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath); + auto builtOutputs = evalStore->queryPartialDerivationOutputMap(drvPath); auto maybeOutputPath = builtOutputs.at(outputName); assert(maybeOutputPath); diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 02ccbe541..841d50fd3 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -54,10 +54,7 @@ static int main_nix_copy_closure(int argc, char ** argv) for (auto & path : storePaths) storePaths2.insert(from->followLinksToStorePath(path)); - RealisedPath::Set closure; - RealisedPath::closure(*from, storePaths2, closure); - - copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); + copyClosure(*from, *to, storePaths2, NoRepair, NoCheckSigs, useSubstitutes); return 0; } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 95903d882..25d0fa3ba 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -153,8 +153,9 @@ static int main_nix_instantiate(int argc, char * * argv) settings.readOnlyMode = true; auto store = openStore(); + auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + auto state = std::make_unique<EvalState>(myArgs.searchPath, evalStore, store); state->repair = repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/app.cc b/src/nix/app.cc index 01a0064db..9719a65dd 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -100,7 +100,8 @@ UnresolvedApp Installable::toApp(EvalState & state) throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); } -App UnresolvedApp::resolve(ref<Store> store) +// FIXME: move to libcmd +App UnresolvedApp::resolve(ref<Store> evalStore, ref<Store> store) { auto res = unresolved; @@ -110,7 +111,7 @@ App UnresolvedApp::resolve(ref<Store> store) installableContext.push_back( std::make_shared<InstallableDerivedPath>(store, ctxElt.toDerivedPath())); - auto builtContext = build(store, Realise::Outputs, installableContext); + auto builtContext = build(evalStore, store, Realise::Outputs, installableContext); res.program = resolveString(*store, unresolved.program, builtContext); if (!store->isInStore(res.program)) throw Error("app program '%s' is not in the Nix store", res.program); diff --git a/src/nix/build.cc b/src/nix/build.cc index 15923ebc3..13eb66ac6 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -52,7 +52,10 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile void run(ref<Store> store) override { - auto buildables = build(store, dryRun ? Realise::Nothing : Realise::Outputs, installables, buildMode); + auto buildables = build( + getEvalStore(), store, + dryRun ? Realise::Nothing : Realise::Outputs, + installables, buildMode); if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 88bc3d1d1..cedb5704c 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -69,7 +69,7 @@ struct CmdBundle : InstallableCommand { auto evalState = getEvalState(); - auto app = installable->toApp(*evalState).resolve(store); + auto app = installable->toApp(*evalState).resolve(getEvalStore(), store); auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 674cce4b4..0489dfe06 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -90,7 +90,7 @@ struct CmdCopy : BuiltPathsCommand } copyPaths( - srcStore, dstStore, stuffToCopy, NoRepair, checkSigs, substitute); + *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute); } }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 9ac2791f8..55023545d 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -7,6 +7,7 @@ #include "derivations.hh" #include "affinity.hh" #include "progress-bar.hh" +#include "run.hh" #include <nlohmann/json.hpp> @@ -171,15 +172,15 @@ const static std::string getEnvSh = modified derivation with the same dependencies and nearly the same initial environment variables, that just writes the resulting environment to a file and exits. */ -StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) +static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore, const StorePath & drvPath) { - auto drv = store->derivationFromPath(drvPath); + auto drv = evalStore->derivationFromPath(drvPath); auto builder = baseNameOf(drv.builder); if (builder != "bash") throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); - auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); + auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {}); drv.args = {store->printStorePath(getEnvShPath)}; @@ -205,7 +206,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; drv.env[output.first] = ""; } - Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); + Hash h = std::get<0>(hashDerivationModulo(*evalStore, drv, true)); for (auto & output : drv.outputs) { auto outPath = store->makeOutputPath(output.first, h, drv.name); @@ -214,12 +215,12 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) } } - auto shellDrvPath = writeDerivation(*store, drv); + auto shellDrvPath = writeDerivation(*evalStore, drv); /* Build the derivation. */ - store->buildPaths({DerivedPath::Built{shellDrvPath}}); + store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore); - for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) { + for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); auto & outPath = *optPath; assert(store->isValidPath(outPath)); @@ -307,7 +308,7 @@ struct Common : InstallableCommand, MixProfile auto dir = absPath(dir_); auto installable = parseInstallable(store, installable_); auto builtPaths = toStorePaths( - store, Realise::Nothing, OperateOn::Output, {installable}); + getEvalStore(), store, Realise::Nothing, OperateOn::Output, {installable}); for (auto & path: builtPaths) { auto from = store->printStorePath(path); if (script.find(from) == std::string::npos) @@ -347,7 +348,7 @@ struct Common : InstallableCommand, MixProfile auto & drvPath = *drvs.begin(); - return getDerivationEnvironment(store, drvPath); + return getDerivationEnvironment(store, getEvalStore(), drvPath); } } @@ -361,7 +362,7 @@ struct Common : InstallableCommand, MixProfile debug("reading environment file '%s'", strPath); - return {BuildEnvironment::fromJSON(readFile(strPath)), strPath}; + return {BuildEnvironment::fromJSON(readFile(store->toRealPath(shellOutPath))), strPath}; } }; @@ -472,8 +473,6 @@ struct CmdDevelop : Common, MixEnvironment writeFull(rcFileFd.get(), script); - stopProgressBar(); - setEnviron(); // prevent garbage collection until shell exits setenv("NIX_GCROOT", gcroot.data(), 1); @@ -495,8 +494,8 @@ struct CmdDevelop : Common, MixEnvironment Strings{"legacyPackages." + settings.thisSystem.get() + "."}, nixpkgsLockFlags); - shell = state->store->printStorePath( - toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; + shell = store->printStorePath( + toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; } catch (Error &) { ignoreException(); } @@ -506,11 +505,7 @@ struct CmdDevelop : Common, MixEnvironment auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; - restoreProcessContext(); - - execvp(shell.c_str(), stringsToCharPtrs(args).data()); - - throw SysError("executing shell '%s'", shell); + runProgramInStore(store, shell, args); } }; diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 0c7d531c1..734c41e0e 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -131,9 +131,9 @@ struct CmdDiffClosures : SourceExprCommand void run(ref<Store> store) override { auto before = parseInstallable(store, _before); - auto beforePath = toStorePath(store, Realise::Outputs, operateOn, before); + auto beforePath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, before); auto after = parseInstallable(store, _after); - auto afterPath = toStorePath(store, Realise::Outputs, operateOn, after); + auto afterPath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, after); printClosureDiff(store, beforePath, afterPath, ""); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 23feed24b..abb0fd3b4 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -841,7 +841,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (!dryRun && !dstUri.empty()) { ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(store, dstStore, sources); + copyPaths(*store, *dstStore, sources); } } }; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 511771f89..8cef6d0b6 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -253,7 +253,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile manifest.elements.emplace_back(std::move(element)); } else { - auto buildables = build(store, Realise::Outputs, {installable}, bmNormal); + auto buildables = build(getEvalStore(), store, Realise::Outputs, {installable}, bmNormal); for (auto & buildable : buildables) { ProfileElement element; diff --git a/src/nix/run.cc b/src/nix/run.cc index c0ba05a3e..7597b61f7 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,3 +1,4 @@ +#include "run.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -20,45 +21,43 @@ using namespace nix; std::string chrootHelperName = "__run_in_chroot"; -struct RunCommon : virtual Command -{ +namespace nix { - using Command::run; +void runProgramInStore(ref<Store> store, + const std::string & program, + const Strings & args) +{ + stopProgressBar(); - void runProgram(ref<Store> store, - const std::string & program, - const Strings & args) - { - stopProgressBar(); + restoreProcessContext(); - restoreProcessContext(); + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded program + (which "nix" is), so we exec() a single-threaded helper program + (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast<LocalStore>(); - /* If this is a diverted store (i.e. its "logical" location - (typically /nix/store) differs from its "physical" location - (e.g. /home/eelco/nix/store), then run the command in a - chroot. For non-root users, this requires running it in new - mount and user namespaces. Unfortunately, - unshare(CLONE_NEWUSER) doesn't work in a multithreaded - program (which "nix" is), so we exec() a single-threaded - helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast<LocalStore>(); + if (store2 && store->storeDir != store2->getRealStoreDir()) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; + for (auto & arg : args) helperArgs.push_back(arg); - if (store2 && store->storeDir != store2->getRealStoreDir()) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; - for (auto & arg : args) helperArgs.push_back(arg); + execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); - execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + throw SysError("could not execute chroot helper"); + } - throw SysError("could not execute chroot helper"); - } + execvp(program.c_str(), stringsToCharPtrs(args).data()); - execvp(program.c_str(), stringsToCharPtrs(args).data()); + throw SysError("unable to execute '%s'", program); +} - throw SysError("unable to execute '%s'", program); - } -}; +} -struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment +struct CmdShell : InstallablesCommand, MixEnvironment { using InstallablesCommand::run; @@ -93,7 +92,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment void run(ref<Store> store) override { - auto outPaths = toStorePaths(store, Realise::Outputs, OperateOn::Output, installables); + auto outPaths = toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); auto accessor = store->getFSAccessor(); @@ -125,13 +124,13 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment Strings args; for (auto & arg : command) args.push_back(arg); - runProgram(store, *command.begin(), args); + runProgramInStore(store, *command.begin(), args); } }; static auto rCmdShell = registerCommand<CmdShell>("shell"); -struct CmdRun : InstallableCommand, RunCommon +struct CmdRun : InstallableCommand { using InstallableCommand::run; @@ -178,12 +177,12 @@ struct CmdRun : InstallableCommand, RunCommon { auto state = getEvalState(); - auto app = installable->toApp(*state).resolve(store); + auto app = installable->toApp(*state).resolve(getEvalStore(), store); Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgram(store, app.program, allArgs); + runProgramInStore(store, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh new file mode 100644 index 000000000..6180a87dd --- /dev/null +++ b/src/nix/run.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +void runProgramInStore(ref<Store> store, + const std::string & program, + const Strings & args); + +} diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 7a4ca5172..2f6b361bb 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -62,9 +62,9 @@ struct CmdWhyDepends : SourceExprCommand void run(ref<Store> store) override { auto package = parseInstallable(store, _package); - auto packagePath = toStorePath(store, Realise::Outputs, operateOn, package); + auto packagePath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, package); auto dependency = parseInstallable(store, _dependency); - auto dependencyPath = toStorePath(store, Realise::Derivation, operateOn, dependency); + auto dependencyPath = toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency); auto dependencyPathHash = dependencyPath.hashPart(); StorePathSet closure; |