diff options
Diffstat (limited to 'src')
56 files changed, 1014 insertions, 546 deletions
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 0f6125f11..84bbb5292 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -85,11 +85,12 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions { std::optional<Path> file; std::optional<std::string> expr; + bool readOnlyMode = false; // FIXME: move this; not all commands (e.g. 'nix run') use it. OperateOn operateOn = OperateOn::Output; - SourceExprCommand(); + SourceExprCommand(bool supportReadOnlyMode = false); std::vector<std::shared_ptr<Installable>> parseInstallables( ref<Store> store, std::vector<std::string> ss); @@ -128,13 +129,13 @@ struct InstallableCommand : virtual Args, SourceExprCommand { std::shared_ptr<Installable> installable; - InstallableCommand(); + InstallableCommand(bool supportReadOnlyMode = false); void prepare() override; std::optional<FlakeRef> getFlakeRefForCompletion() override { - return parseFlakeRef(_installable, absPath(".")); + return parseFlakeRefWithFragment(_installable, absPath(".")).first; } private: diff --git a/src/libexpr/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e50ff244c..5b6e82388 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -7,6 +7,7 @@ #include "registry.hh" #include "flake/flakeref.hh" #include "store-api.hh" +#include "command.hh" namespace nix { @@ -59,6 +60,9 @@ MixEvalArgs::MixEvalArgs() fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); + }}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(openStore(), prefix); }} }); diff --git a/src/libexpr/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 03fa226aa..03fa226aa 100644 --- a/src/libexpr/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 784117569..74c8a6df5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,4 +1,6 @@ +#include "globals.hh" #include "installables.hh" +#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" @@ -99,6 +101,14 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), parseFlakeRef(flakeRef, absPath("."), true)); + }}, + .completer = {[&](size_t n, std::string_view prefix) { + if (n == 0) { + if (auto flakeRef = getFlakeRefForCompletion()) + completeFlakeInputPath(getEvalState(), *flakeRef, prefix); + } else if (n == 1) { + completeFlakeRef(getEvalState()->store, prefix); + } }} }); @@ -129,7 +139,7 @@ MixFlakeOptions::MixFlakeOptions() }); } -SourceExprCommand::SourceExprCommand() +SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) { addFlag({ .longName = "file", @@ -157,6 +167,17 @@ SourceExprCommand::SourceExprCommand() .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); + + if (supportReadOnlyMode) { + addFlag({ + .longName = "read-only", + .description = + "Do not instantiate each evaluated derivation. " + "This improves performance, but can cause errors when accessing " + "store paths of derivations during evaluation.", + .handler = {&readOnlyMode, true}, + }); + } } Strings SourceExprCommand::getDefaultFlakeAttrPaths() @@ -182,6 +203,8 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() void SourceExprCommand::completeInstallable(std::string_view prefix) { if (file) { + completionType = ctAttrs; + evalSettings.pureEval = false; auto state = getEvalState(); Expr *e = state->parseExprFromFile( @@ -210,13 +233,14 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) Value v2; state->autoCallFunction(*autoArgs, v1, v2); - completionType = ctAttrs; - if (v2.type() == nAttrs) { for (auto & i : *v2.attrs) { std::string name = i.name; if (name.find(searchWord) == 0) { - completions->add(i.name); + if (prefix_ == "") + completions->add(name); + else + completions->add(prefix_ + "." + name); } } } @@ -244,10 +268,11 @@ void completeFlakeRefWithFragment( if (hash == std::string::npos) { completeFlakeRef(evalState->store, prefix); } else { + completionType = ctAttrs; + auto fragment = prefix.substr(hash + 1); auto flakeRefS = std::string(prefix.substr(0, hash)); - // FIXME: do tilde expansion. - auto flakeRef = parseFlakeRef(flakeRefS, absPath(".")); + auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags))); @@ -259,8 +284,6 @@ void completeFlakeRefWithFragment( flake. */ attrPathPrefixes.push_back(""); - completionType = ctAttrs; - for (auto & attrPathPrefixS : attrPathPrefixes) { auto attrPathPrefix = parseAttrPath(*evalState, attrPathPrefixS); auto attrPathS = attrPathPrefixS + std::string(fragment); @@ -334,16 +357,16 @@ DerivedPath Installable::toDerivedPath() return std::move(buildables[0]); } -std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> +std::vector<ref<eval_cache::AttrCursor>> Installable::getCursors(EvalState & state) { auto evalCache = std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state, [&]() { return toValue(state).first; }); - return {{evalCache->getRoot(), ""}}; + return {evalCache->getRoot()}; } -std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string> +ref<eval_cache::AttrCursor> Installable::getCursor(EvalState & state) { auto cursors = getCursors(state); @@ -566,43 +589,21 @@ InstallableFlake::InstallableFlake( std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation() { - auto lockedFlake = getLockedFlake(); - - auto cache = openEvalCache(*state, lockedFlake); - auto root = cache->getRoot(); - - Suggestions suggestions; - - for (auto & attrPath : getActualAttrPaths()) { - debug("trying flake output attribute '%s'", attrPath); - - auto attrOrSuggestions = root->findAlongAttrPath( - parseAttrPath(*state, attrPath), - true - ); - - if (!attrOrSuggestions) { - suggestions += attrOrSuggestions.getSuggestions(); - continue; - } - - auto attr = *attrOrSuggestions; + auto attr = getCursor(*state); - if (!attr->isDerivation()) - throw Error("flake output attribute '%s' is not a derivation", attrPath); + auto attrPath = attr->getAttrPathStr(); - auto drvPath = attr->forceDerivation(); + if (!attr->isDerivation()) + throw Error("flake output attribute '%s' is not a derivation", attrPath); - auto drvInfo = DerivationInfo { - std::move(drvPath), - attr->getAttr(state->sOutputName)->getString() - }; + auto drvPath = attr->forceDerivation(); - return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)}; - } + auto drvInfo = DerivationInfo { + std::move(drvPath), + attr->getAttr(state->sOutputName)->getString() + }; - throw Error(suggestions, "flake '%s' does not provide attribute %s", - flakeRef, showAttrPaths(getActualAttrPaths())); + return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; } std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() @@ -614,33 +615,10 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state) { - auto lockedFlake = getLockedFlake(); - - auto vOutputs = getFlakeOutputs(state, *lockedFlake); - - auto emptyArgs = state.allocBindings(0); - - Suggestions suggestions; - - for (auto & attrPath : getActualAttrPaths()) { - try { - auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); - state.forceValue(*v, pos); - return {v, pos}; - } catch (AttrPathNotFound & e) { - suggestions += e.info().suggestions; - } - } - - throw Error( - suggestions, - "flake '%s' does not provide attribute %s", - flakeRef, - showAttrPaths(getActualAttrPaths()) - ); + return {&getCursor(state)->forceValue(), noPos}; } -std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> +std::vector<ref<eval_cache::AttrCursor>> InstallableFlake::getCursors(EvalState & state) { auto evalCache = openEvalCache(state, @@ -648,21 +626,55 @@ InstallableFlake::getCursors(EvalState & state) auto root = evalCache->getRoot(); - std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res; + std::vector<ref<eval_cache::AttrCursor>> res; for (auto & attrPath : getActualAttrPaths()) { auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back({*attr, attrPath}); + if (attr) res.push_back(ref(*attr)); } return res; } +ref<eval_cache::AttrCursor> InstallableFlake::getCursor(EvalState & state) +{ + auto lockedFlake = getLockedFlake(); + + auto cache = openEvalCache(state, lockedFlake); + auto root = cache->getRoot(); + + Suggestions suggestions; + + auto attrPaths = getActualAttrPaths(); + + for (auto & attrPath : attrPaths) { + debug("trying flake output attribute '%s'", attrPath); + + auto attrOrSuggestions = root->findAlongAttrPath( + parseAttrPath(state, attrPath), + true + ); + + if (!attrOrSuggestions) { + suggestions += attrOrSuggestions.getSuggestions(); + continue; + } + + return *attrOrSuggestions; + } + + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(attrPaths)); +} + std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const { - flake::LockFlags lockFlagsApplyConfig = lockFlags; - lockFlagsApplyConfig.applyNixConfig = true; if (!_lockedFlake) { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + lockFlagsApplyConfig.applyNixConfig = true; _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; @@ -687,6 +699,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( { std::vector<std::shared_ptr<Installable>> result; + if (readOnlyMode) { + settings.readOnlyMode = true; + } + if (file || expr) { if (file && expr) throw UsageError("'--file' and '--expr' are exclusive"); @@ -756,55 +772,20 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable( return installables.front(); } -BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths) +BuiltPaths Installable::build( + ref<Store> evalStore, + ref<Store> store, + Realise mode, + const std::vector<std::shared_ptr<Installable>> & installables, + BuildMode bMode) { BuiltPaths res; - for (const auto & b : hopefullyBuiltPaths) - std::visit( - overloaded{ - [&](const DerivedPath::Opaque & bo) { - res.push_back(BuiltPath::Opaque{bo.path}); - }, - [&](const DerivedPath::Built & bfd) { - OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - auto drvOutputs = drv.outputsAndOptPaths(*store); - for (auto & output : bfd.outputs) { - if (!outputHashes.count(output)) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store->printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - auto outputId = - DrvOutput{outputHashes.at(output), output}; - auto realisation = - store->queryRealisation(outputId); - if (!realisation) - throw Error( - "cannot operate on an output of unbuilt " - "content-addressed derivation '%s'", - outputId.to_string()); - outputs.insert_or_assign( - output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - assert(drvOutputs.count(output)); - assert(drvOutputs.at(output).second); - outputs.insert_or_assign( - output, *drvOutputs.at(output).second); - } - } - res.push_back(BuiltPath::Built{bfd.drvPath, outputs}); - }, - }, - b.raw()); - + for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode)) + res.push_back(builtPath); return res; } -BuiltPaths Installable::build( +std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2( ref<Store> evalStore, ref<Store> store, Realise mode, @@ -815,39 +796,93 @@ BuiltPaths Installable::build( settings.readOnlyMode = true; std::vector<DerivedPath> pathsToBuild; + std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap; for (auto & i : installables) { - auto b = i->toDerivedPaths(); - pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); + for (auto b : i->toDerivedPaths()) { + pathsToBuild.push_back(b); + backmap[b].push_back(i); + } } + std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res; + switch (mode) { + case Realise::Nothing: case Realise::Derivation: printMissing(store, pathsToBuild, lvlError); - return getBuiltPaths(evalStore, store, pathsToBuild); + + for (auto & path : pathsToBuild) { + for (auto & installable : backmap[path]) { + std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + OutputPathMap outputs; + auto drv = evalStore->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive + auto drvOutputs = drv.outputsAndOptPaths(*store); + for (auto & output : bfd.outputs) { + if (!outputHashes.count(output)) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store->printStorePath(bfd.drvPath), output); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + DrvOutput outputId { outputHashes.at(output), output }; + auto realisation = store->queryRealisation(outputId); + if (!realisation) + throw Error( + "cannot operate on an output of unbuilt " + "content-addressed derivation '%s'", + outputId.to_string()); + outputs.insert_or_assign(output, realisation->outPath); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + assert(drvOutputs.count(output)); + assert(drvOutputs.at(output).second); + outputs.insert_or_assign( + output, *drvOutputs.at(output).second); + } + } + res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); + }, + [&](const DerivedPath::Opaque & bo) { + res.push_back({installable, BuiltPath::Opaque { bo.path }}); + }, + }, path.raw()); + } + } + + break; + case Realise::Outputs: { - BuiltPaths res; for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { if (!buildResult.success()) buildResult.rethrow(); - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - std::map<std::string, StorePath> outputs; - for (auto & path : buildResult.builtOutputs) - outputs.emplace(path.first.outputName, path.second.outPath); - res.push_back(BuiltPath::Built { bfd.drvPath, outputs }); - }, - [&](const DerivedPath::Opaque & bo) { - res.push_back(BuiltPath::Opaque { bo.path }); - }, - }, buildResult.path.raw()); + + for (auto & installable : backmap[buildResult.path]) { + std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + std::map<std::string, StorePath> outputs; + for (auto & path : buildResult.builtOutputs) + outputs.emplace(path.first.outputName, path.second.outPath); + res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); + }, + [&](const DerivedPath::Opaque & bo) { + res.push_back({installable, BuiltPath::Opaque { bo.path }}); + }, + }, buildResult.path.raw()); + } } - return res; + + break; } + default: assert(false); } + + return res; } BuiltPaths Installable::toBuiltPaths( @@ -935,7 +970,7 @@ InstallablesCommand::InstallablesCommand() void InstallablesCommand::prepare() { if (_installables.empty() && useDefaultInstallables()) - // FIXME: commands like "nix install" should not have a + // FIXME: commands like "nix profile install" should not have a // default, probably. _installables.push_back("."); installables = parseInstallables(getStore(), _installables); @@ -945,13 +980,14 @@ std::optional<FlakeRef> InstallablesCommand::getFlakeRefForCompletion() { if (_installables.empty()) { if (useDefaultInstallables()) - return parseFlakeRef(".", absPath(".")); + return parseFlakeRefWithFragment(".", absPath(".")).first; return {}; } - return parseFlakeRef(_installables.front(), absPath(".")); + return parseFlakeRefWithFragment(_installables.front(), absPath(".")).first; } -InstallableCommand::InstallableCommand() +InstallableCommand::InstallableCommand(bool supportReadOnlyMode) + : SourceExprCommand(supportReadOnlyMode) { expectArgs({ .label = "installable", diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index e172b71b0..b847f8939 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -80,10 +80,10 @@ struct Installable return {}; } - virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> + virtual std::vector<ref<eval_cache::AttrCursor>> getCursors(EvalState & state); - std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string> + virtual ref<eval_cache::AttrCursor> getCursor(EvalState & state); virtual FlakeRef nixpkgsFlakeRef() const @@ -98,6 +98,13 @@ struct Installable const std::vector<std::shared_ptr<Installable>> & installables, BuildMode bMode = bmNormal); + static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2( + ref<Store> evalStore, + ref<Store> store, + Realise mode, + const std::vector<std::shared_ptr<Installable>> & installables, + BuildMode bMode = bmNormal); + static std::set<StorePath> toStorePaths( ref<Store> evalStore, ref<Store> store, @@ -173,9 +180,15 @@ struct InstallableFlake : InstallableValue std::pair<Value *, Pos> toValue(EvalState & state) override; - std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> + /* Get a cursor to every attrpath in getActualAttrPaths() that + exists. */ + std::vector<ref<eval_cache::AttrCursor>> getCursors(EvalState & state) override; + /* Get a cursor to the first attrpath in getActualAttrPaths() that + exists, or throw an exception with suggestions if none exists. */ + ref<eval_cache::AttrCursor> getCursor(EvalState & state) override; + std::shared_ptr<flake::LockedFlake> getLockedFlake() const; FlakeRef nixpkgsFlakeRef() const override; @@ -185,9 +198,4 @@ ref<eval_cache::EvalCache> openEvalCache( EvalState & state, std::shared_ptr<flake::LockedFlake> lockedFlake); -BuiltPaths getBuiltPaths( - ref<Store> evalStore, - ref<Store> store, - const DerivedPaths & hopefullyBuiltPaths); - } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 54fa9b741..7d3fd01a4 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -306,9 +306,9 @@ Value * EvalCache::getRootValue() return *value; } -std::shared_ptr<AttrCursor> EvalCache::getRoot() +ref<AttrCursor> EvalCache::getRoot() { - return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt); + return make_ref<AttrCursor>(ref(shared_from_this()), std::nullopt); } AttrCursor::AttrCursor( diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index c9a9bf471..b0709ebc2 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -33,7 +33,7 @@ public: EvalState & state, RootLoader rootLoader); - std::shared_ptr<AttrCursor> getRoot(); + ref<AttrCursor> getRoot(); }; enum AttrType { @@ -104,6 +104,8 @@ public: ref<AttrCursor> getAttr(std::string_view name); + /* Get an attribute along a chain of attrsets. Note that this does + not auto-call functors or functions. */ OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); std::string getString(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 437c7fc53..b87e06ef5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -436,6 +436,7 @@ EvalState::EvalState( , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) , sContentAddressed(symbols.create("__contentAddressed")) + , sImpure(symbols.create("__impure")) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e7915dd99..7ed376e8d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -78,7 +78,7 @@ public: sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, - sContentAddressed, + sContentAddressed, sImpure, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index eca9d6358..f7b77fccb 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -989,9 +989,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; bool contentAddressed = false; + bool isImpure = false; std::optional<std::string> outputHash; std::string outputHashAlgo; - auto ingestionMethod = FileIngestionMethod::Flat; + std::optional<FileIngestionMethod> ingestionMethod; StringSet outputs; outputs.insert("out"); @@ -1051,6 +1052,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * settings.requireExperimentalFeature(Xp::CaDerivations); } + else if (i->name == state.sImpure) { + isImpure = state.forceBool(*i->value, pos); + if (isImpure) + settings.requireExperimentalFeature(Xp::ImpureDerivations); + } + /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { @@ -1183,12 +1190,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * .errPos = posDrvName }); - std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); - Hash h = newHashAllowEmpty(*outputHash, ht); + auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); + auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { { - .method = ingestionMethod, + .method = method, .hash = h, }, {}, @@ -1197,21 +1204,36 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drv.outputs.insert_or_assign("out", DerivationOutput::CAFixed { .hash = FixedOutputHash { - .method = ingestionMethod, + .method = method, .hash = std::move(h), }, }); } - else if (contentAddressed) { - HashType ht = parseHashType(outputHashAlgo); + else if (contentAddressed || isImpure) { + if (contentAddressed && isImpure) + throw EvalError({ + .msg = hintfmt("derivation cannot be both content-addressed and impure"), + .errPos = posDrvName + }); + + auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256); + auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); + for (auto & i : outputs) { drv.env[i] = hashPlaceholder(i); - drv.outputs.insert_or_assign(i, - DerivationOutput::CAFloating { - .method = ingestionMethod, - .hashType = ht, - }); + if (isImpure) + drv.outputs.insert_or_assign(i, + DerivationOutput::Impure { + .method = method, + .hashType = ht, + }); + else + drv.outputs.insert_or_assign(i, + DerivationOutput::CAFloating { + .method = method, + .hashType = ht, + }); } } @@ -1228,34 +1250,26 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * DerivationOutput::Deferred { }); } - // Regular, non-CA derivation should always return a single hash and not - // hash per output. - auto hashModulo = hashDerivationModulo(*state.store, drv, true); - std::visit(overloaded { - [&](const DrvHash & drvHash) { - auto & h = drvHash.hash; - switch (drvHash.kind) { - case DrvHash::Kind::Deferred: - /* Outputs already deferred, nothing to do */ - break; - case DrvHash::Kind::Regular: - for (auto & [outputName, output] : drv.outputs) { - auto outPath = state.store->makeOutputPath(outputName, h, drvName); - drv.env[outputName] = state.store->printStorePath(outPath); - output = DerivationOutput::InputAddressed { - .path = std::move(outPath), - }; - } - break; - } - }, - [&](const CaOutputHashes &) { - // Shouldn't happen as the toplevel derivation is not CA. - assert(false); - }, - }, - hashModulo.raw()); - + auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); + switch (hashModulo.kind) { + case DrvHash::Kind::Regular: + for (auto & i : outputs) { + auto h = hashModulo.hashes.at(i); + auto outPath = state.store->makeOutputPath(i, h, drvName); + drv.env[i] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign( + i, + DerivationOutputInputAddressed { + .path = std::move(outPath), + }); + } + break; + ; + case DrvHash::Kind::Deferred: + for (auto & i : outputs) { + drv.outputs.insert_or_assign(i, DerivationOutputDeferred {}); + } + } } /* Write the resulting term into the Nix store directory. */ diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index efeb93daf..821eba698 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -61,6 +61,12 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); + if (!parsedURL.query.empty()) + throw Error({ + .msg = hintfmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), + .errPos = pos + }); + auto fromStore = openStore(parsedURL.to_string()); if (toCA) { @@ -87,7 +93,8 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args }); } } else { - copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); + if (!state.store->isValidPath(*fromPath)) + copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); toPath = fromPath; } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 517da4c01..7b35abca2 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -84,7 +84,8 @@ void printValueAsJSON(EvalState & state, bool strict, .msg = hintfmt("cannot convert %1% to JSON", showType(v)), .errPos = v.determinePos(pos) }); - throw e.addTrace(pos, hintfmt("message for the trace")); + e.addTrace(pos, hintfmt("message for the trace")); + throw e; } } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 4c457966d..2a1529a60 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -244,9 +244,18 @@ std::optional<std::string> Input::getRef() const std::optional<Hash> Input::getRev() const { - if (auto s = maybeGetStrAttr(attrs, "rev")) - return Hash::parseAny(*s, htSHA1); - return {}; + std::optional<Hash> hash = {}; + + if (auto s = maybeGetStrAttr(attrs, "rev")) { + try { + hash = Hash::parseAnyPrefixed(*s); + } catch (BadHash &e) { + // Default to sha1 for backwards compatibility with existing flakes + hash = Hash::parseAny(*s, htSHA1); + } + } + + return hash; } std::optional<uint64_t> Input::getRevCount() const diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d75c5d3ae..34b1342a0 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -28,9 +28,7 @@ static std::string readHead(const Path & path) static bool isNotDotGitDirectory(const Path & path) { - static const std::regex gitDirRegex("^(?:.*/)?\\.git$"); - - return not std::regex_match(path, gitDirRegex); + return baseNameOf(path) != ".git"; } struct GitInputScheme : InputScheme @@ -189,8 +187,16 @@ struct GitInputScheme : InputScheme if (submodules) cacheType += "-submodules"; if (allRefs) cacheType += "-all-refs"; + auto checkHashType = [&](const std::optional<Hash> & hash) + { + if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); + }; + auto getLockedAttrs = [&]() { + checkHashType(input.getRev()); + return Attrs({ {"type", cacheType}, {"name", name}, @@ -285,9 +291,11 @@ struct GitInputScheme : InputScheme auto files = tokenizeString<std::set<std::string>>( runProgram("git", true, gitOpts), "\0"s); + Path actualPath(absPath(actualUrl)); + PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualUrl)); - std::string file(p, actualUrl.size() + 1); + assert(hasPrefix(p, actualPath)); + std::string file(p, actualPath.size() + 1); auto st = lstat(p); @@ -300,13 +308,13 @@ struct GitInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); // FIXME: maybe we should use the timestamp of the last // modified dirty file? input.attrs.insert_or_assign( "lastModified", - hasHead ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return {std::move(storePath), input}; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 8b82e9daa..1beb8b944 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -178,9 +178,11 @@ struct MercurialInputScheme : InputScheme auto files = tokenizeString<std::set<std::string>>( runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + Path actualPath(absPath(actualUrl)); + PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualUrl)); - std::string file(p, actualUrl.size() + 1); + assert(hasPrefix(p, actualPath)); + std::string file(p, actualPath.size() + 1); auto st = lstat(p); @@ -193,7 +195,7 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); return {std::move(storePath), input}; } @@ -201,8 +203,17 @@ struct MercurialInputScheme : InputScheme if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); + auto checkHashType = [&](const std::optional<Hash> & hash) + { + if (hash.has_value() && hash->type != htSHA1) + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); + }; + + auto getLockedAttrs = [&]() { + checkHashType(input.getRev()); + return Attrs({ {"type", "hg"}, {"name", name}, diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index cb6d19b8e..24fb1f763 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -1,6 +1,7 @@ #pragma once #include "realisation.hh" +#include "derived-path.hh" #include <string> #include <chrono> @@ -30,6 +31,8 @@ struct BuildResult ResolvesToAlreadyValid, NoSubstituters, } status = MiscFailure; + + // FIXME: include entire ErrorInfo object. std::string errorMsg; std::string toString() const { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 3d1c4fbc1..53f212c1d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -204,9 +204,34 @@ void DerivationGoal::haveDerivation() { trace("have derivation"); + parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + if (!drv->type().hasKnownOutputPaths()) settings.requireExperimentalFeature(Xp::CaDerivations); + if (!drv->type().isPure()) { + settings.requireExperimentalFeature(Xp::ImpureDerivations); + + for (auto & [outputName, output] : drv->outputs) { + auto randomPath = StorePath::random(outputPathName(drv->name, outputName)); + assert(!worker.store.isValidPath(randomPath)); + initialOutputs.insert({ + outputName, + InitialOutput { + .wanted = true, + .outputHash = impureOutputHash, + .known = InitialOutputStatus { + .path = randomPath, + .status = PathStatus::Absent + } + } + }); + } + + gaveUpOnSubstitution(); + return; + } + for (auto & i : drv->outputsAndOptPaths(worker.store)) if (i.second.second) worker.store.addTempRoot(*i.second.second); @@ -230,9 +255,6 @@ void DerivationGoal::haveDerivation() return; } - parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); - - /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ @@ -266,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); + assert(drv->type().isPure()); + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { done(BuildResult::TransientFailure, {}, Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", @@ -315,9 +339,21 @@ void DerivationGoal::outputsSubstitutionTried() void DerivationGoal::gaveUpOnSubstitution() { /* The inputs must be built before we can build this goal. */ + inputDrvOutputs.clear(); if (useDerivation) - for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) + for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) { + /* Ensure that pure, non-fixed-output derivations don't + depend on impure derivations. */ + if (drv->type().isPure() && !drv->type().isFixed()) { + auto inputDrv = worker.evalStore.readDerivation(i.first); + if (!inputDrv.type().isPure()) + throw Error("pure derivation '%s' depends on impure derivation '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(i.first)); + } + addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + } /* Copy the input sources from the eval store to the build store. */ @@ -345,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution() void DerivationGoal::repairClosure() { + assert(drv->type().isPure()); + /* If we're repairing, we now know that our own outputs are valid. Now check whether the other paths in the outputs closure are good. If not, then start derivation goals for the derivations @@ -452,22 +490,24 @@ void DerivationGoal::inputsRealised() drvs. */ : true); }, + [&](const DerivationType::Impure &) { + return true; + } }, drvType.raw()); - if (resolveDrv) - { + if (resolveDrv && !fullDrv.inputDrvs.empty()) { settings.requireExperimentalFeature(Xp::CaDerivations); /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); + now-known results of dependencies. If so, we become a + stub goal aliasing that resolved derivation goal. */ + std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs); assert(attempt); Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); - auto msg = fmt("Resolved derivation: '%s' -> '%s'", + auto msg = fmt("resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), worker.store.printStorePath(pathResolved)); act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg, @@ -488,21 +528,13 @@ void DerivationGoal::inputsRealised() /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - assert(worker.evalStore.isValidPath(drvPath)); - auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else + for (auto & j : wantedDepOutputs) + if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) + worker.store.computeFSClosure(*outPath, inputPaths); + else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); - } } } @@ -923,7 +955,7 @@ void DerivationGoal::buildDone() st = dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - derivationType.isImpure() || diskFull ? BuildResult::TransientFailure : + !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } @@ -934,60 +966,53 @@ void DerivationGoal::buildDone() void DerivationGoal::resolvedFinished() { + trace("resolved derivation finished"); + assert(resolvedDrvGoal); auto resolvedDrv = *resolvedDrvGoal->drv; + auto & resolvedResult = resolvedDrvGoal->buildResult; - auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); + DrvOutputs builtOutputs; - StorePathSet outputPaths; + if (resolvedResult.success()) { + auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); - // `wantedOutputs` might be empty, which means “all the outputs” - auto realWantedOutputs = wantedOutputs; - if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv.outputNames(); + StorePathSet outputPaths; - DrvOutputs builtOutputs; + // `wantedOutputs` might be empty, which means “all the outputs” + auto realWantedOutputs = wantedOutputs; + if (realWantedOutputs.empty()) + realWantedOutputs = resolvedDrv.outputNames(); + + for (auto & wantedOutput : realWantedOutputs) { + assert(initialOutputs.count(wantedOutput) != 0); + assert(resolvedHashes.count(wantedOutput) != 0); + auto realisation = resolvedResult.builtOutputs.at( + DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput }); + if (drv->type().isPure()) { + auto newRealisation = realisation; + newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput }; + newRealisation.signatures.clear(); + if (!drv->type().isFixed()) + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); + signRealisation(newRealisation); + worker.store.registerDrvOutput(newRealisation); + } + outputPaths.insert(realisation.outPath); + builtOutputs.emplace(realisation.id, realisation); + } - for (auto & wantedOutput : realWantedOutputs) { - assert(initialOutputs.count(wantedOutput) != 0); - assert(resolvedHashes.count(wantedOutput) != 0); - auto realisation = worker.store.queryRealisation( - DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths ); - // We've just built it, but maybe the build failed, in which case the - // realisation won't be there - if (realisation) { - auto newRealisation = *realisation; - newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; - newRealisation.signatures.clear(); - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); - signRealisation(newRealisation); - worker.store.registerDrvOutput(newRealisation); - outputPaths.insert(realisation->outPath); - builtOutputs.emplace(realisation->id, *realisation); - } else { - // If we don't have a realisation, then it must mean that something - // failed when building the resolved drv - assert(!buildResult.success()); - } } - runPostBuildHook( - worker.store, - *logger, - drvPath, - outputPaths - ); - - auto status = [&]() { - auto & resolvedResult = resolvedDrvGoal->buildResult; - switch (resolvedResult.status) { - case BuildResult::AlreadyValid: - return BuildResult::ResolvesToAlreadyValid; - default: - return resolvedResult.status; - } - }(); + auto status = resolvedResult.status; + if (status == BuildResult::AlreadyValid) + status = BuildResult::ResolvesToAlreadyValid; done(status, std::move(builtOutputs)); } @@ -1236,6 +1261,7 @@ void DerivationGoal::flushLine() std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap() { + assert(drv->type().isPure()); if (!useDerivation || drv->type().hasKnownOutputPaths()) { std::map<std::string, std::optional<StorePath>> res; for (auto & [name, output] : drv->outputs) @@ -1248,6 +1274,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri OutputPathMap DerivationGoal::queryDerivationOutputMap() { + assert(drv->type().isPure()); if (!useDerivation || drv->type().hasKnownOutputPaths()) { OutputPathMap res; for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) @@ -1261,6 +1288,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() { + if (!drv->type().isPure()) return { false, {} }; + bool checkHash = buildMode == bmRepair; auto wantedOutputsLeft = wantedOutputs; DrvOutputs validOutputs; @@ -1304,6 +1333,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() if (info.wanted && info.known && info.known->isValid()) validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); } + // If we requested all the outputs via the empty set, we are always fine. // If we requested specific elements, the loop above removes all the valid // ones, so any that are left must be invalid. @@ -1341,9 +1371,7 @@ void DerivationGoal::done( { buildResult.status = status; if (ex) - // FIXME: strip: "error: " - buildResult.errorMsg = ex->what(); - amDone(buildResult.success() ? ecSuccess : ecFailed, ex); + buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg)); if (buildResult.status == BuildResult::TimedOut) worker.timedOut = true; if (buildResult.status == BuildResult::PermanentFailure) @@ -1370,7 +1398,21 @@ void DerivationGoal::done( fs.open(traceBuiltOutputsFile, std::fstream::out); fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } + + amDone(buildResult.success() ? ecSuccess : ecFailed, ex); } +void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) +{ + Goal::waiteeDone(waitee, result); + + if (waitee->buildResult.success()) + if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path)) + for (auto & [output, realisation] : waitee->buildResult.builtOutputs) + inputDrvOutputs.insert_or_assign( + { bfd->drvPath, output.outputName }, + realisation.outPath); +} + } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index f556b6f25..2d8bfd592 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -57,6 +57,11 @@ struct DerivationGoal : public Goal them. */ StringSet wantedOutputs; + /* Mapping from input derivations + output names to actual store + paths. This is filled in by waiteeDone() as each dependency + finishes, before inputsRealised() is reached, */ + std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs; + /* Whether additional wanted outputs have been added. */ bool needRestart = false; @@ -224,6 +229,8 @@ struct DerivationGoal : public Goal DrvOutputs builtOutputs = {}, std::optional<Error> ex = {}); + void waiteeDone(GoalPtr waitee, ExitCode result) override; + StorePathSet exportReferences(const StorePathSet & storePaths); }; diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index e50292c1e..b7f7b5ab1 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -41,7 +41,7 @@ void DrvOutputSubstitutionGoal::tryNext() if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ - debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string()); + debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 008ce0050..1467c18af 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationType.isImpure()) && !noChroot; + useChroot = derivationType.isSandboxed() && !noChroot; } auto & localStore = getLocalStore(); @@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder() "nogroup:x:65534:\n", sandboxGid())); /* Create /etc/hosts with localhost entry. */ - if (!(derivationType.isImpure())) + if (derivationType.isSandboxed()) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -704,6 +704,9 @@ void LocalDerivationGoal::startBuilder() /* Run the builder. */ printMsg(lvlChatty, "executing builder '%1%'", drv->builder); + printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args)); + for (auto & i : drv->env) + printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); /* Create the log file. */ Path logFile = openLogFile(); @@ -796,7 +799,7 @@ void LocalDerivationGoal::startBuilder() us. */ - if (!(derivationType.isImpure())) + if (derivationType.isSandboxed()) privateNetwork = true; userNamespaceSync.create(); @@ -1060,7 +1063,7 @@ void LocalDerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (derivationType.isImpure()) { + if (!derivationType.isSandboxed()) { for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) env[i] = getEnv(i).value_or(""); } @@ -1674,7 +1677,7 @@ void LocalDerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (derivationType.isImpure()) { + if (!derivationType.isSandboxed()) { // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the @@ -1918,7 +1921,7 @@ void LocalDerivationGoal::runChild() sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - if (derivationType.isImpure()) + if (!derivationType.isSandboxed()) sandboxProfile += "(import \"sandbox-network.sb\")\n"; /* Add the output paths we'll use at build-time to the chroot */ @@ -2390,6 +2393,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs() assert(false); }, + [&](const DerivationOutput::Impure & doi) { + return newInfoFromCA(DerivationOutput::CAFloating { + .method = doi.method, + .hashType = doi.hashType, + }); + }, + }, output.raw()); /* FIXME: set proper permissions in restorePath() so @@ -2600,7 +2610,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs() }, .outPath = newInfo.path }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + && drv->type().isPure()) + { signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 2aaa89a57..2aee09f21 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -24,9 +24,16 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } -void PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status) +void PathSubstitutionGoal::done( + ExitCode result, + BuildResult::Status status, + std::optional<std::string> errorMsg) { buildResult.status = status; + if (errorMsg) { + debug(*errorMsg); + buildResult.errorMsg = *errorMsg; + } amDone(result); } @@ -67,12 +74,14 @@ void PathSubstitutionGoal::tryNext() if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - done(substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters); + done( + substituterFailed ? ecFailed : ecNoSubstituters, + BuildResult::NoSubstituters, + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); if (substituterFailed) { worker.failedSubstitutions++; @@ -171,10 +180,10 @@ void PathSubstitutionGoal::referencesValid() trace("all references realised"); if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); done( nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, - BuildResult::DependencyFailed); + BuildResult::DependencyFailed, + fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); return; } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 946f13841..a73f8e666 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -53,7 +53,10 @@ struct PathSubstitutionGoal : public Goal /* Content address for recomputing store path */ std::optional<ContentAddress> ca; - void done(ExitCode result, BuildResult::Status status); + void done( + ExitCode result, + BuildResult::Status status, + std::optional<std::string> errorMsg = {}); public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1b64819f6..08d5e5c5e 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -25,26 +25,42 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string [](const DerivationOutput::Deferred &) -> std::optional<StorePath> { return std::nullopt; }, + [](const DerivationOutput::Impure &) -> std::optional<StorePath> { + return std::nullopt; + }, }, raw()); } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { +StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const +{ return store.makeFixedOutputPath( outputPathName(drvName, outputName), { hash, {} }); } -bool DerivationType::isCA() const { +bool DerivationType::isCA() const +{ /* Normally we do the full `std::visit` to make sure we have exhaustively handled all variants, but so long as there is a variant called `ContentAddressed`, it must be the only one for which `isCA` is true for this to make sense!. */ - return std::holds_alternative<ContentAddressed>(raw()); + return std::visit(overloaded { + [](const InputAddressed & ia) { + return false; + }, + [](const ContentAddressed & ca) { + return true; + }, + [](const Impure &) { + return true; + }, + }, raw()); } -bool DerivationType::isFixed() const { +bool DerivationType::isFixed() const +{ return std::visit(overloaded { [](const InputAddressed & ia) { return false; @@ -52,10 +68,14 @@ bool DerivationType::isFixed() const { [](const ContentAddressed & ca) { return ca.fixed; }, + [](const Impure &) { + return false; + }, }, raw()); } -bool DerivationType::hasKnownOutputPaths() const { +bool DerivationType::hasKnownOutputPaths() const +{ return std::visit(overloaded { [](const InputAddressed & ia) { return !ia.deferred; @@ -63,17 +83,40 @@ bool DerivationType::hasKnownOutputPaths() const { [](const ContentAddressed & ca) { return ca.fixed; }, + [](const Impure &) { + return false; + }, }, raw()); } -bool DerivationType::isImpure() const { +bool DerivationType::isSandboxed() const +{ return std::visit(overloaded { [](const InputAddressed & ia) { + return true; + }, + [](const ContentAddressed & ca) { + return ca.sandboxed; + }, + [](const Impure &) { return false; }, + }, raw()); +} + + +bool DerivationType::isPure() const +{ + return std::visit(overloaded { + [](const InputAddressed & ia) { + return true; + }, [](const ContentAddressed & ca) { - return !ca.pure; + return true; + }, + [](const Impure &) { + return false; }, }, raw()); } @@ -176,7 +219,14 @@ static DerivationOutput parseDerivationOutput(const Store & store, hashAlgo = hashAlgo.substr(2); } const auto hashType = parseHashType(hashAlgo); - if (hash != "") { + if (hash == "impure") { + settings.requireExperimentalFeature(Xp::ImpureDerivations); + assert(pathS == ""); + return DerivationOutput::Impure { + .method = std::move(method), + .hashType = std::move(hashType), + }; + } else if (hash != "") { validatePath(pathS); return DerivationOutput::CAFixed { .hash = FixedOutputHash { @@ -345,6 +395,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); + }, + [&](const DerivationOutputImpure & doi) { + // FIXME + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); + s += ','; printUnquotedString(s, "impure"); } }, i.second.raw()); s += ')'; @@ -410,8 +466,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName DerivationType BasicDerivation::type() const { - std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs; + std::set<std::string_view> + inputAddressedOutputs, + fixedCAOutputs, + floatingCAOutputs, + deferredIAOutputs, + impureOutputs; std::optional<HashType> floatingHashType; + for (auto & i : outputs) { std::visit(overloaded { [&](const DerivationOutput::InputAddressed &) { @@ -426,43 +488,78 @@ DerivationType BasicDerivation::type() const floatingHashType = dof.hashType; } else { if (*floatingHashType != dof.hashType) - throw Error("All floating outputs must use the same hash type"); + throw Error("all floating outputs must use the same hash type"); } }, [&](const DerivationOutput::Deferred &) { - deferredIAOutputs.insert(i.first); + deferredIAOutputs.insert(i.first); + }, + [&](const DerivationOutput::Impure &) { + impureOutputs.insert(i.first); }, }, i.second.raw()); } - if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { - throw Error("Must have at least one output"); - } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) + throw Error("must have at least one output"); + + if (!inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) return DerivationType::InputAddressed { .deferred = false, }; - } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { + + if (inputAddressedOutputs.empty() + && !fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) + { if (fixedCAOutputs.size() > 1) // FIXME: Experimental feature? - throw Error("Only one fixed output is allowed for now"); + throw Error("only one fixed output is allowed for now"); if (*fixedCAOutputs.begin() != "out") - throw Error("Single fixed output must be named \"out\""); + throw Error("single fixed output must be named \"out\""); return DerivationType::ContentAddressed { - .pure = false, + .sandboxed = false, .fixed = true, }; - } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) { + } + + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && !floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) return DerivationType::ContentAddressed { - .pure = true, + .sandboxed = true, .fixed = false, }; - } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) { + + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && !deferredIAOutputs.empty() + && impureOutputs.empty()) return DerivationType::InputAddressed { .deferred = true, }; - } else { - throw Error("Can't mix derivation output types"); - } + + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && !impureOutputs.empty()) + return DerivationType::Impure { }; + + throw Error("can't mix derivation output types"); } @@ -474,7 +571,7 @@ Sync<DrvHashes> drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) +static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) { { auto hashes = drvHashes.lock(); @@ -509,7 +606,7 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { auto type = drv.type(); @@ -524,7 +621,20 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return outputHashes; + return DrvHash { + .hashes = outputHashes, + .kind = DrvHash::Kind::Regular, + }; + } + + if (!type.isPure()) { + std::map<std::string, Hash> outputHashes; + for (const auto & [outputName, _] : drv.outputs) + outputHashes.insert_or_assign(outputName, impureOutputHash); + return DrvHash { + .hashes = outputHashes, + .kind = DrvHash::Kind::Deferred, + }; } auto kind = std::visit(overloaded { @@ -538,67 +648,41 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m ? DrvHash::Kind::Regular : DrvHash::Kind::Deferred; }, + [](const DerivationType::Impure &) -> DrvHash::Kind { + assert(false); + } }, drv.type().raw()); - /* For other derivations, replace the inputs paths with recursive - calls to this function. */ std::map<std::string, StringSet> inputs2; for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { // Avoid lambda capture restriction with standard / Clang auto & inputOutputs = inputOutputs0; const auto & res = pathDerivationModulo(store, drvPath); - std::visit(overloaded { - // Regular non-CA derivation, replace derivation - [&](const DrvHash & drvHash) { - kind |= drvHash.kind; - inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs); - }, - // CA derivation's output hashes - [&](const CaOutputHashes & outputHashes) { - std::set<std::string> justOut = { "out" }; - for (auto & output : inputOutputs) { - /* Put each one in with a single "out" output.. */ - const auto h = outputHashes.at(output); - inputs2.insert_or_assign( - h.to_string(Base16, false), - justOut); - } - }, - }, res.raw()); + if (res.kind == DrvHash::Kind::Deferred) + kind = DrvHash::Kind::Deferred; + for (auto & outputName : inputOutputs) { + const auto h = res.hashes.at(outputName); + inputs2[h.to_string(Base16, false)].insert(outputName); + } } auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); - return DrvHash { .hash = hash, .kind = kind }; -} - - -void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept -{ - switch (other) { - case DrvHash::Kind::Regular: - break; - case DrvHash::Kind::Deferred: - self = other; - break; + std::map<std::string, Hash> outputHashes; + for (const auto & [outputName, _] : drv.outputs) { + outputHashes.insert_or_assign(outputName, hash); } + + return DrvHash { + .hashes = outputHashes, + .kind = kind, + }; } std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv) { - std::map<std::string, Hash> res; - std::visit(overloaded { - [&](const DrvHash & drvHash) { - for (auto & outputName : drv.outputNames()) { - res.insert({outputName, drvHash.hash}); - } - }, - [&](const CaOutputHashes & outputHashes) { - res = outputHashes; - }, - }, hashDerivationModulo(store, drv, true).raw()); - return res; + return hashDerivationModulo(store, drv, true).hashes; } @@ -625,7 +709,8 @@ StringSet BasicDerivation::outputNames() const return names; } -DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const +{ DerivationOutputsAndOptPaths outsAndOptPaths; for (auto output : outputs) outsAndOptPaths.insert(std::make_pair( @@ -636,7 +721,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s return outsAndOptPaths; } -std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { +std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) +{ auto nameWithSuffix = drvPath.name(); constexpr std::string_view extension = ".drv"; assert(hasSuffix(nameWithSuffix, extension)); @@ -698,6 +784,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr << "" << ""; }, + [&](const DerivationOutput::Impure & doi) { + out << "" + << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)) + << "impure"; + }, }, i.second.raw()); } worker_proto::write(store, out, drv.inputSrcs); @@ -723,21 +814,19 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath } -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { - - debug("Rewriting the derivation"); - - for (auto &rewrite: rewrites) { +static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) +{ + for (auto & rewrite : rewrites) { debug("rewriting %s as %s", rewrite.first, rewrite.second); } drv.builder = rewriteStrings(drv.builder, rewrites); - for (auto & arg: drv.args) { + for (auto & arg : drv.args) { arg = rewriteStrings(arg, rewrites); } StringPairs newEnv; - for (auto & envVar: drv.env) { + for (auto & envVar : drv.env) { auto envName = rewriteStrings(envVar.first, rewrites); auto envValue = rewriteStrings(envVar.second, rewrites); newEnv.emplace(envName, envValue); @@ -747,7 +836,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) { - auto & h = hashModulo.requireNoFixedNonDeferred(); + auto & h = hashModulo.hashes.at(outputName); auto outPath = store.makeOutputPath(outputName, h, drv.name); drv.env[outputName] = store.printStorePath(outPath); output = DerivationOutput::InputAddressed { @@ -758,55 +847,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -const Hash & DrvHashModulo::requireNoFixedNonDeferred() const { - auto * drvHashOpt = std::get_if<DrvHash>(&raw()); - assert(drvHashOpt); - assert(drvHashOpt->kind == DrvHash::Kind::Regular); - return drvHashOpt->hash; -} - -static bool tryResolveInput( - Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, - const StorePath & inputDrv, const StringSet & inputOutputs) +std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const { - auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv); - - auto getOutput = [&](const std::string & outputName) { - auto & actualPathOpt = inputDrvOutputs.at(outputName); - if (!actualPathOpt) - warn("output %s of input %s missing, aborting the resolving", - outputName, - store.printStorePath(inputDrv) - ); - return actualPathOpt; - }; + std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs; - for (auto & outputName : inputOutputs) { - auto actualPathOpt = getOutput(outputName); - if (!actualPathOpt) return false; - auto actualPath = *actualPathOpt; - inputRewrites.emplace( - downstreamPlaceholder(store, inputDrv, outputName), - store.printStorePath(actualPath)); - inputSrcs.insert(std::move(actualPath)); - } + for (auto & input : inputDrvs) + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first)) + if (outputPath) + inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath); - return true; + return tryResolve(store, inputDrvOutputs); } -std::optional<BasicDerivation> Derivation::tryResolve(Store & store) { +std::optional<BasicDerivation> Derivation::tryResolve( + Store & store, + const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const +{ BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & [inputDrv, inputOutputs] : inputDrvs) - if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs)) - return std::nullopt; + for (auto & [inputDrv, inputOutputs] : inputDrvs) { + for (auto & outputName : inputOutputs) { + if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { + inputRewrites.emplace( + downstreamPlaceholder(store, inputDrv, outputName), + store.printStorePath(*actualPath)); + resolved.inputSrcs.insert(*actualPath); + } else { + warn("output '%s' of input '%s' missing, aborting the resolving", + outputName, + store.printStorePath(inputDrv)); + return {}; + } + } + } rewriteDerivation(store, resolved, inputRewrites); return resolved; } +const Hash impureOutputHash = hashString(htSHA256, "impure"); + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 8dea90abf..af198a767 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -41,15 +41,26 @@ struct DerivationOutputCAFloating }; /* Input-addressed output which depends on a (CA) derivation whose hash isn't - * known atm + * known yet. */ struct DerivationOutputDeferred {}; +/* Impure output which is moved to a content-addressed location (like + CAFloating) but isn't registered as a realization. + */ +struct DerivationOutputImpure +{ + /* information used for expected hash computation */ + FileIngestionMethod method; + HashType hashType; +}; + typedef std::variant< DerivationOutputInputAddressed, DerivationOutputCAFixed, DerivationOutputCAFloating, - DerivationOutputDeferred + DerivationOutputDeferred, + DerivationOutputImpure > _DerivationOutputRaw; struct DerivationOutput : _DerivationOutputRaw @@ -61,6 +72,7 @@ struct DerivationOutput : _DerivationOutputRaw using CAFixed = DerivationOutputCAFixed; using CAFloating = DerivationOutputCAFloating; using Deferred = DerivationOutputDeferred; + using Impure = DerivationOutputImpure; /* Note, when you use this function you should make sure that you're passing the right derivation name. When in doubt, you should use the safer @@ -90,13 +102,17 @@ struct DerivationType_InputAddressed { }; struct DerivationType_ContentAddressed { - bool pure; + bool sandboxed; bool fixed; }; +struct DerivationType_Impure { +}; + typedef std::variant< DerivationType_InputAddressed, - DerivationType_ContentAddressed + DerivationType_ContentAddressed, + DerivationType_Impure > _DerivationTypeRaw; struct DerivationType : _DerivationTypeRaw { @@ -104,7 +120,7 @@ struct DerivationType : _DerivationTypeRaw { using Raw::Raw; using InputAddressed = DerivationType_InputAddressed; using ContentAddressed = DerivationType_ContentAddressed; - + using Impure = DerivationType_Impure; /* Do the outputs of the derivation have paths calculated from their content, or from the derivation itself? */ @@ -114,10 +130,18 @@ struct DerivationType : _DerivationTypeRaw { non-CA derivations. */ bool isFixed() const; - /* Is the derivation impure and needs to access non-deterministic resources, or - pure and can be sandboxed? Note that whether or not we actually sandbox the - derivation is controlled separately. Never true for non-CA derivations. */ - bool isImpure() const; + /* Whether the derivation is fully sandboxed. If false, the + sandbox is opened up, e.g. the derivation has access to the + network. Note that whether or not we actually sandbox the + derivation is controlled separately. Always true for non-CA + derivations. */ + bool isSandboxed() const; + + /* Whether the derivation is expected to produce the same result + every time, and therefore it only needs to be built once. This + is only false for derivations that have the attribute '__impure + = true'. */ + bool isPure() const; /* Does the derivation knows its own output paths? Only true when there's no floating-ca derivation involved in the @@ -173,7 +197,14 @@ struct Derivation : BasicDerivation added directly to input sources. 2. Input placeholders are replaced with realized input store paths. */ - std::optional<BasicDerivation> tryResolve(Store & store); + std::optional<BasicDerivation> tryResolve(Store & store) const; + + /* Like the above, but instead of querying the Nix database for + realisations, uses a given mapping from input derivation paths + + output names to actual output store paths. */ + std::optional<BasicDerivation> tryResolve( + Store & store, + const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const; Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } @@ -202,14 +233,16 @@ bool isDerivation(const std::string & fileName); the output name is "out". */ std::string outputPathName(std::string_view drvName, std::string_view outputName); -// known CA drv's output hashes, current just for fixed-output derivations -// whose output hashes are always known since they are fixed up-front. -typedef std::map<std::string, Hash> CaOutputHashes; +// The hashes modulo of a derivation. +// +// Each output is given a hash, although in practice only the content-addressed +// derivations (fixed-output or not) will have a different hash for each +// output. struct DrvHash { - Hash hash; + std::map<std::string, Hash> hashes; - enum struct Kind: bool { + enum struct Kind : bool { // Statically determined derivations. // This hash will be directly used to compute the output paths Regular, @@ -222,28 +255,6 @@ struct DrvHash { void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; -typedef std::variant< - // Regular normalized derivation hash, and whether it was deferred (because - // an ancestor derivation is a floating content addressed derivation). - DrvHash, - // Fixed-output derivation hashes - CaOutputHashes -> _DrvHashModuloRaw; - -struct DrvHashModulo : _DrvHashModuloRaw { - using Raw = _DrvHashModuloRaw; - using Raw::Raw; - - /* Get hash, throwing if it is per-output CA hashes or a - deferred Drv hash. - */ - const Hash & requireNoFixedNonDeferred() const; - - inline const Raw & raw() const { - return static_cast<const Raw &>(*this); - } -}; - /* Returns hashes with the details of fixed-output subderivations expunged. @@ -267,16 +278,18 @@ struct DrvHashModulo : _DrvHashModuloRaw { ATerm, after subderivations have been likewise expunged from that derivation. */ -DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /* Return a map associating each output to a hash that uniquely identifies its derivation (modulo the self-references). + + FIXME: what is the Hash in this map? */ -std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv); +std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv); /* Memoisation of hashDerivationModulo(). */ -typedef std::map<StorePath, DrvHashModulo> DrvHashes; +typedef std::map<StorePath, DrvHash> DrvHashes; // FIXME: global, though at least thread-safe. extern Sync<DrvHashes> drvHashes; @@ -306,4 +319,6 @@ std::string hashPlaceholder(const std::string_view outputName); dependency which is a CA derivation. */ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); +extern const Hash impureOutputHash; + } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 5a38203ae..5d9107ab2 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -26,6 +26,9 @@ struct DerivedPathOpaque { nlohmann::json toJSON(ref<Store> store) const; std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); + + bool operator < (const DerivedPathOpaque & b) const + { return path < b.path; } }; /** @@ -47,6 +50,9 @@ struct DerivedPathBuilt { std::string to_string(const Store & store) const; static DerivedPathBuilt parse(const Store & store, std::string_view); nlohmann::json toJSON(ref<Store> store) const; + + bool operator < (const DerivedPathBuilt & b) const + { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); } }; using _DerivedPathRaw = std::variant< diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index ca61e3937..40e7cf52c 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -31,7 +31,7 @@ struct FileTransferSettings : Config R"( The timeout (in seconds) for establishing connections in the binary cache substituter. It corresponds to `curl`’s - `--connect-timeout` option. + `--connect-timeout` option. A value of 0 means no limit. )"}; Setting<unsigned long> stalledDownloadTimeout{ @@ -123,8 +123,6 @@ public: template<typename... Args> FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args); - - virtual const char* sname() const override { return "FileTransferError"; } }; bool isUri(std::string_view s); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index fd7106e40..40133cd0c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -695,16 +695,15 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat // combinations that are currently prohibited. drv.type(); - std::optional<Hash> h; + std::optional<DrvHash> hashesModulo; for (auto & i : drv.outputs) { std::visit(overloaded { [&](const DerivationOutput::InputAddressed & doia) { - if (!h) { + if (!hashesModulo) { // somewhat expensive so we do lazily - auto h0 = hashDerivationModulo(*this, drv, true); - h = h0.requireNoFixedNonDeferred(); + hashesModulo = hashDerivationModulo(*this, drv, true); } - StorePath recomputed = makeOutputPath(i.first, *h, drvName); + StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName); if (doia.path != recomputed) throw Error("derivation '%s' has incorrect output '%s', should be '%s'", printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); @@ -720,6 +719,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat [&](const DerivationOutput::Deferred &) { /* Nothing to check */ }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, }, i.second.raw()); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 231bdf808..2d5b8d4b9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -276,15 +276,15 @@ std::map<DrvOutput, StorePath> drvOutputReferences( { std::set<Realisation> inputRealisations; - for (const auto& [inputDrv, outputNames] : drv.inputDrvs) { + for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { auto outputHashes = staticOutputHashes(store, store.readDerivation(inputDrv)); - for (const auto& outputName : outputNames) { + for (const auto & outputName : outputNames) { auto thisRealisation = store.queryRealisation( DrvOutput{outputHashes.at(outputName), outputName}); if (!thisRealisation) throw Error( - "output '%s' of derivation '%s' isn’t built", outputName, + "output '%s' of derivation '%s' isn't built", outputName, store.printStorePath(inputDrv)); inputRealisations.insert(*thisRealisation); } @@ -294,4 +294,5 @@ std::map<DrvOutput, StorePath> drvOutputReferences( return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); } + } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index e642abcd5..392db225e 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,5 +1,7 @@ #include "store-api.hh" +#include <sodium.h> + namespace nix { static void checkName(std::string_view path, std::string_view name) @@ -41,6 +43,13 @@ bool StorePath::isDerivation() const StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); +StorePath StorePath::random(std::string_view name) +{ + Hash hash(htSHA1); + randombytes_buf(hash.hash, hash.hashSize); + return StorePath(hash, name); +} + StorePath Store::parseStorePath(std::string_view path) const { auto p = canonPath(std::string(path)); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 6d91b8e7b..6771f0ba5 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -59,6 +59,8 @@ public: } static StorePath dummy; + + static StorePath random(std::string_view name); }; typedef std::set<StorePath> StorePathSet; diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index e6ecadd7f..1d82b4ab1 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -215,7 +215,6 @@ void handleSQLiteBusy(const SQLiteBusy & e) if (now > lastWarned + 10) { lastWarned = now; logWarning({ - .name = "Sqlite busy", .msg = hintfmt(e.what()) }); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 69aa0d094..4b8c55686 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -127,11 +127,11 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) if (flag.handler.arity == ArityAny) break; throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); } - if (flag.completer) - if (auto prefix = needsCompletion(*pos)) { - anyCompleted = true; + if (auto prefix = needsCompletion(*pos)) { + anyCompleted = true; + if (flag.completer) flag.completer(n, *prefix); - } + } args.push_back(*pos++); } if (!anyCompleted) @@ -146,6 +146,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) && hasPrefix(name, std::string(*prefix, 2))) completions->add("--" + name, flag->description); } + return false; } auto i = longFlags.find(std::string(*pos, 2)); if (i == longFlags.end()) return false; @@ -187,10 +188,12 @@ bool Args::processArgs(const Strings & args, bool finish) { std::vector<std::string> ss; for (const auto &[n, s] : enumerate(args)) { - ss.push_back(s); - if (exp.completer) - if (auto prefix = needsCompletion(s)) + if (auto prefix = needsCompletion(s)) { + ss.push_back(*prefix); + if (exp.completer) exp.completer(n, *prefix); + } else + ss.push_back(s); } exp.handler.fun(ss); expectedArgs.pop_front(); @@ -279,21 +282,22 @@ static void _completePath(std::string_view prefix, bool onlyDirs) { completionType = ctFilenames; glob_t globbuf; - int flags = GLOB_NOESCAPE | GLOB_TILDE; + int flags = GLOB_NOESCAPE; #ifdef GLOB_ONLYDIR if (onlyDirs) flags |= GLOB_ONLYDIR; #endif - if (glob((std::string(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) { + // using expandTilde here instead of GLOB_TILDE(_CHECK) so that ~<Tab> expands to /home/user/ + if (glob((expandTilde(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) { for (size_t i = 0; i < globbuf.gl_pathc; ++i) { if (onlyDirs) { - auto st = lstat(globbuf.gl_pathv[i]); + auto st = stat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } completions->add(globbuf.gl_pathv[i]); } - globfree(&globbuf); } + globfree(&globbuf); } void completePath(size_t, std::string_view prefix) @@ -322,11 +326,6 @@ MultiCommand::MultiCommand(const Commands & commands_) .optional = true, .handler = {[=](std::string s) { assert(!command); - if (auto prefix = needsCompletion(s)) { - for (auto & [name, command] : commands) - if (hasPrefix(name, *prefix)) - completions->add(name); - } auto i = commands.find(s); if (i == commands.end()) { std::set<std::string> commandNames; @@ -337,6 +336,11 @@ MultiCommand::MultiCommand(const Commands & commands_) } command = {s, i->second()}; command->second->parent = this; + }}, + .completer = {[&](size_t, std::string_view prefix) { + for (auto & [name, command] : commands) + if (hasPrefix(name, prefix)) + completions->add(name); }} }); diff --git a/src/libutil/error.cc b/src/libutil/error.cc index b2dfb35b2..9172f67a6 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -9,10 +9,9 @@ namespace nix { const std::string nativeSystem = SYSTEM; -BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint) +void BaseError::addTrace(std::optional<ErrPos> e, hintformat hint) { err.traces.push_front(Trace { .pos = e, .hint = hint }); - return *this; } // c++ std::exception descendants must have a 'const char* what()' function. @@ -22,12 +21,9 @@ const std::string & BaseError::calcWhat() const if (what_.has_value()) return *what_; else { - err.name = sname(); - std::ostringstream oss; showErrorInfo(oss, err, loggerSettings.showTrace); what_ = oss.str(); - return *what_; } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index a7cd08d74..7a5316896 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -109,7 +109,6 @@ struct Trace { struct ErrorInfo { Verbosity level; - std::string name; // FIXME: rename hintformat msg; std::optional<ErrPos> errPos; std::list<Trace> traces; @@ -162,8 +161,6 @@ public: : err(e) { } - virtual const char* sname() const { return "BaseError"; } - #ifdef EXCEPTION_NEEDS_THROW_SPEC ~BaseError() throw () { }; const char * what() const throw () { return calcWhat().c_str(); } @@ -175,12 +172,12 @@ public: const ErrorInfo & info() const { calcWhat(); return err; } template<typename... Args> - BaseError & addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args) + void addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args) { - return addTrace(e, hintfmt(fs, args...)); + addTrace(e, hintfmt(fs, args...)); } - BaseError & addTrace(std::optional<ErrPos> e, hintformat hint); + void addTrace(std::optional<ErrPos> e, hintformat hint); bool hasTrace() const { return !err.traces.empty(); } }; @@ -190,7 +187,6 @@ public: { \ public: \ using superClass::superClass; \ - virtual const char* sname() const override { return #newClass; } \ } MakeError(Error, BaseError); @@ -209,8 +205,6 @@ public: auto hf = hintfmt(args...); err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } - - virtual const char* sname() const override { return "SysError"; } }; } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 01f318fa3..e033a4116 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -7,6 +7,7 @@ namespace nix { std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = { { Xp::CaDerivations, "ca-derivations" }, + { Xp::ImpureDerivations, "impure-derivations" }, { Xp::Flakes, "flakes" }, { Xp::NixCommand, "nix-command" }, { Xp::RecursiveNix, "recursive-nix" }, diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index b5140dcfe..266e41a22 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -16,6 +16,7 @@ namespace nix { enum struct ExperimentalFeature { CaDerivations, + ImpureDerivations, Flakes, NixCommand, RecursiveNix, @@ -48,10 +49,6 @@ public: ExperimentalFeature missingFeature; MissingExperimentalFeature(ExperimentalFeature); - virtual const char * sname() const override - { - return "MissingExperimentalFeature"; - } }; } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index a4d632161..d2fd0c15a 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -155,7 +155,7 @@ static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_ { bool isSRI = false; - // Parse the has type before the separater, if there was one. + // Parse the hash type before the separator, if there was one. std::optional<HashType> optParsedType; { auto hashRaw = splitPrefixTo(rest, ':'); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 56b5938b3..00f70a572 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -93,13 +93,11 @@ public: std::string gitRev() const { - assert(type == htSHA1); return to_string(Base16, false); } std::string gitShortRev() const { - assert(type == htSHA1); return std::string(to_string(Base16, false), 0, 7); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 6445b3f1b..8ff904583 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -357,7 +357,7 @@ Sink & operator << (Sink & sink, const Error & ex) sink << "Error" << info.level - << info.name + << "Error" // removed << info.msg.str() << 0 // FIXME: info.errPos << info.traces.size(); @@ -426,11 +426,10 @@ Error readError(Source & source) auto type = readString(source); assert(type == "Error"); auto level = (Verbosity) readInt(source); - auto name = readString(source); + auto name = readString(source); // removed auto msg = readString(source); ErrorInfo info { .level = level, - .name = name, .msg = hintformat(std::move(format("%s") % msg)), }; auto havePos = readNum<size_t>(source); diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index f20e2dc41..c3b233797 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -178,7 +178,7 @@ namespace nix { } TEST(parseURL, parseFileURLWithQueryAndFragment) { - auto s = "file:///none/of/your/business"; + auto s = "file:///none/of//your/business"; auto parsed = parseURL(s); ParsedURL expected { @@ -186,7 +186,7 @@ namespace nix { .base = "", .scheme = "file", .authority = "", - .path = "/none/of/your/business", + .path = "/none/of//your/business", .query = (StringMap) { }, .fragment = "", }; diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index da10a6bbc..d5e6a2736 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -18,7 +18,7 @@ const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncod const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; -const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string segmentRegex = "(?:" + pcharRegex + "*)"; const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 70eaf4f9c..656804007 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -71,13 +71,11 @@ void clearEnv() unsetenv(name.first.c_str()); } -void replaceEnv(std::map<std::string, std::string> newEnv) +void replaceEnv(const std::map<std::string, std::string> & newEnv) { clearEnv(); - for (auto newEnvVar : newEnv) - { + for (auto & newEnvVar : newEnv) setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); - } } @@ -200,6 +198,17 @@ std::string_view baseNameOf(std::string_view path) } +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + + bool isInDir(std::string_view path, std::string_view dir) { return path.substr(0, 1) == "/" @@ -215,6 +224,15 @@ bool isDirOrInDir(std::string_view path, std::string_view dir) } +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + struct stat lstat(const Path & path) { struct stat st; @@ -1261,9 +1279,9 @@ template<class C> C tokenizeString(std::string_view s, std::string_view separato { C result; auto pos = s.find_first_not_of(separators, 0); - while (pos != std::string::npos) { + while (pos != std::string_view::npos) { auto end = s.find_first_of(separators, pos + 1); - if (end == std::string::npos) end = s.size(); + if (end == std::string_view::npos) end = s.size(); result.insert(result.end(), std::string(s, pos, end - pos)); pos = s.find_first_not_of(separators, end); } @@ -1473,6 +1491,7 @@ constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv std::string base64Encode(std::string_view s) { std::string res; + res.reserve((s.size() + 2) / 3 * 4); int data = 0, nbits = 0; for (char c : s) { @@ -1504,6 +1523,9 @@ std::string base64Decode(std::string_view s) }(); std::string res; + // Some sequences are missing the padding consisting of up to two '='. + // vvv + res.reserve((s.size() + 2) / 4 * 3); unsigned int d = 0, bits = 0; for (char c : s) { @@ -1690,7 +1712,9 @@ void setStackSize(size_t stackSize) #endif } +#if __linux__ static AutoCloseFD fdSavedMountNamespace; +#endif void saveMountNamespace() { @@ -1709,8 +1733,13 @@ void restoreMountNamespace() { #if __linux__ try { + auto savedCwd = absPath("."); + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); + if (chdir(savedCwd.c_str()) == -1) { + throw SysError("restoring cwd"); + } } catch (Error & e) { debug(e.msg()); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 20591952d..a1d0e0e6b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -68,6 +68,9 @@ Path dirOf(const PathView path); following the final `/' (trailing slashes are removed). */ std::string_view baseNameOf(std::string_view path); +/* Perform tilde expansion on a path. */ +std::string expandTilde(std::string_view path); + /* Check whether 'path' is a descendant of 'dir'. Both paths must be canonicalized. */ bool isInDir(std::string_view path, std::string_view dir); @@ -77,6 +80,7 @@ bool isInDir(std::string_view path, std::string_view dir); bool isDirOrInDir(std::string_view path, std::string_view dir); /* Get status of `path'. */ +struct stat stat(const Path & path); struct stat lstat(const Path & path); /* Return true iff the given path exists. */ diff --git a/src/nix/app.cc b/src/nix/app.cc index 803d028f0..df7303e15 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -35,7 +35,7 @@ struct InstallableDerivedPath : Installable /** * Return the rewrites that are needed to resolve a string whose context is - * included in `dependencies` + * included in `dependencies`. */ StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) { @@ -51,7 +51,7 @@ StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) } /** - * Resolve the given string assuming the given context + * Resolve the given string assuming the given context. */ std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies) { @@ -61,14 +61,18 @@ std::string resolveString(Store & store, const std::string & toResolve, const Bu UnresolvedApp Installable::toApp(EvalState & state) { - auto [cursor, attrPath] = getCursor(state); + auto cursor = getCursor(state); + auto attrPath = cursor->getAttrPath(); auto type = cursor->getAttr("type")->getString(); + std::string expected = !attrPath.empty() && attrPath[0] == "apps" ? "app" : "derivation"; + if (type != expected) + throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected); + if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - std::vector<StorePathWithOutputs> context2; for (auto & [path, name] : context) context2.push_back({path, {name}}); @@ -101,7 +105,7 @@ UnresolvedApp Installable::toApp(EvalState & state) } else - throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); + throw Error("attribute '%s' has unsupported type '%s'", cursor->getAttrPathStr(), type); } // FIXME: move to libcmd diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 7ed558dee..81fb8464a 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -9,7 +9,7 @@ using namespace nix; struct CmdBundle : InstallableCommand { - std::string bundler = "github:matthewbauer/nix-bundle"; + std::string bundler = "github:NixOS/bundlers"; std::optional<Path> outLink; CmdBundle() diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d2f9b5a6a..7fc74d34e 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -204,10 +204,10 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore output.second = DerivationOutput::Deferred { }; drv.env[output.first] = ""; } - auto h0 = hashDerivationModulo(*evalStore, drv, true); - const Hash & h = h0.requireNoFixedNonDeferred(); + auto hashesModulo = hashDerivationModulo(*evalStore, drv, true); for (auto & output : drv.outputs) { + Hash h = hashesModulo.hashes.at(output.first); auto outPath = store->makeOutputPath(output.first, h, drv.name); output.second = DerivationOutput::InputAddressed { .path = outPath, diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 4f3003448..ea87e3d87 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -24,12 +24,12 @@ std::string formatProtocol(unsigned int proto) } bool checkPass(const std::string & msg) { - logger->log(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); + notice(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); return true; } bool checkFail(const std::string & msg) { - logger->log(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); + notice(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); return false; } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 8cd04d5fe..733b93661 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -16,7 +16,7 @@ struct CmdEval : MixJSON, InstallableCommand std::optional<std::string> apply; std::optional<Path> writeTo; - CmdEval() + CmdEval() : InstallableCommand(true /* supportReadOnlyMode */) { addFlag({ .longName = "raw", diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 47a380238..04b23ed0f 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -463,7 +463,7 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *v.attrs) { std::string name(attr.name); - if (name != "path" && name != "description") + if (name != "path" && name != "description" && name != "welcomeText") throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); } } catch (Error & e) { @@ -508,6 +508,7 @@ struct CmdFlakeCheck : FlakeCommand name == "defaultBundler" ? "bundlers.<system>.default" : name == "overlay" ? "overlays.default" : name == "devShell" ? "devShells.<system>.default" : + name == "nixosModule" ? "nixosModules.default" : ""; if (replacement != "") warn("flake output attribute '%s' is deprecated; use '%s' instead", name, replacement); @@ -527,6 +528,16 @@ struct CmdFlakeCheck : FlakeCommand } } + else if (name == "formatter") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkApp( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } + else if (name == "packages" || name == "devShells") { state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) { @@ -704,7 +715,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand defaultTemplateAttrPathsPrefixes, lockFlags); - auto [cursor, attrPath] = installable.getCursor(*evalState); + auto cursor = installable.getCursor(*evalState); auto templateDirAttr = cursor->getAttr("path"); auto templateDir = templateDirAttr->getString(); @@ -1010,6 +1021,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 1 && ( attrPath[0] == "defaultPackage" || attrPath[0] == "devShell" + || attrPath[0] == "formatter" || attrPath[0] == "nixosConfigurations" || attrPath[0] == "nixosModules" || attrPath[0] == "defaultApp" @@ -1026,7 +1038,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON } else if ( - (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell")) + (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell" || attrPath[0] == "formatter")) || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages" || attrPath[0] == "devShells")) ) { diff --git a/src/nix/flake.md b/src/nix/flake.md index d59915eeb..7d179a6c4 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -177,8 +177,8 @@ Currently the `type` attribute can be one of the following: attribute `url`. In URL form, the schema must be `http://`, `https://` or `file://` - URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz`, - `.tar.bz2` or `.tar.zst`. + URLs and the extension must be `.zip`, `.tar`, `.tgz`, `.tar.gz`, + `.tar.xz`, `.tar.bz2` or `.tar.zst`. * `github`: A more efficient way to fetch repositories from GitHub. The following attributes are required: diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc new file mode 100644 index 000000000..e5d44bd38 --- /dev/null +++ b/src/nix/fmt.cc @@ -0,0 +1,53 @@ +#include "command.hh" +#include "run.hh" + +using namespace nix; + +struct CmdFmt : SourceExprCommand { + std::vector<std::string> args; + + CmdFmt() { expectArgs({.label = "args", .handler = {&args}}); } + + std::string description() override { + return "reformat your code in the standard style"; + } + + std::string doc() override { + return + #include "fmt.md" + ; + } + + Category category() override { return catSecondary; } + + Strings getDefaultFlakeAttrPaths() override { + return Strings{"formatter." + settings.thisSystem.get()}; + } + + Strings getDefaultFlakeAttrPathPrefixes() override { return Strings{}; } + + void run(ref<Store> store) { + auto evalState = getEvalState(); + auto evalStore = getEvalStore(); + + auto installable = parseInstallable(store, "."); + auto app = installable->toApp(*evalState).resolve(evalStore, store); + + Strings programArgs{app.program}; + + // Propagate arguments from the CLI + if (args.empty()) { + // Format the current flake out of the box + programArgs.push_back("."); + } else { + // User wants more power, let them decide which paths to include/exclude + for (auto &i : args) { + programArgs.push_back(i); + } + } + + runProgramInStore(store, app.program, programArgs); + }; +}; + +static auto r2 = registerCommand<CmdFmt>("fmt"); diff --git a/src/nix/fmt.md b/src/nix/fmt.md new file mode 100644 index 000000000..1c78bb36f --- /dev/null +++ b/src/nix/fmt.md @@ -0,0 +1,53 @@ +R""( + +# Examples + +With [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt): + +```nix +# flake.nix +{ + outputs = { nixpkgs, self }: { + formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixpkgs-fmt; + }; +} +``` + +- Format the current flake: `$ nix fmt` + +- Format a specific folder or file: `$ nix fmt ./folder ./file.nix` + +With [nixfmt](https://github.com/serokell/nixfmt): + +```nix +# flake.nix +{ + outputs = { nixpkgs, self }: { + formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt; + }; +} +``` + +- Format specific files: `$ nix fmt ./file1.nix ./file2.nix` + +With [Alejandra](https://github.com/kamadorueda/alejandra): + +```nix +# flake.nix +{ + outputs = { nixpkgs, self }: { + formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.alejandra; + }; +} +``` + +- Format the current flake: `$ nix fmt` + +- Format a specific folder or file: `$ nix fmt ./folder ./file.nix` + +# Description + +`nix fmt` will rewrite all Nix files (\*.nix) to a canonical format +using the formatter specified in your flake. + +)"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 1d9da087c..3a32a7b55 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -62,22 +62,21 @@ struct ProfileElement return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); } - void updateStorePaths(ref<Store> evalStore, ref<Store> store, Installable & installable) + void updateStorePaths( + ref<Store> evalStore, + ref<Store> store, + const BuiltPaths & builtPaths) { // FIXME: respect meta.outputsToInstall storePaths.clear(); - for (auto & buildable : getBuiltPaths(evalStore, store, installable.toDerivedPaths())) { + for (auto & buildable : builtPaths) { std::visit(overloaded { [&](const BuiltPath::Opaque & bo) { storePaths.insert(bo.path); }, [&](const BuiltPath::Built & bfd) { - // TODO: Why are we querying if we know the output - // names already? Is it just to figure out what the - // default one is? - for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { + for (auto & output : bfd.outputs) storePaths.insert(output.second); - } }, }, buildable.raw()); } @@ -98,19 +97,30 @@ struct ProfileManifest auto json = nlohmann::json::parse(readFile(manifestPath)); auto version = json.value("version", 0); - if (version != 1) - throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + std::string sUrl; + std::string sOriginalUrl; + switch(version){ + case 1: + sUrl = "uri"; + sOriginalUrl = "originalUri"; + break; + case 2: + sUrl = "url"; + sOriginalUrl = "originalUrl"; + break; + default: + throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + } for (auto & e : json["elements"]) { ProfileElement element; for (auto & p : e["storePaths"]) element.storePaths.insert(state.store->parseStorePath((std::string) p)); element.active = e["active"]; - if (e.value("uri", "") != "") { - auto originalUrl = e.value("originalUrl", e["originalUri"]); + if (e.value(sUrl,"") != "") { element.source = ProfileElementSource{ - parseFlakeRef(originalUrl), - parseFlakeRef(e["uri"]), + parseFlakeRef(e[sOriginalUrl]), + parseFlakeRef(e[sUrl]), e["attrPath"] }; } @@ -145,13 +155,13 @@ struct ProfileManifest obj["active"] = element.active; if (element.source) { obj["originalUrl"] = element.source->originalRef.to_string(); - obj["uri"] = element.source->resolvedRef.to_string(); + obj["url"] = element.source->resolvedRef.to_string(); obj["attrPath"] = element.source->attrPath; } array.push_back(obj); } nlohmann::json json; - json["version"] = 1; + json["version"] = 2; json["elements"] = array; return json.dump(); } @@ -245,6 +255,16 @@ struct ProfileManifest } }; +static std::map<Installable *, BuiltPaths> +builtPathsPerInstallable( + const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> & builtPaths) +{ + std::map<Installable *, BuiltPaths> res; + for (auto & [installable, builtPath] : builtPaths) + res[installable.get()].push_back(builtPath); + return res; +} + struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { std::string description() override @@ -263,7 +283,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal); + auto builtPaths = builtPathsPerInstallable( + Installable::build2( + getEvalStore(), store, Realise::Outputs, installables, bmNormal)); for (auto & installable : installables) { ProfileElement element; @@ -278,7 +300,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile }; } - element.updateStorePaths(getEvalStore(), store, *installable); + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); manifest.elements.push_back(std::move(element)); } @@ -466,12 +488,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf warn ("Use 'nix profile list' to see the current profile."); } - auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal); + auto builtPaths = builtPathsPerInstallable( + Installable::build2( + getEvalStore(), store, Realise::Outputs, installables, bmNormal)); for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, *installable); + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); } updateProfile(manifest.build(store)); diff --git a/src/nix/run.cc b/src/nix/run.cc index 033263c36..25a8fa8d3 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -38,9 +38,12 @@ void runProgramInStore(ref<Store> store, unshare(CLONE_NEWUSER) doesn't work in a multithreaded program (which "nix" is), so we exec() a single-threaded helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast<LocalStore>(); + auto store2 = store.dynamic_pointer_cast<LocalFSStore>(); - if (store2 && store->storeDir != store2->getRealStoreDir()) { + if (!store2) + throw Error("store '%s' is not a local store so it does not support command execution", store->getUri()); + + if (store->storeDir != store2->getRealStoreDir()) { Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; for (auto & arg : args) helperArgs.push_back(arg); @@ -179,6 +182,7 @@ struct CmdRun : InstallableCommand { auto state = getEvalState(); + lockFlags.applyNixConfig = true; auto app = installable->toApp(*state).resolve(getEvalStore(), store); Strings allArgs{app.program}; diff --git a/src/nix/search.cc b/src/nix/search.cc index e9307342c..e96a85ea2 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -165,8 +165,8 @@ struct CmdSearch : InstallableCommand, MixJSON } }; - for (auto & [cursor, prefix] : installable->getCursors(*state)) - visit(*cursor, parseAttrPath(*state, prefix), true); + for (auto & cursor : installable->getCursors(*state)) + visit(*cursor, cursor->getAttrPath(), true); if (!json && !results) throw Error("no results for the given search term(s)!"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 0d9655732..fb46b4dbf 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -77,6 +77,10 @@ struct CmdShowDerivation : InstallablesCommand outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); }, [&](const DerivationOutput::Deferred &) {}, + [&](const DerivationOutput::Impure & doi) { + outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); + outputObj.attr("impure", true); + }, }, output.raw()); } } |