diff options
53 files changed, 1125 insertions, 424 deletions
@@ -1 +1 @@ -2.7.0
\ No newline at end of file +2.8.0
\ No newline at end of file diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 4e2afa20e..f0f9457d2 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -72,6 +72,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md) - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md) - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md) - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md) diff --git a/doc/manual/src/release-notes/rl-2.7.md b/doc/manual/src/release-notes/rl-2.7.md new file mode 100644 index 000000000..dc92a9cde --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.7.md @@ -0,0 +1,33 @@ +# Release X.Y (2022-03-07) + +* Nix will now make some helpful suggestions when you mistype + something on the command line. For instance, if you type `nix build + nixpkgs#thunderbrd`, it will suggest `thunderbird`. + +* A number of "default" flake output attributes have been + renamed. These are: + + * `defaultPackage.<system>` → `packages.<system>.default` + * `defaultApps.<system>` → `apps.<system>.default` + * `defaultTemplate` → `templates.default` + * `defaultBundler.<system>` → `bundlers.<system>.default` + * `overlay` → `overlays.default` + * `devShell.<system>` → `devShells.<system>.default` + + The old flake output attributes still work, but `nix flake check` + will warn about them. + +* Breaking API change: `nix bundle` now supports bundlers of the form + `bundler.<system>.<name>= derivation: another-derivation;`. This + supports additional functionality to inspect evaluation information + during bundling. A new + [repository](https://github.com/NixOS/bundlers) has various bundlers + implemented. + +* `nix store ping` now reports the version of the remote Nix daemon. + +* `nix flake {init,new}` now display information about which files have been + created. + +* Templates can now define a `welcomeText` attribute, which is printed out by + `nix flake {init,new} --template <template>`. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index d81e54e49..52e3b6240 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,31 +1,4 @@ # Release X.Y (202?-??-??) -* A number of "default" flake output attributes have been - renamed. These are: - - * `defaultPackage.<system>` → `packages.<system>.default` - * `defaultApps.<system>` → `apps.<system>.default` - * `defaultTemplate` → `templates.default` - * `defaultBundler.<system>` → `bundlers.<system>.default` - * `overlay` → `overlays.default` - * `devShell.<system>` → `devShells.<system>.default` - - The old flake output attributes still work, but `nix flake check` - will warn about them. - -* `nix bundle` breaking API change now supports bundlers of the form - `bundler.<system>.<name>= derivation: another-derivation;`. This supports - additional functionality to inspect evaluation information during bundling. A - new [repository](https://github.com/NixOS/bundlers) has various bundlers - implemented. - -* `nix store ping` now reports the version of the remote Nix daemon. - -* `nix flake {init,new}` now display information about which files have been - created. - -* Templates can now define a `welcomeText` attribute, which is printed out by - `nix flake {init,new} --template <template>`. - * Nix can now be built with LTO by passing `--enable-lto` to `configure`. LTO is currently only supported when building with GCC. diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 3209456bf..b7623d4ba 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -12,6 +12,7 @@ #include "eval-cache.hh" #include "url.hh" #include "registry.hh" +#include "build-result.hh" #include <regex> #include <queue> @@ -272,9 +273,9 @@ void completeFlakeRefWithFragment( auto attr = root->findAlongAttrPath(attrPath); if (!attr) continue; - for (auto & attr2 : attr->getAttrs()) { + for (auto & attr2 : (*attr)->getAttrs()) { if (hasPrefix(attr2, lastAttr)) { - auto attrPath2 = attr->getAttrPath(attr2); + auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); completions->add(flakeRefS + "#" + concatStringsSep(".", attrPath2)); @@ -568,15 +569,22 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF auto cache = openEvalCache(*state, lockedFlake); auto root = cache->getRoot(); + Suggestions suggestions; + for (auto & attrPath : getActualAttrPaths()) { debug("trying flake output attribute '%s'", attrPath); - auto attr = root->findAlongAttrPath( + auto attrOrSuggestions = root->findAlongAttrPath( parseAttrPath(*state, attrPath), true ); - if (!attr) continue; + if (!attrOrSuggestions) { + suggestions += attrOrSuggestions.getSuggestions(); + continue; + } + + auto attr = *attrOrSuggestions; if (!attr->isDerivation()) throw Error("flake output attribute '%s' is not a derivation", attrPath); @@ -591,7 +599,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)}; } - throw Error("flake '%s' does not provide attribute %s", + throw Error(suggestions, "flake '%s' does not provide attribute %s", flakeRef, showAttrPaths(getActualAttrPaths())); } @@ -610,17 +618,24 @@ std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state) auto emptyArgs = state.allocBindings(0); + Suggestions suggestions; + for (auto & attrPath : getActualAttrPaths()) { try { auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); state.forceValue(*v, pos); return {v, pos}; } catch (AttrPathNotFound & e) { + suggestions += e.info().suggestions; } } - throw Error("flake '%s' does not provide attribute %s", - flakeRef, showAttrPaths(getActualAttrPaths())); + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(getActualAttrPaths()) + ); } std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> @@ -635,7 +650,7 @@ InstallableFlake::getCursors(EvalState & state) for (auto & attrPath : getActualAttrPaths()) { auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back({attr, attrPath}); + if (attr) res.push_back({*attr, attrPath}); } return res; @@ -755,8 +770,7 @@ BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPa throw Error( "the derivation '%s' doesn't have an output named '%s'", store->printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled( - Xp::CaDerivations)) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { auto outputId = DrvOutput{outputHashes.at(output), output}; auto realisation = @@ -802,12 +816,33 @@ BuiltPaths Installable::build( pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); } - if (mode == Realise::Nothing || mode == Realise::Derivation) + switch (mode) { + case Realise::Nothing: + case Realise::Derivation: printMissing(store, pathsToBuild, lvlError); - else if (mode == Realise::Outputs) - store->buildPaths(pathsToBuild, bMode, evalStore); - - return getBuiltPaths(evalStore, store, pathsToBuild); + return getBuiltPaths(evalStore, store, pathsToBuild); + case Realise::Outputs: { + BuiltPaths res; + for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { + if (!buildResult.success()) + buildResult.rethrow(); + std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + std::map<std::string, StorePath> outputs; + for (auto & path : buildResult.builtOutputs) + outputs.emplace(path.first.outputName, path.second.outPath); + res.push_back(BuiltPath::Built { bfd.drvPath, outputs }); + }, + [&](const DerivedPath::Opaque & bo) { + res.push_back(BuiltPath::Opaque { bo.path }); + }, + }, buildResult.path.raw()); + } + return res; + } + default: + assert(false); + } } BuiltPaths Installable::toBuiltPaths( diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index eb0e706c7..32deecfae 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -74,8 +74,14 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const std::string & throw Error("empty attribute name in selection path '%1%'", attrPath); Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); - if (a == v->attrs->end()) - throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); + if (a == v->attrs->end()) { + std::set<std::string> attrNames; + for (auto & attr : *v->attrs) + attrNames.insert(attr.name); + + auto suggestions = Suggestions::bestMatches(attrNames, attr); + throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath); + } v = &*a->value; pos = *a->pos; } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 00d0749f9..188223957 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -406,6 +406,16 @@ Value & AttrCursor::forceValue() return v; } +Suggestions AttrCursor::getSuggestionsForAttr(Symbol name) +{ + auto attrNames = getAttrs(); + std::set<std::string> strAttrNames; + for (auto & name : attrNames) + strAttrNames.insert(std::string(name)); + + return Suggestions::bestMatches(strAttrNames, name); +} + std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) { if (root->db) { @@ -446,6 +456,11 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + for (auto & attr : *v.attrs) { + if (root->db) + root->db->setPlaceholder({cachedValue->first, attr.name}); + } + auto attr = v.attrs->get(name); if (!attr) { @@ -464,7 +479,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()}; } - return std::make_shared<AttrCursor>( + return make_ref<AttrCursor>( root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2)); } @@ -473,27 +488,31 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name) return maybeGetAttr(root->state.symbols.create(name)); } -std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors) +ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors) { auto p = maybeGetAttr(name, forceErrors); if (!p) throw Error("attribute '%s' does not exist", getAttrPathStr(name)); - return p; + return ref(p); } -std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name) +ref<AttrCursor> AttrCursor::getAttr(std::string_view name) { return getAttr(root->state.symbols.create(name)); } -std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force) +OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force) { auto res = shared_from_this(); for (auto & attr : attrPath) { - res = res->maybeGetAttr(attr, force); - if (!res) return {}; + auto child = res->maybeGetAttr(attr, force); + if (!child) { + auto suggestions = res->getSuggestionsForAttr(attr); + return OrSuggestions<ref<AttrCursor>>::failed(suggestions); + } + res = child; } - return res; + return ref(res); } std::string AttrCursor::getString() diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 43b34ebcb..40f1d4ffc 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -94,15 +94,17 @@ public: std::string getAttrPathStr(Symbol name) const; + Suggestions getSuggestionsForAttr(Symbol name); + std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false); std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name); - std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false); + ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false); - std::shared_ptr<AttrCursor> getAttr(std::string_view name); + ref<AttrCursor> getAttr(std::string_view name); - std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); + OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); std::string getString(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 193358161..5bf161cc0 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -63,9 +63,15 @@ static char * dupString(const char * s) } -static char * dupStringWithLen(const char * s, size_t size) +// When there's no need to write to the string, we can optimize away empty +// string allocations. +// This function handles makeImmutableStringWithLen(null, 0) by returning the +// empty string. +static const char * makeImmutableStringWithLen(const char * s, size_t size) { char * t; + if (size == 0) + return ""; #if HAVE_BOEHMGC t = GC_STRNDUP(s, size); #else @@ -75,6 +81,10 @@ static char * dupStringWithLen(const char * s, size_t size) return t; } +static inline const char * makeImmutableString(std::string_view s) { + return makeImmutableStringWithLen(s.data(), s.size()); +} + RootValue allocRootValue(Value * v) { @@ -86,15 +96,10 @@ RootValue allocRootValue(Value * v) } -void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) +void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v) { checkInterrupt(); - if (!active.insert(&v).second) { - str << "<CYCLE>"; - return; - } - switch (v.internalType) { case tInt: str << v.integer; @@ -120,24 +125,32 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu str << "null"; break; case tAttrs: { - str << "{ "; - for (auto & i : v.attrs->lexicographicOrder()) { - str << i->name << " = "; - printValue(str, active, *i->value); - str << "; "; + if (!v.attrs->empty() && !seen.insert(v.attrs).second) + str << "<REPEAT>"; + else { + str << "{ "; + for (auto & i : v.attrs->lexicographicOrder()) { + str << i->name << " = "; + printValue(str, seen, *i->value); + str << "; "; + } + str << "}"; } - str << "}"; break; } case tList1: case tList2: case tListN: - str << "[ "; - for (auto v2 : v.listItems()) { - printValue(str, active, *v2); - str << " "; + if (v.listSize() && !seen.insert(v.listElems()).second) + str << "<REPEAT>"; + else { + str << "[ "; + for (auto v2 : v.listItems()) { + printValue(str, seen, *v2); + str << " "; + } + str << "]"; } - str << "]"; break; case tThunk: case tApp: @@ -161,15 +174,13 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu default: abort(); } - - active.erase(&v); } std::ostream & operator << (std::ostream & str, const Value & v) { - std::set<const Value *> active; - printValue(str, active, v); + std::set<const void *> seen; + printValue(str, seen, v); return str; } @@ -804,7 +815,7 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con void Value::mkString(std::string_view s) { - mkString(dupStringWithLen(s.data(), s.size())); + mkString(makeImmutableString(s)); } @@ -835,7 +846,7 @@ void Value::mkStringMove(const char * s, const PathSet & context) void Value::mkPath(std::string_view s) { - mkPath(dupStringWithLen(s.data(), s.size())); + mkPath(makeImmutableString(s)); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 3fdff71a5..d0fa93e92 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -114,8 +114,8 @@ struct Value private: InternalType internalType; -friend std::string showType(const Value & v); -friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v); + friend std::string showType(const Value & v); + friend void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v); public: diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 58a2aded3..04c9feda0 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -29,7 +29,7 @@ struct FetchSettings : public Config * Github: the token value is the OAUTH-TOKEN string obtained as the Personal Access Token from the Github server (see - https://docs.github.com/en/developers/apps/authorizing-oath-apps). + https://docs.github.com/en/developers/apps/building-oauth-apps/authorizing-oauth-apps). * Gitlab: the token value is either the OAuth2 token or the Personal Access Token (these are different types tokens diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index a0b0f4cb3..562d1b414 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,6 +1,7 @@ #include "globals.hh" #include "shared.hh" #include "store-api.hh" +#include "gc-store.hh" #include "util.hh" #include "loggers.hh" diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 018ba31b3..cb6d19b8e 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -28,6 +28,7 @@ struct BuildResult LogLimitExceeded, NotDeterministic, ResolvesToAlreadyValid, + NoSubstituters, } status = MiscFailure; std::string errorMsg; @@ -63,15 +64,26 @@ struct BuildResult non-determinism.) */ bool isNonDeterministic = false; + /* The derivation we built or the store path we substituted. */ + DerivedPath path; + + /* For derivations, a mapping from the names of the wanted outputs + to actual paths. */ DrvOutputs builtOutputs; /* The start/stop times of the build (or one of the rounds, if it was repeated). */ time_t startTime = 0, stopTime = 0; - bool success() { + bool success() + { return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; } + + void rethrow() + { + throw Error("%s", errorMsg); + } }; } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 95da1841d..afed9bf16 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -66,7 +66,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) + : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -85,7 +85,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) + : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -135,7 +135,7 @@ void DerivationGoal::killChild() void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, ex); + done(BuildResult::TimedOut, {}, ex); } @@ -182,7 +182,7 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); + done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); return; } @@ -215,28 +215,20 @@ void DerivationGoal::haveDerivation() auto outputHashes = staticOutputHashes(worker.evalStore, *drv); for (auto & [outputName, outputHash] : outputHashes) - initialOutputs.insert({ + initialOutputs.insert({ outputName, - InitialOutput{ + InitialOutput { .wanted = true, // Will be refined later .outputHash = outputHash } - }); + }); /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } + auto [allValid, validOutputs] = checkPathValidity(); /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); + done(BuildResult::AlreadyValid, std::move(validOutputs)); return; } @@ -277,7 +269,7 @@ void DerivationGoal::outputsSubstitutionTried() trace("all outputs substituted (maybe)"); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, + done(BuildResult::TransientFailure, {}, Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", worker.store.printStorePath(drvPath))); return; @@ -301,23 +293,17 @@ void DerivationGoal::outputsSubstitutionTried() return; } - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } + auto [allValid, validOutputs] = checkPathValidity(); - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); + if (buildMode == bmNormal && allValid) { + done(BuildResult::Substituted, std::move(validOutputs)); return; } - if (buildMode == bmRepair && nrInvalid == 0) { + if (buildMode == bmRepair && allValid) { repairClosure(); return; } - if (buildMode == bmCheck && nrInvalid > 0) + if (buildMode == bmCheck && !allValid) throw Error("some outputs of '%s' are not valid, so checking is not possible", worker.store.printStorePath(drvPath)); @@ -409,7 +395,7 @@ void DerivationGoal::repairClosure() } if (waitees.empty()) { - done(BuildResult::AlreadyValid); + done(BuildResult::AlreadyValid, assertPathValidity()); return; } @@ -423,7 +409,7 @@ void DerivationGoal::closureRepaired() if (nrFailed > 0) throw Error("some paths in the output closure of derivation '%s' could not be repaired", worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); + done(BuildResult::AlreadyValid, assertPathValidity()); } @@ -434,7 +420,7 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( + done(BuildResult::DependencyFailed, {}, Error( "%s dependencies of derivation '%s' failed to build", nrFailed, worker.store.printStorePath(drvPath))); return; @@ -523,10 +509,11 @@ void DerivationGoal::inputsRealised() state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); - result = BuildResult(); + buildResult = BuildResult { .path = buildResult.path }; } -void DerivationGoal::started() { +void DerivationGoal::started() +{ auto msg = fmt( buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmCheck ? "checking outputs of '%s'" : @@ -588,19 +575,12 @@ void DerivationGoal::tryToBuild() omitted, but that would be less efficient.) Note that since we now hold the locks on the output paths, no other process can build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } + auto [allValid, validOutputs] = checkPathValidity(); + if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); + done(BuildResult::AlreadyValid, std::move(validOutputs)); return; } @@ -626,7 +606,7 @@ void DerivationGoal::tryToBuild() /* Yes, it has started doing so. Wait until we get EOF from the hook. */ actLock.reset(); - result.startTime = time(0); // inexact + buildResult.startTime = time(0); // inexact state = &DerivationGoal::buildDone; started(); return; @@ -830,8 +810,8 @@ void DerivationGoal::buildDone() debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - result.timesBuilt++; - result.stopTime = time(0); + buildResult.timesBuilt++; + buildResult.stopTime = time(0); /* So the child is gone now. */ worker.childTerminated(this); @@ -876,11 +856,11 @@ void DerivationGoal::buildDone() /* Compute the FS closure of the outputs and register them as being valid. */ - registerOutputs(); + auto builtOutputs = registerOutputs(); StorePathSet outputPaths; - for (auto & [_, path] : finalOutputs) - outputPaths.insert(path); + for (auto & [_, output] : buildResult.builtOutputs) + outputPaths.insert(output.outPath); runPostBuildHook( worker.store, *logger, @@ -890,7 +870,7 @@ void DerivationGoal::buildDone() if (buildMode == bmCheck) { cleanupPostOutputsRegisteredModeCheck(); - done(BuildResult::Built); + done(BuildResult::Built, std::move(builtOutputs)); return; } @@ -911,6 +891,8 @@ void DerivationGoal::buildDone() outputLocks.setDeletion(true); outputLocks.unlock(); + done(BuildResult::Built, std::move(builtOutputs)); + } catch (BuildError & e) { outputLocks.unlock(); @@ -930,14 +912,13 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, e); + done(st, {}, e); return; } - - done(BuildResult::Built); } -void DerivationGoal::resolvedFinished() { +void DerivationGoal::resolvedFinished() +{ assert(resolvedDrvGoal); auto resolvedDrv = *resolvedDrvGoal->drv; @@ -950,11 +931,13 @@ void DerivationGoal::resolvedFinished() { if (realWantedOutputs.empty()) realWantedOutputs = resolvedDrv.outputNames(); + DrvOutputs builtOutputs; + for (auto & wantedOutput : realWantedOutputs) { assert(initialOutputs.count(wantedOutput) != 0); assert(resolvedHashes.count(wantedOutput) != 0); auto realisation = worker.store.queryRealisation( - DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} + DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} ); // We've just built it, but maybe the build failed, in which case the // realisation won't be there @@ -966,10 +949,11 @@ void DerivationGoal::resolvedFinished() { signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); outputPaths.insert(realisation->outPath); + builtOutputs.emplace(realisation->id, *realisation); } else { // If we don't have a realisation, then it must mean that something // failed when building the resolved drv - assert(!result.success()); + assert(!buildResult.success()); } } @@ -981,7 +965,7 @@ void DerivationGoal::resolvedFinished() { ); auto status = [&]() { - auto resolvedResult = resolvedDrvGoal->getResult(); + auto & resolvedResult = resolvedDrvGoal->buildResult; switch (resolvedResult.status) { case BuildResult::AlreadyValid: return BuildResult::ResolvesToAlreadyValid; @@ -990,7 +974,7 @@ void DerivationGoal::resolvedFinished() { } }(); - done(status); + done(status, std::move(builtOutputs)); } HookReply DerivationGoal::tryBuildHook() @@ -1100,7 +1084,7 @@ HookReply DerivationGoal::tryBuildHook() } -void DerivationGoal::registerOutputs() +DrvOutputs DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -1109,21 +1093,7 @@ void DerivationGoal::registerOutputs() We can only early return when the outputs are known a priori. For floating content-addressed derivations this isn't the case. */ - for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { - if (!wantOutput(outputName, wantedOutputs)) - continue; - if (!optOutputPath) - throw BuildError( - "output '%s' from derivation '%s' does not have a known output path", - outputName, worker.store.printStorePath(drvPath)); - auto & outputPath = *optOutputPath; - if (!worker.store.isValidPath(outputPath)) - throw BuildError( - "output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid", - outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath)); - - finalOutputs.insert_or_assign(outputName, outputPath); - } + return assertPathValidity(); } Path DerivationGoal::openLogFile() @@ -1175,16 +1145,17 @@ bool DerivationGoal::isReadDesc(int fd) return fd == hook->builderOut.readSide.get(); } - void DerivationGoal::handleChildOutput(int fd, std::string_view data) { - if (isReadDesc(fd)) + // local & `ssh://`-builds are dealt with here. + auto isWrittenToLog = isReadDesc(fd); + if (isWrittenToLog) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { killChild(); done( - BuildResult::LogLimitExceeded, + BuildResult::LogLimitExceeded, {}, Error("%s killed after writing more than %d bytes of log output", getName(), settings.maxLogSize)); return; @@ -1207,7 +1178,16 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) if (hook && fd == hook->fromHook.readSide.get()) { for (auto c : data) if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); + auto json = parseJSONMessage(currentHookLine); + if (json) { + auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); + // ensure that logs from a builder using `ssh-ng://` as protocol + // are also available to `nix log`. + if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) { + auto f = (*json)["fields"]; + (*logSink)((f.size() > 0 ? f.at(0).get<std::string>() : "") + "\n"); + } + } currentHookLine.clear(); } else currentHookLine += c; @@ -1264,10 +1244,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() } -void DerivationGoal::checkPathValidity() +std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() { bool checkHash = buildMode == bmRepair; auto wantedOutputsLeft = wantedOutputs; + DrvOutputs validOutputs; + for (auto & i : queryPartialDerivationOutputMap()) { InitialOutput & info = initialOutputs.at(i.first); info.wanted = wantOutput(i.first, wantedOutputs); @@ -1284,26 +1266,28 @@ void DerivationGoal::checkPathValidity() : PathStatus::Corrupt, }; } + auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first}; if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first}; if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { .path = real->outPath, .status = PathStatus::Valid, }; - } else if (info.known && info.known->status == PathStatus::Valid) { - // We know the output because it' a static output of the + } else if (info.known && info.known->isValid()) { + // We know the output because it's a static output of the // derivation, and the output path is valid, but we don't have // its realisation stored (probably because it has been built - // without the `ca-derivations` experimental flag) + // without the `ca-derivations` experimental flag). worker.store.registerDrvOutput( - Realisation{ + Realisation { drvOutput, info.known->path, } ); } } + if (info.wanted && info.known && info.known->isValid()) + validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); } // If we requested all the outputs via the empty set, we are always fine. // If we requested specific elements, the loop above removes all the valid @@ -1312,24 +1296,50 @@ void DerivationGoal::checkPathValidity() throw Error("derivation '%s' does not have wanted outputs %s", worker.store.printStorePath(drvPath), concatStringsSep(", ", quoteStrings(wantedOutputsLeft))); + + bool allValid = true; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) { + allValid = false; + break; + } + } + + return { allValid, validOutputs }; +} + + +DrvOutputs DerivationGoal::assertPathValidity() +{ + auto [allValid, validOutputs] = checkPathValidity(); + if (!allValid) + throw Error("some outputs are unexpectedly invalid"); + return validOutputs; } -void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex) +void DerivationGoal::done( + BuildResult::Status status, + DrvOutputs builtOutputs, + std::optional<Error> ex) { - result.status = status; + buildResult.status = status; if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) + // FIXME: strip: "error: " + buildResult.errorMsg = ex->what(); + amDone(buildResult.success() ? ecSuccess : ecFailed, ex); + if (buildResult.status == BuildResult::TimedOut) worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) + if (buildResult.status == BuildResult::PermanentFailure) worker.permanentFailure = true; mcExpectedBuilds.reset(); mcRunningBuilds.reset(); - if (result.success()) { + if (buildResult.success()) { + assert(!builtOutputs.empty()); + buildResult.builtOutputs = std::move(builtOutputs); if (status == BuildResult::Built) worker.doneBuilds++; } else { @@ -1343,7 +1353,7 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex) if (traceBuiltOutputsFile != "") { std::fstream fs; fs.open(traceBuiltOutputsFile, std::fstream::out); - fs << worker.store.printStorePath(drvPath) << "\t" << result.toString() << std::endl; + fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 6f158bdbf..ea2db89b2 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -2,7 +2,6 @@ #include "parsed-derivations.hh" #include "lock.hh" -#include "build-result.hh" #include "store-api.hh" #include "pathlocks.hh" #include "goal.hh" @@ -105,20 +104,8 @@ struct DerivationGoal : public Goal typedef void (DerivationGoal::*GoalState)(); GoalState state; - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - BuildMode buildMode; - BuildResult result; - /* The current round, if we're building multiple times. */ size_t curRound = 1; @@ -153,8 +140,6 @@ struct DerivationGoal : public Goal /* Add wanted outputs to an already existing derivation goal. */ void addWantedOutputs(const StringSet & outputs); - BuildResult getResult() { return result; } - /* The states. */ void getDerivation(); void loadDerivation(); @@ -176,7 +161,7 @@ struct DerivationGoal : public Goal /* Check that the derivation outputs all exist and register them as valid. */ - virtual void registerOutputs(); + virtual DrvOutputs registerOutputs(); /* Open a log file and a pipe to it. */ Path openLogFile(); @@ -211,8 +196,17 @@ struct DerivationGoal : public Goal std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(); OutputPathMap queryDerivationOutputMap(); - /* Return the set of (in)valid paths. */ - void checkPathValidity(); + /* Update 'initialOutputs' to determine the current status of the + outputs of the derivation. Also returns a Boolean denoting + whether all outputs are valid and non-corrupt, and a + 'DrvOutputs' structure containing the valid and wanted + outputs. */ + std::pair<bool, DrvOutputs> checkPathValidity(); + + /* Aborts if any output is not valid or corrupt, and otherwise + returns a 'DrvOutputs' structure containing the wanted + outputs. */ + DrvOutputs assertPathValidity(); /* Forcibly kill the child process, if any. */ virtual void killChild(); @@ -223,6 +217,7 @@ struct DerivationGoal : public Goal void done( BuildResult::Status status, + DrvOutputs builtOutputs = {}, std::optional<Error> ex = {}); StorePathSet exportReferences(const StorePathSet & storePaths); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index e6e810cdd..e50292c1e 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -6,8 +6,12 @@ namespace nix { -DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca) - : Goal(worker) +DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( + const DrvOutput & id, + Worker & worker, + RepairFlag repair, + std::optional<ContentAddress> ca) + : Goal(worker, DerivedPath::Opaque { StorePath::dummy }) , id(id) { state = &DrvOutputSubstitutionGoal::init; @@ -32,7 +36,7 @@ void DrvOutputSubstitutionGoal::init() void DrvOutputSubstitutionGoal::tryNext() { - trace("Trying next substituter"); + trace("trying next substituter"); if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal @@ -119,7 +123,7 @@ void DrvOutputSubstitutionGoal::realisationFetched() void DrvOutputSubstitutionGoal::outPathValid() { assert(outputInfo); - trace("Output path substituted"); + trace("output path substituted"); if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 9b4cfd835..bea7363db 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -47,43 +47,51 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod } } +std::vector<BuildResult> Store::buildPathsWithResults( + const std::vector<DerivedPath> & reqs, + BuildMode buildMode, + std::shared_ptr<Store> evalStore) +{ + Worker worker(*this, evalStore ? *evalStore : *this); + + Goals goals; + for (const auto & br : reqs) { + std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); + }, + [&](const DerivedPath::Opaque & bo) { + goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); + }, + }, br.raw()); + } + + worker.run(goals); + + std::vector<BuildResult> results; + + for (auto & i : goals) + results.push_back(i->buildResult); + + return results; +} + BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { Worker worker(*this, *this); auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - BuildResult result; - try { worker.run(Goals{goal}); - result = goal->getResult(); + return goal->buildResult; } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - // XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's - // extended to return the full realisation for each output - auto staticDrvOutputs = drv.outputsAndOptPaths(*this); - auto outputHashes = staticOutputHashes(*this, drv); - for (auto & [outputName, staticOutput] : staticDrvOutputs) { - auto outputId = DrvOutput{outputHashes.at(outputName), outputName}; - if (staticOutput.second) - result.builtOutputs.insert_or_assign( - outputId, - Realisation{ outputId, *staticOutput.second} - ); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) { - auto realisation = this->queryRealisation(outputId); - if (realisation) - result.builtOutputs.insert_or_assign( - outputId, - *realisation - ); - } - } - - return result; + return BuildResult { + .status = BuildResult::MiscFailure, + .errorMsg = e.msg(), + .path = DerivedPath::Built { .drvPath = drvPath }, + }; + }; } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 68e08bdf9..07c752bb9 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -2,6 +2,7 @@ #include "types.hh" #include "store-api.hh" +#include "build-result.hh" namespace nix { @@ -55,10 +56,15 @@ struct Goal : public std::enable_shared_from_this<Goal> /* Whether the goal is finished. */ ExitCode exitCode; + /* Build result. */ + BuildResult buildResult; + /* Exception containing an error message, if any. */ std::optional<Error> ex; - Goal(Worker & worker) : worker(worker) + Goal(Worker & worker, DerivedPath path) + : worker(worker) + , buildResult { .path = std::move(path) } { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; exitCode = ecBusy; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7e69d4145..a372728f5 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,4 +1,5 @@ #include "local-derivation-goal.hh" +#include "gc-store.hh" #include "hook-instance.hh" #include "worker.hh" #include "builtins.hh" @@ -193,7 +194,7 @@ void LocalDerivationGoal::tryLocalBuild() { outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, e); + done(BuildResult::InputRejected, {}, e); return; } @@ -755,7 +756,7 @@ void LocalDerivationGoal::startBuilder() if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) throw SysError("putting pseudoterminal into raw mode"); - result.startTime = time(0); + buildResult.startTime = time(0); /* Fork a child to build the package. */ @@ -1127,7 +1128,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore { ref<LocalStore> next; @@ -1260,6 +1261,16 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override { + for (auto & result : buildPathsWithResults(paths, buildMode, evalStore)) + if (!result.success()) + result.rethrow(); + } + + std::vector<BuildResult> buildPathsWithResults( + const std::vector<DerivedPath> & paths, + BuildMode buildMode = bmNormal, + std::shared_ptr<Store> evalStore = nullptr) override + { assert(!evalStore); if (buildMode != bmNormal) throw Error("unsupported build mode"); @@ -1272,26 +1283,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); } - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - auto p = std::get_if<DerivedPath::Built>(&path); - if (!p) continue; - auto & bfd = *p; - auto drv = readDerivation(bfd.drvPath); - auto drvHashes = staticOutputHashes(*this, drv); - auto outputs = next->queryDerivationOutputMap(bfd.drvPath); - for (auto & [outputName, outputPath] : outputs) - if (wantOutput(outputName, bfd.outputs)) { - newPaths.insert(outputPath); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - auto thisRealisation = next->queryRealisation( - DrvOutput{drvHashes.at(outputName), outputName} - ); - assert(thisRealisation); - newRealisations.insert(*thisRealisation); - } - } + auto results = next->buildPathsWithResults(paths, buildMode); + + for (auto & result : results) { + for (auto & [outputName, output] : result.builtOutputs) { + newPaths.insert(output.outPath); + newRealisations.insert(output); + } } StorePathSet closure; @@ -1300,6 +1298,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo goal.addDependency(path); for (auto & real : Realisation::closure(*next, newRealisations)) goal.addedDrvOutputs.insert(real.id); + + return results; } BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, @@ -2068,7 +2068,7 @@ void LocalDerivationGoal::runChild() } -void LocalDerivationGoal::registerOutputs() +DrvOutputs LocalDerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -2077,10 +2077,8 @@ void LocalDerivationGoal::registerOutputs() We can only early return when the outputs are known a priori. For floating content-addressed derivations this isn't the case. */ - if (hook) { - DerivationGoal::registerOutputs(); - return; - } + if (hook) + return DerivationGoal::registerOutputs(); std::map<std::string, ValidPathInfo> infos; @@ -2203,6 +2201,8 @@ void LocalDerivationGoal::registerOutputs() std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); + OutputPathMap finalOutputs; + for (auto & outputName : sortedOutputNames) { auto output = drv->outputs.at(outputName); auto & scratchPath = scratchOutputs.at(outputName); @@ -2339,6 +2339,7 @@ void LocalDerivationGoal::registerOutputs() }; ValidPathInfo newInfo = std::visit(overloaded { + [&](const DerivationOutputInputAddressed & output) { /* input-addressed case */ auto requiredFinalPath = output.path; @@ -2358,6 +2359,7 @@ void LocalDerivationGoal::registerOutputs() newInfo0.references.insert(newInfo0.path); return newInfo0; }, + [&](const DerivationOutputCAFixed & dof) { auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { .method = dof.hash.method, @@ -2380,14 +2382,17 @@ void LocalDerivationGoal::registerOutputs() } return newInfo0; }, - [&](DerivationOutputCAFloating dof) { + + [&](DerivationOutputCAFloating & dof) { return newInfoFromCA(dof); }, + [&](DerivationOutputDeferred) -> ValidPathInfo { // No derivation should reach that point without having been // rewritten first assert(false); }, + }, output.output); /* FIXME: set proper permissions in restorePath() so @@ -2498,11 +2503,12 @@ void LocalDerivationGoal::registerOutputs() } if (buildMode == bmCheck) { - // In case of FOD mismatches on `--check` an error must be thrown as this is also - // a source for non-determinism. + /* In case of fixed-output derivations, if there are + mismatches on `--check` an error must be thrown as this is + also a source for non-determinism. */ if (delayedException) std::rethrow_exception(delayedException); - return; + return assertPathValidity(); } /* Apply output checks. */ @@ -2514,7 +2520,7 @@ void LocalDerivationGoal::registerOutputs() assert(prevInfos.size() == infos.size()); for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { - result.isNonDeterministic = true; + buildResult.isNonDeterministic = true; Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); hintformat hint = prevExists @@ -2552,7 +2558,7 @@ void LocalDerivationGoal::registerOutputs() if (curRound < nrRounds) { prevInfos = std::move(infos); - return; + return {}; } /* Remove the .check directories if we're done. FIXME: keep them @@ -2587,17 +2593,24 @@ void LocalDerivationGoal::registerOutputs() means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ + DrvOutputs builtOutputs; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - for (auto& [outputName, newInfo] : infos) { - auto thisRealisation = Realisation{ - .id = DrvOutput{initialOutputs.at(outputName).outputHash, - outputName}, - .outPath = newInfo.path}; + for (auto & [outputName, newInfo] : infos) { + auto thisRealisation = Realisation { + .id = DrvOutput { + initialOutputs.at(outputName).outputHash, + outputName + }, + .outPath = newInfo.path + }; + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } + builtOutputs.emplace(thisRealisation.id, thisRealisation); } + + return builtOutputs; } void LocalDerivationGoal::signRealisation(Realisation & realisation) @@ -2606,7 +2619,7 @@ void LocalDerivationGoal::signRealisation(Realisation & realisation) } -void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) +void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo> & outputs) { std::map<Path, const ValidPathInfo &> outputsByPath; for (auto & output : outputs) @@ -2678,8 +2691,8 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out for (auto & i : *value) { if (worker.store.isStorePath(i)) spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); + else if (outputs.count(i)) + spec.insert(outputs.at(i).path); else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 95692c60d..d456e9cae 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -169,7 +169,7 @@ struct LocalDerivationGoal : public DerivationGoal /* Check that the derivation outputs all exist and register them as valid. */ - void registerOutputs() override; + DrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index c1bb1941d..31e6dbc9f 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -6,7 +6,7 @@ namespace nix { PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca) - : Goal(worker) + : Goal(worker, DerivedPath::Opaque { storePath }) , storePath(storePath) , repair(repair) , ca(ca) @@ -24,6 +24,13 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } +void PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status) +{ + buildResult.status = status; + amDone(result); +} + + void PathSubstitutionGoal::work() { (this->*state)(); @@ -38,7 +45,7 @@ void PathSubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); + done(ecSuccess, BuildResult::AlreadyValid); return; } @@ -65,7 +72,7 @@ void PathSubstitutionGoal::tryNext() /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); + done(substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters); if (substituterFailed) { worker.failedSubstitutions++; @@ -163,7 +170,9 @@ void PathSubstitutionGoal::referencesValid() if (nrFailed > 0) { debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); + done( + nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, + BuildResult::DependencyFailed); return; } @@ -268,7 +277,7 @@ void PathSubstitutionGoal::finished() worker.updateProgress(); - amDone(ecSuccess); + done(ecSuccess, BuildResult::Substituted); } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index d8399d2a8..946f13841 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -53,6 +53,8 @@ struct PathSubstitutionGoal : public Goal /* Content address for recomputing store path */ std::optional<ContentAddress> ca; + void done(ExitCode result, BuildResult::Status status); + public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); ~PathSubstitutionGoal(); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 9d4f6b4a4..e6760664c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -3,6 +3,7 @@ #include "worker-protocol.hh" #include "build-result.hh" #include "store-api.hh" +#include "gc-store.hh" #include "path-with-outputs.hh" #include "finally.hh" #include "archive.hh" @@ -531,6 +532,25 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } + case wopBuildPathsWithResults: { + auto drvs = readDerivedPaths(*store, clientVersion, from); + BuildMode mode = bmNormal; + mode = (BuildMode) readInt(from); + + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ + if (mode == bmRepair && !trusted) + throw Error("repairing is not allowed because you are not in 'trusted-users'"); + + logger->startWork(); + auto results = store->buildPathsWithResults(drvs, mode); + logger->stopWork(); + + worker_proto::write(*store, to, results); + + break; + } + case wopBuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; @@ -623,9 +643,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopAddIndirectRoot: { Path path = absPath(readString(from)); + logger->startWork(); - store->addIndirectRoot(path); + auto & gcStore = requireGcStore(*store); + gcStore.addIndirectRoot(path); logger->stopWork(); + to << 1; break; } @@ -640,7 +663,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopFindRoots: { logger->startWork(); - Roots roots = store->findRoots(!trusted); + auto & gcStore = requireGcStore(*store); + Roots roots = gcStore.findRoots(!trusted); logger->stopWork(); size_t size = 0; @@ -671,7 +695,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); if (options.ignoreLiveness) throw Error("you are not allowed to ignore liveness"); - store->collectGarbage(options, results); + auto & gcStore = requireGcStore(*store); + gcStore.collectGarbage(options, results); logger->stopWork(); to << results.paths << results.bytesFreed << 0 /* obsolete */; diff --git a/src/libstore/gc-store.cc b/src/libstore/gc-store.cc new file mode 100644 index 000000000..3dbdec53b --- /dev/null +++ b/src/libstore/gc-store.cc @@ -0,0 +1,13 @@ +#include "gc-store.hh" + +namespace nix { + +GcStore & requireGcStore(Store & store) +{ + auto * gcStore = dynamic_cast<GcStore *>(&store); + if (!gcStore) + throw UsageError("Garbage collection not supported by this store"); + return *gcStore; +} + +} diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh new file mode 100644 index 000000000..829f70dc4 --- /dev/null +++ b/src/libstore/gc-store.hh @@ -0,0 +1,84 @@ +#pragma once + +#include "store-api.hh" + + +namespace nix { + + +typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots; + + +struct GCOptions +{ + /* Garbage collector operation: + + - `gcReturnLive': return the set of paths reachable from + (i.e. in the closure of) the roots. + + - `gcReturnDead': return the set of paths not reachable from + the roots. + + - `gcDeleteDead': actually delete the latter set. + + - `gcDeleteSpecific': delete the paths listed in + `pathsToDelete', insofar as they are not reachable. + */ + typedef enum { + gcReturnLive, + gcReturnDead, + gcDeleteDead, + gcDeleteSpecific, + } GCAction; + + GCAction action{gcDeleteDead}; + + /* If `ignoreLiveness' is set, then reachability from the roots is + ignored (dangerous!). However, the paths must still be + unreferenced *within* the store (i.e., there can be no other + store paths that depend on them). */ + bool ignoreLiveness{false}; + + /* For `gcDeleteSpecific', the paths to delete. */ + StorePathSet pathsToDelete; + + /* Stop after at least `maxFreed' bytes have been freed. */ + uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; +}; + + +struct GCResults +{ + /* Depending on the action, the GC roots, or the paths that would + be or have been deleted. */ + PathSet paths; + + /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the + number of bytes that would be or was freed. */ + uint64_t bytesFreed = 0; +}; + + +struct GcStore : public virtual Store +{ + /* Add an indirect root, which is merely a symlink to `path' from + /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed + to be a symlink to a store path. The garbage collector will + automatically remove the indirect root when it finds that + `path' has disappeared. */ + virtual void addIndirectRoot(const Path & path) = 0; + + /* Find the roots of the garbage collector. Each root is a pair + (link, storepath) where `link' is the path of the symlink + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) = 0; + + /* Perform a garbage collection. */ + virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; +}; + +GcStore & requireGcStore(Store & store); + +} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 16c2b90c6..dd34b19c6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -279,7 +279,7 @@ public: conn->to.flush(); - BuildResult status; + BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } }; status.status = (BuildResult::Status) readInt(conn->from); conn->from >> status.errorMsg; @@ -317,7 +317,7 @@ public: conn->to.flush(); - BuildResult result; + BuildResult result { .path = DerivedPath::Opaque { StorePath::dummy } }; result.status = (BuildResult::Status) readInt(conn->from); if (!result.success()) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index d34f0cb62..fbd49dc2c 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -1,6 +1,7 @@ #pragma once #include "store-api.hh" +#include "gc-store.hh" namespace nix { @@ -23,7 +24,7 @@ struct LocalFSStoreConfig : virtual StoreConfig "physical path to the Nix store"}; }; -class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store +class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store, virtual GcStore { public: diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1a278c9a8..70d225be3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -5,6 +5,7 @@ #include "pathlocks.hh" #include "store-api.hh" #include "local-fs-store.hh" +#include "gc-store.hh" #include "sync.hh" #include "util.hh" @@ -43,7 +44,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig }; -class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore +class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore { private: diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 97aa01b57..078c117bd 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -22,9 +22,9 @@ DerivedPath StorePathWithOutputs::toDerivedPath() const std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> ss) { - std::vector<DerivedPath> reqs; - for (auto & s : ss) reqs.push_back(s.toDerivedPath()); - return reqs; + std::vector<DerivedPath> reqs; + for (auto & s : ss) reqs.push_back(s.toDerivedPath()); + return reqs; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cbcb75c00..347e32094 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,6 +1,7 @@ #include "serialise.hh" #include "util.hh" #include "path-with-outputs.hh" +#include "gc-store.hh" #include "remote-fs-accessor.hh" #include "build-result.hh" #include "remote-store.hh" @@ -90,6 +91,35 @@ void write(const Store & store, Sink & out, const DrvOutput & drvOutput) } +BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _) +{ + auto path = worker_proto::read(store, from, Phantom<DerivedPath> {}); + BuildResult res { .path = path }; + res.status = (BuildResult::Status) readInt(from); + from + >> res.errorMsg + >> res.timesBuilt + >> res.isNonDeterministic + >> res.startTime + >> res.stopTime; + res.builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {}); + return res; +} + +void write(const Store & store, Sink & to, const BuildResult & res) +{ + worker_proto::write(store, to, res.path); + to + << res.status + << res.errorMsg + << res.timesBuilt + << res.isNonDeterministic + << res.startTime + << res.stopTime; + worker_proto::write(store, to, res.builtOutputs); +} + + std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _) { auto s = readString(from); @@ -746,17 +776,24 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons } } -void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) +void RemoteStore::copyDrvsFromEvalStore( + const std::vector<DerivedPath> & paths, + std::shared_ptr<Store> evalStore) { if (evalStore && evalStore.get() != this) { /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ RealisedPath::Set drvPaths2; - for (auto & i : drvPaths) + for (auto & i : paths) if (auto p = std::get_if<DerivedPath::Built>(&i)) drvPaths2.insert(p->drvPath); copyClosure(*evalStore, *this, drvPaths2); } +} + +void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) +{ + copyDrvsFromEvalStore(drvPaths, evalStore); auto conn(getConnection()); conn->to << wopBuildPaths; @@ -773,6 +810,91 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod readInt(conn->from); } +std::vector<BuildResult> RemoteStore::buildPathsWithResults( + const std::vector<DerivedPath> & paths, + BuildMode buildMode, + std::shared_ptr<Store> evalStore) +{ + copyDrvsFromEvalStore(paths, evalStore); + + std::optional<ConnectionHandle> conn_(getConnection()); + auto & conn = *conn_; + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { + conn->to << wopBuildPathsWithResults; + writeDerivedPaths(*this, conn, paths); + conn->to << buildMode; + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom<std::vector<BuildResult>> {}); + } else { + // Avoid deadlock. + conn_.reset(); + + // Note: this throws an exception if a build/substitution + // fails, but meh. + buildPaths(paths, buildMode, evalStore); + + std::vector<BuildResult> results; + + for (auto & path : paths) { + std::visit( + overloaded { + [&](const DerivedPath::Opaque & bo) { + results.push_back(BuildResult { + .status = BuildResult::Substituted, + .path = bo, + }); + }, + [&](const DerivedPath::Built & bfd) { + BuildResult res { + .status = BuildResult::Built, + .path = bfd, + }; + + OutputPathMap outputs; + auto drv = evalStore->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive + auto drvOutputs = drv.outputsAndOptPaths(*this); + for (auto & output : bfd.outputs) { + if (!outputHashes.count(output)) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + printStorePath(bfd.drvPath), output); + auto outputId = + DrvOutput{outputHashes.at(output), output}; + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + auto realisation = + queryRealisation(outputId); + if (!realisation) + throw Error( + "cannot operate on an output of unbuilt " + "content-addressed derivation '%s'", + outputId.to_string()); + res.builtOutputs.emplace(realisation->id, *realisation); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + assert(drvOutputs.count(output)); + assert(drvOutputs.at(output).second); + res.builtOutputs.emplace( + outputId, + Realisation { + .id = outputId, + .outPath = *drvOutputs.at(output).second + }); + } + } + + results.push_back(res); + } + }, + path.raw()); + } + + return results; + } +} + BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) @@ -782,7 +904,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res; + BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } }; res.status = (BuildResult::Status) readInt(conn->from); conn->from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b94216d31..9f6f50593 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -4,6 +4,7 @@ #include <string> #include "store-api.hh" +#include "gc-store.hh" namespace nix { @@ -29,7 +30,7 @@ struct RemoteStoreConfig : virtual StoreConfig /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public virtual RemoteStoreConfig, public virtual Store +class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore { public: @@ -96,6 +97,11 @@ public: void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override; + std::vector<BuildResult> buildPathsWithResults( + const std::vector<DerivedPath> & paths, + BuildMode buildMode, + std::shared_ptr<Store> evalStore) override; + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -170,6 +176,9 @@ private: std::atomic_bool failed{false}; + void copyDrvsFromEvalStore( + const std::vector<DerivedPath> & paths, + std::shared_ptr<Store> evalStore); }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8c57596d0..e99a3f2cb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -76,59 +76,6 @@ enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; const uint32_t exportMagic = 0x4558494e; -typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots; - - -struct GCOptions -{ - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ - typedef enum { - gcReturnLive, - gcReturnDead, - gcDeleteDead, - gcDeleteSpecific, - } GCAction; - - GCAction action{gcDeleteDead}; - - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ - bool ignoreLiveness{false}; - - /* For `gcDeleteSpecific', the paths to delete. */ - StorePathSet pathsToDelete; - - /* Stop after at least `maxFreed' bytes have been freed. */ - uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; -}; - - -struct GCResults -{ - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ - PathSet paths; - - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ - uint64_t bytesFreed = 0; -}; - - enum BuildMode { bmNormal, bmRepair, bmCheck }; struct BuildResult; @@ -485,6 +432,16 @@ public: BuildMode buildMode = bmNormal, std::shared_ptr<Store> evalStore = nullptr); + /* Like `buildPaths()`, but return a vector of `BuildResult`s + corresponding to each element in `paths`. Note that in case of + a build/substitution error, this function won't throw an + exception, but return a `BuildResult` containing an error + message. */ + virtual std::vector<BuildResult> buildPathsWithResults( + const std::vector<DerivedPath> & paths, + BuildMode buildMode = bmNormal, + std::shared_ptr<Store> evalStore = nullptr); + /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). @@ -531,26 +488,6 @@ public: virtual void addTempRoot(const StorePath & path) { debug("not creating temporary root, store doesn't support GC"); } - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ - virtual void addIndirectRoot(const Path & path) - { unsupported("addIndirectRoot"); } - - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ - virtual Roots findRoots(bool censor) - { unsupported("findRoots"); } - - /* Perform a garbage collection. */ - virtual void collectGarbage(const GCOptions & options, GCResults & results) - { unsupported("collectGarbage"); } - /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or `nix-store --register-validity'. */ diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c8332afe6..87088a3ac 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 33) +#define PROTOCOL_VERSION (1 << 8 | 34) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -57,6 +57,7 @@ typedef enum { wopQueryRealisation = 43, wopAddMultipleToStore = 44, wopAddBuildLog = 45, + wopBuildPathsWithResults = 46, } WorkerOp; @@ -91,6 +92,7 @@ MAKE_WORKER_PROTO(, ContentAddress); MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); +MAKE_WORKER_PROTO(, BuildResult); MAKE_WORKER_PROTO(template<typename T>, std::vector<T>); MAKE_WORKER_PROTO(template<typename T>, std::set<T>); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index f970c0e9e..69aa0d094 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -328,8 +328,13 @@ MultiCommand::MultiCommand(const Commands & commands_) completions->add(name); } auto i = commands.find(s); - if (i == commands.end()) - throw UsageError("'%s' is not a recognised command", s); + if (i == commands.end()) { + std::set<std::string> commandNames; + for (auto & [name, _] : commands) + commandNames.insert(name); + auto suggestions = Suggestions::bestMatches(commandNames, s); + throw UsageError(suggestions, "'%s' is not a recognised command", s); + } command = {s, i->second()}; command->second->parent = this; }} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index dcd2f82a5..b2dfb35b2 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -282,6 +282,13 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s } } + auto suggestions = einfo.suggestions.trim(); + if (! suggestions.suggestions.empty()){ + oss << "Did you mean " << + suggestions.trim() << + "?" << std::endl; + } + // traces if (showTrace && !einfo.traces.empty()) { for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d55e1d701..600e94888 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -1,5 +1,6 @@ #pragma once +#include "suggestions.hh" #include "ref.hh" #include "types.hh" #include "fmt.hh" @@ -112,6 +113,8 @@ struct ErrorInfo { std::optional<ErrPos> errPos; std::list<Trace> traces; + Suggestions suggestions; + static std::optional<std::string> programName; }; @@ -141,6 +144,11 @@ public: : err { .level = lvlError, .msg = hintfmt(fs, args...) } { } + template<typename... Args> + BaseError(const Suggestions & sug, const Args & ... args) + : err { .level = lvlError, .msg = hintfmt(args...), .suggestions = sug } + { } + BaseError(hintformat hint) : err { .level = lvlError, .msg = hint } { } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 74ee2f063..cb2b15b41 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -266,51 +266,63 @@ static Logger::Fields getFields(nlohmann::json & json) return fields; } -bool handleJSONLogMessage(const std::string & msg, - const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted) +std::optional<nlohmann::json> parseJSONMessage(const std::string & msg) { - if (!hasPrefix(msg, "@nix ")) return false; - + if (!hasPrefix(msg, "@nix ")) return std::nullopt; try { - auto json = nlohmann::json::parse(std::string(msg, 5)); - - std::string action = json["action"]; - - if (action == "start") { - auto type = (ActivityType) json["type"]; - if (trusted || type == actFileTransfer) - activities.emplace(std::piecewise_construct, - std::forward_as_tuple(json["id"]), - std::forward_as_tuple(*logger, (Verbosity) json["level"], type, - json["text"], getFields(json["fields"]), act.id)); - } + return nlohmann::json::parse(std::string(msg, 5)); + } catch (std::exception & e) { + printError("bad JSON log message from builder: %s", e.what()); + } + return std::nullopt; +} - else if (action == "stop") - activities.erase((ActivityId) json["id"]); +bool handleJSONLogMessage(nlohmann::json & json, + const Activity & act, std::map<ActivityId, Activity> & activities, + bool trusted) +{ + std::string action = json["action"]; + + if (action == "start") { + auto type = (ActivityType) json["type"]; + if (trusted || type == actFileTransfer) + activities.emplace(std::piecewise_construct, + std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity) json["level"], type, + json["text"], getFields(json["fields"]), act.id)); + } - else if (action == "result") { - auto i = activities.find((ActivityId) json["id"]); - if (i != activities.end()) - i->second.result((ResultType) json["type"], getFields(json["fields"])); - } + else if (action == "stop") + activities.erase((ActivityId) json["id"]); - else if (action == "setPhase") { - std::string phase = json["phase"]; - act.result(resSetPhase, phase); - } + else if (action == "result") { + auto i = activities.find((ActivityId) json["id"]); + if (i != activities.end()) + i->second.result((ResultType) json["type"], getFields(json["fields"])); + } - else if (action == "msg") { - std::string msg = json["msg"]; - logger->log((Verbosity) json["level"], msg); - } + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); + } - } catch (std::exception & e) { - printError("bad JSON log message from builder: %s", e.what()); + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity) json["level"], msg); } return true; } +bool handleJSONLogMessage(const std::string & msg, + const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted) +{ + auto json = parseJSONMessage(msg); + if (!json) return false; + + return handleJSONLogMessage(*json, act, activities, trusted); +} + Activity::~Activity() { try { diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index bd28036ae..6f81b92de 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -4,6 +4,8 @@ #include "error.hh" #include "config.hh" +#include <nlohmann/json_fwd.hpp> + namespace nix { typedef enum { @@ -166,6 +168,12 @@ Logger * makeSimpleLogger(bool printBuildLogs = true); Logger * makeJSONLogger(Logger & prevLogger); +std::optional<nlohmann::json> parseJSONMessage(const std::string & msg); + +bool handleJSONLogMessage(nlohmann::json & json, + const Activity & act, std::map<ActivityId, Activity> & activities, + bool trusted); + bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map<ActivityId, Activity> & activities, bool trusted); diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc new file mode 100644 index 000000000..9510a5f0c --- /dev/null +++ b/src/libutil/suggestions.cc @@ -0,0 +1,114 @@ +#include "suggestions.hh" +#include "ansicolor.hh" +#include "util.hh" +#include <algorithm> + +namespace nix { + +int levenshteinDistance(std::string_view first, std::string_view second) +{ + // Implementation borrowed from + // https://en.wikipedia.org/wiki/Levenshtein_distance#Iterative_with_two_matrix_rows + + int m = first.size(); + int n = second.size(); + + auto v0 = std::vector<int>(n+1); + auto v1 = std::vector<int>(n+1); + + for (auto i = 0; i <= n; i++) + v0[i] = i; + + for (auto i = 0; i < m; i++) { + v1[0] = i+1; + + for (auto j = 0; j < n; j++) { + auto deletionCost = v0[j+1] + 1; + auto insertionCost = v1[j] + 1; + auto substitutionCost = first[i] == second[j] ? v0[j] : v0[j] + 1; + v1[j+1] = std::min({deletionCost, insertionCost, substitutionCost}); + } + + std::swap(v0, v1); + } + + return v0[n]; +} + +Suggestions Suggestions::bestMatches ( + std::set<std::string> allMatches, + std::string query) +{ + std::set<Suggestion> res; + for (const auto & possibleMatch : allMatches) { + res.insert(Suggestion { + .distance = levenshteinDistance(query, possibleMatch), + .suggestion = possibleMatch, + }); + } + return Suggestions { res }; +} + +Suggestions Suggestions::trim(int limit, int maxDistance) const +{ + std::set<Suggestion> res; + + int count = 0; + + for (auto & elt : suggestions) { + if (count >= limit || elt.distance > maxDistance) + break; + count++; + res.insert(elt); + } + + return Suggestions{res}; +} + +std::string Suggestion::to_string() const +{ + return ANSI_WARNING + filterANSIEscapes(suggestion) + ANSI_NORMAL; +} + +std::string Suggestions::to_string() const +{ + switch (suggestions.size()) { + case 0: + return ""; + case 1: + return suggestions.begin()->to_string(); + default: { + std::string res = "one of "; + auto iter = suggestions.begin(); + res += iter->to_string(); // Iter can’t be end() because the container isn’t null + iter++; + auto last = suggestions.end(); last--; + for ( ; iter != suggestions.end() ; iter++) { + res += (iter == last) ? " or " : ", "; + res += iter->to_string(); + } + return res; + } + } +} + +Suggestions & Suggestions::operator+=(const Suggestions & other) +{ + suggestions.insert( + other.suggestions.begin(), + other.suggestions.end() + ); + return *this; +} + +std::ostream & operator<<(std::ostream & str, const Suggestion & suggestion) +{ + return str << suggestion.to_string(); +} + +std::ostream & operator<<(std::ostream & str, const Suggestions & suggestions) +{ + return str << suggestions.to_string(); +} + +} diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh new file mode 100644 index 000000000..d54dd8e31 --- /dev/null +++ b/src/libutil/suggestions.hh @@ -0,0 +1,102 @@ +#pragma once + +#include "comparator.hh" +#include "types.hh" +#include <set> + +namespace nix { + +int levenshteinDistance(std::string_view first, std::string_view second); + +/** + * A potential suggestion for the cli interface. + */ +class Suggestion { +public: + int distance; // The smaller the better + std::string suggestion; + + std::string to_string() const; + + GENERATE_CMP(Suggestion, me->distance, me->suggestion) +}; + +class Suggestions { +public: + std::set<Suggestion> suggestions; + + std::string to_string() const; + + Suggestions trim( + int limit = 5, + int maxDistance = 2 + ) const; + + static Suggestions bestMatches ( + std::set<std::string> allMatches, + std::string query + ); + + Suggestions& operator+=(const Suggestions & other); +}; + +std::ostream & operator<<(std::ostream & str, const Suggestion &); +std::ostream & operator<<(std::ostream & str, const Suggestions &); + +// Either a value of type `T`, or some suggestions +template<typename T> +class OrSuggestions { +public: + using Raw = std::variant<T, Suggestions>; + + Raw raw; + + T* operator ->() + { + return &**this; + } + + T& operator *() + { + return std::get<T>(raw); + } + + operator bool() const noexcept + { + return std::holds_alternative<T>(raw); + } + + OrSuggestions(T t) + : raw(t) + { + } + + OrSuggestions() + : raw(Suggestions{}) + { + } + + static OrSuggestions<T> failed(const Suggestions & s) + { + auto res = OrSuggestions<T>(); + res.raw = s; + return res; + } + + static OrSuggestions<T> failed() + { + return OrSuggestions<T>::failed(Suggestions{}); + } + + const Suggestions & getSuggestions() + { + static Suggestions noSuggestions; + if (const auto & suggestions = std::get_if<Suggestions>(&raw)) + return *suggestions; + else + return noSuggestions; + } + +}; + +} diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index cef3bd481..2ffdc2e9b 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -359,7 +359,7 @@ namespace nix { // constructing without access violation. ErrPos ep(invalid); - + // assignment without access violation. ep = invalid; diff --git a/src/libutil/tests/suggestions.cc b/src/libutil/tests/suggestions.cc new file mode 100644 index 000000000..279994abc --- /dev/null +++ b/src/libutil/tests/suggestions.cc @@ -0,0 +1,43 @@ +#include "suggestions.hh" +#include <gtest/gtest.h> + +namespace nix { + + struct LevenshteinDistanceParam { + std::string s1, s2; + int distance; + }; + + class LevenshteinDistanceTest : + public testing::TestWithParam<LevenshteinDistanceParam> { + }; + + TEST_P(LevenshteinDistanceTest, CorrectlyComputed) { + auto params = GetParam(); + + ASSERT_EQ(levenshteinDistance(params.s1, params.s2), params.distance); + ASSERT_EQ(levenshteinDistance(params.s2, params.s1), params.distance); + } + + INSTANTIATE_TEST_SUITE_P(LevenshteinDistance, LevenshteinDistanceTest, + testing::Values( + LevenshteinDistanceParam{"foo", "foo", 0}, + LevenshteinDistanceParam{"foo", "", 3}, + LevenshteinDistanceParam{"", "", 0}, + LevenshteinDistanceParam{"foo", "fo", 1}, + LevenshteinDistanceParam{"foo", "oo", 1}, + LevenshteinDistanceParam{"foo", "fao", 1}, + LevenshteinDistanceParam{"foo", "abc", 3} + ) + ); + + TEST(Suggestions, Trim) { + auto suggestions = Suggestions::bestMatches({"foooo", "bar", "fo", "gao"}, "foo"); + auto onlyOne = suggestions.trim(1); + ASSERT_EQ(onlyOne.suggestions.size(), 1); + ASSERT_TRUE(onlyOne.suggestions.begin()->suggestion == "fo"); + + auto closest = suggestions.trim(999, 2); + ASSERT_EQ(closest.suggestions.size(), 3); + } +} diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 48c030b18..4b28ea6a4 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,4 +1,5 @@ #include "store-api.hh" +#include "gc-store.hh" #include "profiles.hh" #include "shared.hh" #include "globals.hh" @@ -80,10 +81,11 @@ static int main_nix_collect_garbage(int argc, char * * argv) // Run the actual garbage collector. if (!dryRun) { auto store = openStore(); + auto & gcStore = requireGcStore(*store); options.action = GCOptions::gcDeleteDead; GCResults results; PrintFreed freed(true, results); - store->collectGarbage(options, results); + gcStore.collectGarbage(options, results); } return 0; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 1ebc177f5..8ebaf9387 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -3,6 +3,7 @@ #include "dotgraph.hh" #include "globals.hh" #include "build-result.hh" +#include "gc-store.hh" #include "local-store.hh" #include "monitor-fd.hh" #include "serve-protocol.hh" @@ -428,11 +429,12 @@ static void opQuery(Strings opFlags, Strings opArgs) store->computeFSClosure( args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); - Roots roots = store->findRoots(false); + auto & gcStore = requireGcStore(*store); + Roots roots = gcStore.findRoots(false); for (auto & [target, links] : roots) if (referrers.find(target) != referrers.end()) for (auto & link : links) - cout << fmt("%1% -> %2%\n", link, store->printStorePath(target)); + cout << fmt("%1% -> %2%\n", link, gcStore.printStorePath(target)); break; } @@ -588,20 +590,22 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); + auto & gcStore = requireGcStore(*store); + if (printRoots) { - Roots roots = store->findRoots(false); + Roots roots = gcStore.findRoots(false); std::set<std::pair<Path, StorePath>> roots2; // Transpose and sort the roots. for (auto & [target, links] : roots) for (auto & link : links) roots2.emplace(link, target); for (auto & [link, target] : roots2) - std::cout << link << " -> " << store->printStorePath(target) << "\n"; + std::cout << link << " -> " << gcStore.printStorePath(target) << "\n"; } else { PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); - store->collectGarbage(options, results); + gcStore.collectGarbage(options, results); if (options.action != GCOptions::gcDeleteDead) for (auto & i : results.paths) @@ -625,9 +629,11 @@ static void opDelete(Strings opFlags, Strings opArgs) for (auto & i : opArgs) options.pathsToDelete.insert(store->followLinksToStorePath(i)); + auto & gcStore = requireGcStore(*store); + GCResults results; PrintFreed freed(true, results); - store->collectGarbage(options, results); + gcStore.collectGarbage(options, results); } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 144f8f886..47a380238 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -706,9 +706,14 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto [cursor, attrPath] = installable.getCursor(*evalState); - auto templateDir = cursor->getAttr("path")->getString(); - - assert(store->isInStore(templateDir)); + auto templateDirAttr = cursor->getAttr("path"); + auto templateDir = templateDirAttr->getString(); + + if (!store->isInStore(templateDir)) + throw TypeError( + "'%s' was not found in the Nix store\n" + "If you've set '%s' to a string, try using a path instead.", + templateDir, templateDirAttr->getAttrPathStr()); std::vector<Path> files; diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 5c0d44c68..3a51a13e6 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -800,7 +800,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m else printStringValue(str, i.first.c_str()); str << " = "; - if (seen.find(i.second) != seen.end()) + if (seen.count(i.second)) str << "«repeated»"; else try { diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index e4a3cb554..aa7a8b12f 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "gc-store.hh" using namespace nix; @@ -32,12 +33,14 @@ struct CmdStoreDelete : StorePathsCommand void run(ref<Store> store, std::vector<StorePath> && storePaths) override { + auto & gcStore = requireGcStore(*store); + for (auto & path : storePaths) options.pathsToDelete.insert(path); GCResults results; PrintFreed freed(true, results); - store->collectGarbage(options, results); + gcStore.collectGarbage(options, results); } }; diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index a2d74066e..21718dc0c 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "gc-store.hh" using namespace nix; @@ -33,10 +34,12 @@ struct CmdStoreGC : StoreCommand, MixDryRun void run(ref<Store> store) override { + auto & gcStore = requireGcStore(*store); + options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; GCResults results; PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); - store->collectGarbage(options, results); + gcStore.collectGarbage(options, results); } }; diff --git a/tests/build-hook-ca-fixed.nix b/tests/build-hook-ca-fixed.nix index ec7171ac9..4cb9e85d1 100644 --- a/tests/build-hook-ca-fixed.nix +++ b/tests/build-hook-ca-fixed.nix @@ -11,13 +11,13 @@ let args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; outputHashMode = "recursive"; outputHashAlgo = "sha256"; - } // removeAttrs args ["builder" "meta"]) - // { meta = args.meta or {}; }; + } // removeAttrs args ["builder" "meta" "passthru"]) + // { meta = args.meta or {}; passthru = args.passthru or {}; }; input1 = mkDerivation { shell = busybox; name = "build-remote-input-1"; - buildCommand = "echo FOO > $out"; + buildCommand = "echo hi-input1; echo FOO > $out"; requiredSystemFeatures = ["foo"]; outputHash = "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="; }; @@ -25,7 +25,7 @@ let input2 = mkDerivation { shell = busybox; name = "build-remote-input-2"; - buildCommand = "echo BAR > $out"; + buildCommand = "echo hi; echo BAR > $out"; requiredSystemFeatures = ["bar"]; outputHash = "sha256-XArauVH91AVwP9hBBQNlkX9ccuPpSYx9o0zeIHb6e+Q="; }; @@ -34,6 +34,7 @@ let shell = busybox; name = "build-remote-input-3"; buildCommand = '' + echo hi-input3 read x < ${input2} echo $x BAZ > $out ''; @@ -46,6 +47,7 @@ in mkDerivation { shell = busybox; name = "build-remote"; + passthru = { inherit input1 input2 input3; }; buildCommand = '' read x < ${input1} diff --git a/tests/build-hook.nix b/tests/build-hook.nix index eb16676f0..643334caa 100644 --- a/tests/build-hook.nix +++ b/tests/build-hook.nix @@ -9,20 +9,20 @@ let inherit system; builder = busybox; args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; - } // removeAttrs args ["builder" "meta"]) - // { meta = args.meta or {}; }; + } // removeAttrs args ["builder" "meta" "passthru"]) + // { meta = args.meta or {}; passthru = args.passthru or {}; }; input1 = mkDerivation { shell = busybox; name = "build-remote-input-1"; - buildCommand = "echo FOO > $out"; + buildCommand = "echo hi-input1; echo FOO > $out"; requiredSystemFeatures = ["foo"]; }; input2 = mkDerivation { shell = busybox; name = "build-remote-input-2"; - buildCommand = "echo BAR > $out"; + buildCommand = "echo hi; echo BAR > $out"; requiredSystemFeatures = ["bar"]; }; @@ -30,6 +30,7 @@ let shell = busybox; name = "build-remote-input-3"; buildCommand = '' + echo hi-input3 read x < ${input2} echo $x BAZ > $out ''; @@ -41,6 +42,7 @@ in mkDerivation { shell = busybox; name = "build-remote"; + passthru = { inherit input1 input2 input3; }; buildCommand = '' read x < ${input1} diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 806c6d261..094366872 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -54,6 +54,14 @@ nix path-info --store $TEST_ROOT/machine3 --all \ | grep -v builder-build-remote-input-2.sh \ | grep builder-build-remote-input-3.sh + +# Temporarily disabled because of https://github.com/NixOS/nix/issues/6209 +if [[ -z "$CONTENT_ADDRESSED" ]]; then + for i in input1 input3; do + nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i + done +fi + # Behavior of keep-failed out="$(nix-build 2>&1 failing.nix \ --builders "$(join_by '; ' "${builders[@]}")" \ diff --git a/tests/local.mk b/tests/local.mk index c3a6aa1fc..8032fc38a 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -92,8 +92,9 @@ nix_tests = \ bash-profile.sh \ pass-as-file.sh \ describe-stores.sh \ - store-ping.sh \ - nix-profile.sh + nix-profile.sh \ + suggestions.sh \ + store-ping.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/suggestions.sh b/tests/suggestions.sh new file mode 100644 index 000000000..16a5a7004 --- /dev/null +++ b/tests/suggestions.sh @@ -0,0 +1,36 @@ +source common.sh + +clearStore + +cd "$TEST_HOME" + +cat <<EOF > flake.nix +{ + outputs = a: { + packages.$system = { + foo = 1; + fo1 = 1; + fo2 = 1; + fooo = 1; + foooo = 1; + fooooo = 1; + fooooo1 = 1; + fooooo2 = 1; + fooooo3 = 1; + fooooo4 = 1; + fooooo5 = 1; + fooooo6 = 1; + }; + }; +} +EOF + +# Probable typo in the requested attribute path. Suggest some close possibilities +NIX_BUILD_STDERR_WITH_SUGGESTIONS=$(! nix build .\#fob 2>&1 1>/dev/null) +[[ "$NIX_BUILD_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ + fail "The nix build stderr should suggest the three closest possiblities" + +# None of the possible attributes is close to `bar`, so shouldn’t suggest anything +NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) +[[ ! "$NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION" =~ "Did you mean" ]] || \ + fail "The nix build stderr shouldn’t suggest anything if there’s nothing relevant to suggest" |