diff options
Diffstat (limited to 'src')
63 files changed, 1343 insertions, 348 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 2fb17d06f..c1f03a8ef 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -322,7 +322,12 @@ connected: throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); - auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } }); + auto res = sshStore->buildPathsWithResults({ + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All {}, + } + }); // One path to build should produce exactly one build result assert(res.size() == 1); optResult = std::move(res[0]); diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index db9c440e3..c6cc7fa9c 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -8,13 +8,39 @@ namespace nix { -nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = store->printStorePath(path); +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ } - return res; +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::pair<std::string, StorePath> +CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE std::map<std::string, StorePath> +CMP(SingleBuiltPath, BuiltPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +StorePath SingleBuiltPath::outPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) { return p.path; }, + [](const SingleBuiltPath::Built & b) { return b.output.second; }, + }, raw() + ); } StorePathSet BuiltPath::outPaths() const @@ -32,6 +58,40 @@ StorePathSet BuiltPath::outPaths() const ); } +nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + for (const auto & [outputName, outputPath] : outputs) { + res["outputs"][outputName] = store.printStorePath(outputPath); + } + return res; +} + +nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + auto & [outputName, outputPath] = output; + res["output"] = outputName; + res["outputPath"] = store.printStorePath(outputPath); + return res; +} + +nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json BuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const { RealisedPath::Set res; @@ -40,7 +100,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { auto drvHashes = - staticOutputHashes(store, store.readDerivation(p.drvPath)); + staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto& [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled( Xp::CaDerivations)) { @@ -48,7 +108,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const if (!drvOutput) throw Error( "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath), outputName); + store.printStorePath(p.drvPath->outPath()), outputName); auto thisRealisation = store.queryRealisation( DrvOutput{*drvOutput, outputName}); assert(thisRealisation); // We’ve built it, so we must diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 744e8090b..747bcc440 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -3,19 +3,60 @@ namespace nix { +struct SingleBuiltPath; + +struct SingleBuiltPathBuilt { + ref<SingleBuiltPath> drvPath; + std::pair<std::string, StorePath> output; + + std::string to_string(const Store & store) const; + static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; + + DECLARE_CMP(SingleBuiltPathBuilt); +}; + +using _SingleBuiltPathRaw = std::variant< + DerivedPathOpaque, + SingleBuiltPathBuilt +>; + +struct SingleBuiltPath : _SingleBuiltPathRaw { + using Raw = _SingleBuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleBuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } + + StorePath outPath() const; + + static SingleBuiltPath parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; +}; + +static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath) +{ + return make_ref<SingleBuiltPath>(SingleBuiltPath::Opaque { drvPath }); +} + /** * A built derived path with hints in the form of optional concrete output paths. * * See 'BuiltPath' for more an explanation. */ struct BuiltPathBuilt { - StorePath drvPath; + ref<SingleBuiltPath> drvPath; std::map<std::string, StorePath> outputs; - nlohmann::json toJSON(ref<Store> store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view); + std::string to_string(const Store & store) const; + static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; - GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -41,6 +82,7 @@ struct BuiltPath : _BuiltPathRaw { StorePathSet outPaths() const; RealisedPath::Set toRealisedPaths(Store & store) const; + nlohmann::json toJSON(const Store & store) const; }; typedef std::vector<BuiltPath> BuiltPaths; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index b35ca2910..2f89eee02 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -92,7 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() for (auto & [drvPath, outputs] : byDrvPath) res.push_back({ .path = DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = outputs, }, .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value { diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 6ecf54b7c..b45641e8a 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -18,14 +18,7 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() std::optional<StorePath> InstallableDerivedPath::getStorePath() { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; - }, - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - }, derivedPath.raw()); + return derivedPath.getBaseStorePath(); } InstallableDerivedPath InstallableDerivedPath::parse( @@ -42,7 +35,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( // Remove this prior to stabilizing the new CLI. if (storePath.isDerivation()) { auto oldDerivedPath = DerivedPath::Built { - .drvPath = storePath, + .drvPath = makeConstantStorePathRef(storePath), .outputs = OutputsSpec::All { }, }; warn( @@ -55,8 +48,10 @@ InstallableDerivedPath InstallableDerivedPath::parse( }, // If the user did use ^, we just do exactly what is written. [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + auto drv = make_ref<SingleDerivedPath>(SingleDerivedPath::parse(*store, prefix)); + drvRequireExperiment(*drv); return DerivedPath::Built { - .drvPath = store->parseStorePath(prefix), + .drvPath = std::move(drv), .outputs = outputSpec, }; }, diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 4da9b131b..1b169c3bd 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -118,7 +118,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() return {{ .path = DerivedPath::Built { - .drvPath = std::move(drvPath), + .drvPath = makeConstantStorePathRef(std::move(drvPath)), .outputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { std::set<std::string> outputsToInstall; diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 1eff293cc..08ad35105 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -55,7 +55,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths else if (v.type() == nString) { return {{ - .path = state->coerceToDerivedPath(pos, v, errorCtx), + .path = DerivedPath::fromSingle( + state->coerceToSingleDerivedPath(pos, v, errorCtx)), .info = make_ref<ExtraPathInfo>(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 9d593a01f..4c7d134ec 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -515,6 +515,30 @@ ref<Installable> SourceExprCommand::parseInstallable( return installables.front(); } +static SingleBuiltPath getBuiltPath(ref<Store> evalStore, ref<Store> store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { + return SingleBuiltPath::Opaque { bo.path }; + }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. + SingleDerivedPath::Built truncatedBfd { + .drvPath = makeConstantStorePathRef(drvPath.outPath()), + .output = bfd.output, + }; + auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); + return SingleBuiltPath::Built { + .drvPath = make_ref<SingleBuiltPath>(std::move(drvPath)), + .output = { bfd.output, outputPath }, + }; + }, + }, + b.raw()); +} + std::vector<BuiltPathWithResult> Installable::build( ref<Store> evalStore, ref<Store> store, @@ -568,7 +592,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build [&](const DerivedPath::Built & bfd) { auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { @@ -597,7 +624,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build for (auto & [outputName, realisation] : buildResult.builtOutputs) outputs.emplace(outputName, realisation.outPath); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info, .result = buildResult}}); }, @@ -691,7 +721,7 @@ StorePathSet Installable::toDerivations( : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { - drvPaths.insert(bfd.drvPath); + drvPaths.insert(resolveDerivedPath(*store, *bfd.drvPath)); }, }, b.path.raw()); diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index d15162e76..ea7d0e2ec 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -648,7 +648,7 @@ bool NixRepl::processLine(std::string line) if (command == ":b" || command == ":bl") { state->store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 9e734e654..6a60ba87b 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -599,7 +599,7 @@ string_t AttrCursor::getStringWithContext() return d.drvPath; }, [&](const NixStringContextElem::Built & b) -> const StorePath & { - return b.drvPath; + return b.drvPath->getBaseStorePath(); }, [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index e57de6c1d..3e521b732 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1031,17 +1031,18 @@ void EvalState::mkOutputString( Value & value, const StorePath & drvPath, const std::string outputName, - std::optional<StorePath> optOutputPath) + std::optional<StorePath> optOutputPath, + const ExperimentalFeatureSettings & xpSettings) { value.mkString( optOutputPath ? store->printStorePath(*std::move(optOutputPath)) /* Downstream we would substitute this for an actual path once we build the floating CA derivation */ - : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(), + : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(), NixStringContext { NixStringContextElem::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .output = outputName, } }); @@ -2298,7 +2299,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon } -std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) +std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) { NixStringContext context; auto s = forceString(v, context, pos, errorCtx); @@ -2309,21 +2310,16 @@ std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked s, csize) .withTrace(pos, errorCtx).debugThrow<EvalError>(); auto derivedPath = std::visit(overloaded { - [&](NixStringContextElem::Opaque && o) -> DerivedPath { - return DerivedPath::Opaque { - .path = std::move(o.path), - }; + [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { + return std::move(o); }, - [&](NixStringContextElem::DrvDeep &&) -> DerivedPath { + [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath { error( "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", s).withTrace(pos, errorCtx).debugThrow<EvalError>(); }, - [&](NixStringContextElem::Built && b) -> DerivedPath { - return DerivedPath::Built { - .drvPath = std::move(b.drvPath), - .outputs = OutputsSpec::Names { std::move(b.output) }, - }; + [&](NixStringContextElem::Built && b) -> SingleDerivedPath { + return std::move(b); }, }, ((NixStringContextElem &&) *context.begin()).raw()); return { @@ -2333,12 +2329,12 @@ std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked } -DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) +SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) { - auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx); + auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx); auto s = s_; std::visit(overloaded { - [&](const DerivedPath::Opaque & o) { + [&](const SingleDerivedPath::Opaque & o) { auto sExpected = store->printStorePath(o.path); if (s != sExpected) error( @@ -2346,25 +2342,27 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str s, sExpected) .withTrace(pos, errorCtx).debugThrow<EvalError>(); }, - [&](const DerivedPath::Built & b) { - // TODO need derived path with single output to make this - // total. Will add as part of RFC 92 work and then this is - // cleaned up. - auto output = *std::get<OutputsSpec::Names>(b.outputs).begin(); - - auto drv = store->readDerivation(b.drvPath); - auto i = drv.outputs.find(output); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output); - auto optOutputPath = i->second.path(*store, drv.name, output); - // This is testing for the case of CA derivations - auto sExpected = optOutputPath - ? store->printStorePath(*optOutputPath) - : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render(); + [&](const SingleDerivedPath::Built & b) { + auto sExpected = std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + auto drv = store->readDerivation(o.path); + auto i = drv.outputs.find(b.output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); + auto optOutputPath = i->second.path(*store, drv.name, b.output); + // This is testing for the case of CA derivations + return optOutputPath + ? store->printStorePath(*optOutputPath) + : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); + }, + [&](const SingleDerivedPath::Built & o) { + return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); + }, + }, b.drvPath->raw()); if (s != sExpected) error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", - s, output, store->printStorePath(b.drvPath), sExpected) + s, b.output, b.drvPath->to_string(*store), sExpected) .withTrace(pos, errorCtx).debugThrow<EvalError>(); } }, derivedPath.raw()); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 887b9cb97..0268a2a12 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -22,7 +22,7 @@ namespace nix { class Store; class EvalState; class StorePath; -struct DerivedPath; +struct SingleDerivedPath; enum RepairFlag : bool; @@ -532,12 +532,12 @@ public: StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** - * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only. + * Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only. */ - std::pair<DerivedPath, std::string_view> coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); + std::pair<SingleDerivedPath, std::string_view> coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); /** - * Coerce to `DerivedPath`. + * Coerce to `SingleDerivedPath`. * * Must be a string which is either a literal store path or a * "placeholder (see `DownstreamPlaceholder`). @@ -551,7 +551,7 @@ public: * source of truth, and ultimately tells us what we want, and then * we ensure the string corresponds to it. */ - DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); + SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); public: @@ -689,12 +689,15 @@ public: * be passed if and only if output store object is input-addressed. * Will be printed to form string if passed, otherwise a placeholder * will be used (see `DownstreamPlaceholder`). + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ void mkOutputString( Value & value, const StorePath & drvPath, const std::string outputName, - std::optional<StorePath> optOutputPath); + std::optional<StorePath> optOutputPath, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 08adbe0c9..d3fa1d557 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -105,7 +105,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), ""), + FlakeRef(Input::fromURL(parsedURL, isFlake), ""), percentDecode(match.str(6))); } @@ -176,7 +176,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( parsedURL.query.insert_or_assign("shallow", "1"); return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")), fragment); } @@ -204,7 +204,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( std::string fragment; std::swap(fragment, parsedURL.fragment); - auto input = Input::fromURL(parsedURL); + auto input = Input::fromURL(parsedURL, isFlake); input.parent = baseDir; return std::make_pair( diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ddf529b9e..54943b481 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -56,7 +56,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) .drvPath = b.drvPath, .outputs = OutputsSpec::Names { b.output }, }); - ensureValid(b.drvPath); + ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { auto ctxS = store->printStorePath(o.path); @@ -77,29 +77,32 @@ StringMap EvalState::realiseContext(const NixStringContext & context) if (!evalSettings.enableImportFromDerivation) debugThrowLastTrace(Error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - store->printStorePath(drvs.begin()->drvPath))); + drvs.begin()->to_string(*store))); /* Build/substitute the context. */ std::vector<DerivedPath> buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); store->buildPaths(buildReqs); - /* Get all the output paths corresponding to the placeholders we had */ for (auto & drv : drvs) { auto outputs = resolveDerivedPath(*store, drv); for (auto & [outputName, outputPath] : outputs) { - res.insert_or_assign( - DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), - store->printStorePath(outputPath) - ); - } - } - - /* Add the output of this derivations to the allowed - paths. */ - if (allowedPaths) { - for (auto & [_placeholder, outputPath] : res) { - allowPath(store->toRealPath(outputPath)); + /* Add the output of this derivations to the allowed + paths. */ + if (allowedPaths) { + allowPath(outputPath); + } + /* Get all the output paths corresponding to the placeholders we had */ + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + res.insert_or_assign( + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = drv.drvPath, + .output = outputName, + }).render(), + store->printStorePath(outputPath) + ); + } } } @@ -1252,7 +1255,10 @@ drvName, Bindings * attrs, Value & v) } }, [&](const NixStringContextElem::Built & b) { - drv.inputDrvs[b.drvPath].insert(b.output); + if (auto * p = std::get_if<DerivedPath::Opaque>(&*b.drvPath)) + drv.inputDrvs[p->path].insert(b.output); + else + throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported"); }, [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 8b3468009..bfc731744 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -106,7 +106,10 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, contextInfos[std::move(d.drvPath)].allOutputs = true; }, [&](NixStringContextElem::Built && b) { - contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); + // FIXME should eventually show string context as is, no + // resolving here. + auto drvPath = resolveDerivedPath(*state.store, *b.drvPath); + contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output)); }, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; @@ -222,7 +225,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); context.emplace(NixStringContextElem::Built { - .drvPath = namePath, + .drvPath = makeConstantStorePathRef(namePath), .output = std::string { outputName }, }); } diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index 8210efef2..2a5ca64f6 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -21,12 +21,12 @@ TEST_F(DerivedPathExpressionTest, force_init) RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, prop_opaque_path_round_trip, - (const DerivedPath::Opaque & o)) + (const SingleDerivedPath::Opaque & o)) { auto * v = state.allocValue(); state.mkStorePathString(o.path, *v); - auto d = state.coerceToDerivedPath(noPos, *v, ""); - RC_ASSERT(DerivedPath { o } == d); + auto d = state.coerceToSingleDerivedPath(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { o } == d); } // TODO use DerivedPath::Built for parameter once it supports a single output @@ -37,14 +37,21 @@ RC_GTEST_FIXTURE_PROP( prop_built_path_placeholder_round_trip, (const StorePath & drvPath, const StorePathName & outputName)) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + auto * v = state.allocValue(); - state.mkOutputString(*v, drvPath, outputName.name, std::nullopt); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings); + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = makeConstantStorePathRef(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } RC_GTEST_FIXTURE_PROP( @@ -54,12 +61,12 @@ RC_GTEST_FIXTURE_PROP( { auto * v = state.allocValue(); state.mkOutputString(*v, drvPath, outputName.name, outPath); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = makeConstantStorePathRef(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } } /* namespace nix */ diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 0d9381577..c56b50b59 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -8,6 +8,8 @@ namespace nix { +// Test a few cases of invalid string context elements. + TEST(NixStringContextElemTest, empty_invalid) { EXPECT_THROW( NixStringContextElem::parse(""), @@ -38,6 +40,10 @@ TEST(NixStringContextElemTest, slash_invalid) { BadStorePath); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::Opaque`. + */ TEST(NixStringContextElemTest, opaque) { std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; auto elem = NixStringContextElem::parse(opaque); @@ -47,6 +53,10 @@ TEST(NixStringContextElemTest, opaque) { ASSERT_EQ(elem.to_string(), opaque); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::DrvDeep`. + */ TEST(NixStringContextElemTest, drvDeep) { std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(drvDeep); @@ -56,28 +66,62 @@ TEST(NixStringContextElemTest, drvDeep) { ASSERT_EQ(elem.to_string(), drvDeep); } -TEST(NixStringContextElemTest, built) { +/** + * Round trip (string <-> data structure) test for a simpler + * `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_opaque) { std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(built); auto * p = std::get_if<NixStringContextElem::Built>(&elem); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(5) }, + })); ASSERT_EQ(elem.to_string(), built); } +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built, mockXpSettings); + auto * p = std::get_if<NixStringContextElem::Built>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + auto * drvPath = std::get_if<SingleDerivedPath::Built>(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "bar"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(9) }, + })); + ASSERT_EQ(elem.to_string(), built); } -namespace rc { -using namespace nix; +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive string context element. + */ +TEST(NixStringContextElemTest, built_built_xp) { + ASSERT_THROW( + NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); +} -Gen<NixStringContextElem::Opaque> Arbitrary<NixStringContextElem::Opaque>::arbitrary() -{ - return gen::just(NixStringContextElem::Opaque { - .path = *gen::arbitrary<StorePath>(), - }); } +namespace rc { +using namespace nix; + Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary() { return gen::just(NixStringContextElem::DrvDeep { @@ -85,14 +129,6 @@ Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arb }); } -Gen<NixStringContextElem::Built> Arbitrary<NixStringContextElem::Built>::arbitrary() -{ - return gen::just(NixStringContextElem::Built { - .drvPath = *gen::arbitrary<StorePath>(), - .output = (*gen::arbitrary<StorePathName>()).name, - }); -} - Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary() { switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) { diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index f76fc76e4..d8116011e 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -4,29 +4,52 @@ namespace nix { -NixStringContextElem NixStringContextElem::parse(std::string_view s0) +NixStringContextElem NixStringContextElem::parse( + std::string_view s0, + const ExperimentalFeatureSettings & xpSettings) { std::string_view s = s0; + std::function<SingleDerivedPath()> parseRest; + parseRest = [&]() -> SingleDerivedPath { + // Case on whether there is a '!' + size_t index = s.find("!"); + if (index == std::string_view::npos) { + return SingleDerivedPath::Opaque { + .path = StorePath { s }, + }; + } else { + std::string output { s.substr(0, index) }; + // Advance string to parse after the '!' + s = s.substr(index + 1); + auto drv = make_ref<SingleDerivedPath>(parseRest()); + drvRequireExperiment(*drv, xpSettings); + return SingleDerivedPath::Built { + .drvPath = std::move(drv), + .output = std::move(output), + }; + } + }; + if (s.size() == 0) { throw BadNixStringContextElem(s0, "String context element should never be an empty string"); } + switch (s.at(0)) { case '!': { - s = s.substr(1); // advance string to parse after first ! - size_t index = s.find("!"); - // This makes index + 1 safe. Index can be the length (one after index - // of last character), so given any valid character index --- a - // successful find --- we can add one. - if (index == std::string_view::npos) { + // Advance string to parse after the '!' + s = s.substr(1); + + // Find *second* '!' + if (s.find("!") == std::string_view::npos) { throw BadNixStringContextElem(s0, "String content element beginning with '!' should have a second '!'"); } - return NixStringContextElem::Built { - .drvPath = StorePath { s.substr(index + 1) }, - .output = std::string(s.substr(0, index)), - }; + + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } case '=': { return NixStringContextElem::DrvDeep { @@ -34,33 +57,51 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0) }; } default: { - return NixStringContextElem::Opaque { - .path = StorePath { s }, - }; + // Ensure no '!' + if (s.find("!") != std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element not beginning with '!' should not have a second '!'"); + } + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } } } -std::string NixStringContextElem::to_string() const { - return std::visit(overloaded { +std::string NixStringContextElem::to_string() const +{ + std::string res; + + std::function<void(const SingleDerivedPath &)> toStringRest; + toStringRest = [&](auto & p) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + res += o.path.to_string(); + }, + [&](const SingleDerivedPath::Built & o) { + res += o.output; + res += '!'; + toStringRest(*o.drvPath); + }, + }, p.raw()); + }; + + std::visit(overloaded { [&](const NixStringContextElem::Built & b) { - std::string res; - res += '!'; - res += b.output; res += '!'; - res += b.drvPath.to_string(); - return res; + toStringRest(b); + }, + [&](const NixStringContextElem::Opaque & o) { + toStringRest(o); }, [&](const NixStringContextElem::DrvDeep & d) { - std::string res; res += '='; res += d.drvPath.to_string(); - return res; - }, - [&](const NixStringContextElem::Opaque & o) { - return std::string { o.path.to_string() }; }, }, raw()); + + return res; } } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 287ae08a9..a1b71695b 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -3,7 +3,7 @@ #include "util.hh" #include "comparator.hh" -#include "path.hh" +#include "derived-path.hh" #include <variant> @@ -31,11 +31,7 @@ public: * * Encoded as just the path: ‘<path>’. */ -struct NixStringContextElem_Opaque { - StorePath path; - - GENERATE_CMP(NixStringContextElem_Opaque, me->path); -}; +typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque; /** * Path to a derivation and its entire build closure. @@ -57,12 +53,7 @@ struct NixStringContextElem_DrvDeep { * * Encoded in the form ‘!<output>!<drvPath>’. */ -struct NixStringContextElem_Built { - StorePath drvPath; - std::string output; - - GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); -}; +typedef SingleDerivedPath::Built NixStringContextElem_Built; using _NixStringContextElem_Raw = std::variant< NixStringContextElem_Opaque, @@ -93,8 +84,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw { * - ‘<path>’ * - ‘=<path>’ * - ‘!<name>!<path>’ + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static NixStringContextElem parse(std::string_view s); + static NixStringContextElem parse( + std::string_view s, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); std::string to_string() const; }; diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index f86c0604e..e683b9f80 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -13,9 +13,9 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme) inputSchemes->push_back(std::move(inputScheme)); } -Input Input::fromURL(const std::string & url) +Input Input::fromURL(const std::string & url, bool requireTree) { - return fromURL(parseURL(url)); + return fromURL(parseURL(url), requireTree); } static void fixupInput(Input & input) @@ -31,10 +31,10 @@ static void fixupInput(Input & input) input.locked = true; } -Input Input::fromURL(const ParsedURL & url) +Input Input::fromURL(const ParsedURL & url, bool requireTree) { for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromURL(url); + auto res = inputScheme->inputFromURL(url, requireTree); if (res) { res->scheme = inputScheme; fixupInput(*res); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index d0738f619..6e10e9513 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -44,9 +44,9 @@ struct Input std::optional<Path> parent; public: - static Input fromURL(const std::string & url); + static Input fromURL(const std::string & url, bool requireTree = true); - static Input fromURL(const ParsedURL & url); + static Input fromURL(const ParsedURL & url, bool requireTree = true); static Input fromAttrs(Attrs && attrs); @@ -129,7 +129,7 @@ struct InputScheme virtual ~InputScheme() { } - virtual std::optional<Input> inputFromURL(const ParsedURL & url) const = 0; + virtual std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const = 0; virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index be5842d53..f8d89ab2f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -256,7 +256,7 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co struct GitInputScheme : InputScheme { - std::optional<Input> inputFromURL(const ParsedURL & url) const override + std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "git" && url.scheme != "git+http" && diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 80598e7f8..291f457f0 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -30,7 +30,7 @@ struct GitArchiveInputScheme : InputScheme virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0; - std::optional<Input> inputFromURL(const ParsedURL & url) const override + std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != type()) return {}; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index b99504a16..4874a43ff 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -7,7 +7,7 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional<Input> inputFromURL(const ParsedURL & url) const override + std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "flake") return {}; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 86e8f81f4..51fd1ed42 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -43,7 +43,7 @@ static std::string runHg(const Strings & args, const std::optional<std::string> struct MercurialInputScheme : InputScheme { - std::optional<Input> inputFromURL(const ParsedURL & url) const override + std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 61541e69d..01f1be978 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -6,7 +6,7 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional<Input> inputFromURL(const ParsedURL & url) const override + std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "path") return {}; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index a012234e0..107d38e92 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -194,11 +194,11 @@ struct CurlInputScheme : InputScheme || hasSuffix(path, ".tar.zst"); } - virtual bool isValidURL(const ParsedURL & url) const = 0; + virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0; - std::optional<Input> inputFromURL(const ParsedURL & _url) const override + std::optional<Input> inputFromURL(const ParsedURL & _url, bool requireTree) const override { - if (!isValidURL(_url)) + if (!isValidURL(_url, requireTree)) return std::nullopt; Input input; @@ -265,13 +265,13 @@ struct FileInputScheme : CurlInputScheme { const std::string inputType() const override { return "file"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : !hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == inputType() + : (!requireTree && !hasTarballExtension(url.path))); } std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override @@ -285,14 +285,14 @@ struct TarballInputScheme : CurlInputScheme { const std::string inputType() const override { return "tarball"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == inputType() + : (requireTree || hasTarballExtension(url.path))); } std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5e37f7ecb..8bdf2f367 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -65,7 +65,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -74,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -84,7 +84,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -95,7 +95,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -1490,7 +1490,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) for (auto & outputName : outputs->second) { auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = dg->drvPath, + .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 4aa4d6dca..e941b4e65 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -77,7 +77,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); return goal->getBuildResult(DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); } catch (Error & e) { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b7a27490c..920097680 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1172,6 +1172,19 @@ void LocalDerivationGoal::writeStructuredAttrs() } +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { @@ -1179,7 +1192,7 @@ static StorePath pathPartOfReq(const DerivedPath & req) return bo.path; }, [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; + return pathPartOfReq(*bfd.drvPath); }, }, req.raw()); } @@ -2307,7 +2320,6 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { - experimentalFeatureSettings.require(Xp::DiscardReferences); if (auto output = get(*udr, outputName)) { if (!output->is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index a9ca9cbbc..6779dbcf3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -111,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -265,7 +268,7 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) { - topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built{makeConstantStorePathRef(goal->drvPath), goal->wantedOutputs}); } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index b831b2fe5..f4e4980c2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -879,9 +879,11 @@ std::optional<BasicDerivation> Derivation::tryResolve( for (auto & [inputDrv, inputOutputs] : inputDrvs) { for (auto & outputName : inputOutputs) { if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { - inputRewrites.emplace( - DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), - store.printStorePath(*actualPath)); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + inputRewrites.emplace( + DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), + store.printStorePath(*actualPath)); + } resolved.inputSrcs.insert(*actualPath); } else { warn("output '%s' of input '%s' missing, aborting the resolving", @@ -993,6 +995,7 @@ DerivationOutput DerivationOutput::fromJSON( const ExperimentalFeatureSettings & xpSettings) { std::set<std::string_view> keys; + ensureType(_json, nlohmann::detail::value_t::object); auto json = (std::map<std::string, nlohmann::json>) _json; for (const auto & [key, _] : json) @@ -1097,36 +1100,51 @@ Derivation Derivation::fromJSON( const Store & store, const nlohmann::json & json) { + using nlohmann::detail::value_t; + Derivation res; - res.name = json["name"]; + ensureType(json, value_t::object); - { - auto & outputsObj = json["outputs"]; + res.name = ensureType(valueAt(json, "name"), value_t::string); + + try { + auto & outputsObj = ensureType(valueAt(json, "outputs"), value_t::object); for (auto & [outputName, output] : outputsObj.items()) { res.outputs.insert_or_assign( outputName, DerivationOutput::fromJSON(store, res.name, outputName, output)); } + } catch (Error & e) { + e.addTrace({}, "while reading key 'outputs'"); + throw; } - { - auto & inputsList = json["inputSrcs"]; + try { + auto & inputsList = ensureType(valueAt(json, "inputSrcs"), value_t::array); for (auto & input : inputsList) res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputSrcs'"); + throw; } - { - auto & inputDrvsObj = json["inputDrvs"]; - for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + try { + auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object); + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) { + ensureType(inputOutputs, value_t::array); res.inputDrvs[store.parseStorePath(inputDrvPath)] = static_cast<const StringSet &>(inputOutputs); + } + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputDrvs'"); + throw; } - res.platform = json["system"]; - res.builder = json["builder"]; - res.args = json["args"]; - res.env = json["env"]; + res.platform = ensureType(valueAt(json, "system"), value_t::string); + res.builder = ensureType(valueAt(json, "builder"), value_t::string); + res.args = ensureType(valueAt(json, "args"), value_t::array); + res.env = ensureType(valueAt(json, "env"), value_t::object); return res; } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 52d073f81..3594b7570 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -7,52 +7,133 @@ namespace nix { -nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const { +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ + } +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::string +CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE OutputsSpec +CMP(SingleDerivedPath, DerivedPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +{ + return store.printStorePath(path); +} + +nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["path"] = store->printStorePath(path); + res["drvPath"] = drvPath->toJSON(store); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); + res["output"] = output; + auto outputPathIter = outputMap.find(output); + if (outputPathIter == outputMap.end()) + res["outputPath"] = nullptr; + else if (std::optional p = outputPathIter->second) + res["outputPath"] = store.printStorePath(*p); + else + res["outputPath"] = nullptr; return res; } -nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const { +nlohmann::json DerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); + res["drvPath"] = drvPath->toJSON(store); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - const auto outputMap = store->queryPartialDerivationOutputMap(drvPath); + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); for (const auto & [output, outputPathOpt] : outputMap) { if (!outputs.contains(output)) continue; if (outputPathOpt) - res["outputs"][output] = store->printStorePath(*outputPathOpt); + res["outputs"][output] = store.printStorePath(*outputPathOpt); else res["outputs"][output] = nullptr; } return res; } +nlohmann::json SingleDerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json DerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); } +std::string SingleDerivedPath::Built::to_string(const Store & store) const +{ + return drvPath->to_string(store) + "^" + output; +} + +std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +{ + return drvPath->to_string(store) + "!" + output; +} + std::string DerivedPath::Built::to_string(const Store & store) const { - return store.printStorePath(drvPath) + return drvPath->to_string(store) + '^' + outputs.to_string(); } std::string DerivedPath::Built::to_string_legacy(const Store & store) const { - return store.printStorePath(drvPath) - + '!' + return drvPath->to_string_legacy(store) + + "!" + outputs.to_string(); } +std::string SingleDerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + std::string DerivedPath::to_string(const Store & store) const { + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + +std::string SingleDerivedPath::to_string_legacy(const Store & store) const +{ return std::visit(overloaded { - [&](const DerivedPath::Built & req) { return req.to_string(store); }, - [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const SingleDerivedPath::Opaque & req) { return req.to_string(store); }, }, this->raw()); } @@ -70,30 +151,156 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_ return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings) +{ + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque &) { + // plain drv path; no experimental features required. + }, + [&](const SingleDerivedPath::Built &) { + xpSettings.require(Xp::DynamicDerivations); + }, + }, drv.raw()); +} + +SingleDerivedPath::Built SingleDerivedPath::Built::parse( + const Store & store, ref<SingleDerivedPath> drv, + std::string_view output, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); + return { + .drvPath = drv, + .output = std::string { output }, + }; +} + +DerivedPath::Built DerivedPath::Built::parse( + const Store & store, ref<SingleDerivedPath> drv, + std::string_view outputsS, + const ExperimentalFeatureSettings & xpSettings) { + drvRequireExperiment(*drv, xpSettings); return { - .drvPath = store.parseStorePath(drvS), + .drvPath = drv, .outputs = OutputsSpec::parse(outputsS), }; } -static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) +static SingleDerivedPath parseWithSingle( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) { - size_t n = s.find(separator); + size_t n = s.rfind(separator); + return n == s.npos + ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s) + : (SingleDerivedPath) SingleDerivedPath::Built::parse(store, + make_ref<SingleDerivedPath>(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "^", xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "!", xpSettings); +} + +static DerivedPath parseWith( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) +{ + size_t n = s.rfind(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); + : (DerivedPath) DerivedPath::Built::parse(store, + make_ref<SingleDerivedPath>(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +DerivedPath DerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWith(store, s, "^", xpSettings); +} + +DerivedPath DerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWith(store, s, "!", xpSettings); +} + +DerivedPath DerivedPath::fromSingle(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) -> DerivedPath { + return o; + }, + [&](const SingleDerivedPath::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + }, req.raw()); +} + +const StorePath & SingleDerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +const StorePath & DerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +template<typename DP> +static inline const StorePath & getBaseStorePath_(const DP & derivedPath) +{ + return std::visit(overloaded { + [&](const typename DP::Built & bfd) -> auto & { + return bfd.drvPath->getBaseStorePath(); + }, + [&](const typename DP::Opaque & bo) -> auto & { + return bo.path; + }, + }, derivedPath.raw()); } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +const StorePath & SingleDerivedPath::getBaseStorePath() const { - return parseWith(store, s, "^"); + return getBaseStorePath_(*this); } -DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +const StorePath & DerivedPath::getBaseStorePath() const { - return parseWith(store, s, "!"); + return getBaseStorePath_(*this); } } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 7a4261ce0..ec30dac61 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -24,15 +24,135 @@ class Store; struct DerivedPathOpaque { StorePath path; - nlohmann::json toJSON(ref<Store> store) const; std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; +struct SingleDerivedPath; + +/** + * A single derived path that is built from a derivation + * + * Built derived paths are pair of a derivation and an output name. They are + * evaluated by building the derivation, and then taking the resulting output + * path of the given output name. + */ +struct SingleDerivedPathBuilt { + ref<SingleDerivedPath> drvPath; + std::string output; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPathBuilt parse( + const Store & store, ref<SingleDerivedPath> drvPath, + std::string_view outputs, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; + + DECLARE_CMP(SingleDerivedPathBuilt); +}; + +using _SingleDerivedPathRaw = std::variant< + DerivedPathOpaque, + SingleDerivedPathBuilt +>; + /** - * A derived path that is built from a derivation + * A "derived path" is a very simple sort of expression (not a Nix + * language expression! But an expression in a the general sense) that + * evaluates to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ +struct SingleDerivedPath : _SingleDerivedPathRaw { + using Raw = _SingleDerivedPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleDerivedPathBuilt; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + /** + * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; +}; + +static inline ref<SingleDerivedPath> makeConstantStorePathRef(StorePath drvPath) +{ + return make_ref<SingleDerivedPath>(SingleDerivedPath::Opaque { drvPath }); +} + +/** + * A set of derived paths that are built from a derivation * * Built derived paths are pair of a derivation and some output names. * They are evaluated by building the derivation, and then replacing the @@ -44,10 +164,22 @@ struct DerivedPathOpaque { * output name. */ struct DerivedPathBuilt { - StorePath drvPath; + ref<SingleDerivedPath> drvPath; OutputsSpec outputs; /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** * Uses `^` as the separator */ std::string to_string(const Store & store) const; @@ -57,11 +189,16 @@ struct DerivedPathBuilt { std::string to_string_legacy(const Store & store) const; /** * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); - nlohmann::json toJSON(ref<Store> store) const; + static DerivedPathBuilt parse( + const Store & store, ref<SingleDerivedPath>, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; - GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(DerivedPathBuilt); }; using _DerivedPathRaw = std::variant< @@ -71,13 +208,13 @@ using _DerivedPathRaw = std::variant< /** * A "derived path" is a very simple sort of expression that evaluates - * to (concrete) store path. It is either: + * to one or more (concrete) store paths. It is either: * - * - opaque, in which case it is just a concrete store path with + * - opaque, in which case it is just a single concrete store path with * possibly no known derivation * - * - built, in which case it is a pair of a derivation path and an - * output name. + * - built, in which case it is a pair of a derivation path and some + * output names. */ struct DerivedPath : _DerivedPathRaw { using Raw = _DerivedPathRaw; @@ -91,6 +228,18 @@ struct DerivedPath : _DerivedPathRaw { } /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** * Uses `^` as the separator */ std::string to_string(const Store & store) const; @@ -100,14 +249,43 @@ struct DerivedPath : _DerivedPathRaw { std::string to_string_legacy(const Store & store) const; /** * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parse(const Store & store, std::string_view); + static DerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parseLegacy(const Store & store, std::string_view); + static DerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convert a `SingleDerivedPath` to a `DerivedPath`. + */ + static DerivedPath fromSingle(const SingleDerivedPath &); + + nlohmann::json toJSON(Store & store) const; }; typedef std::vector<DerivedPath> DerivedPaths; +/** + * Used by various parser functions to require experimental features as + * needed. + * + * Somewhat unfortunate this cannot just be an implementation detail for + * this module. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 1752738f2..885b3604d 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -11,8 +11,10 @@ std::string DownstreamPlaceholder::render() const DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( const StorePath & drvPath, - std::string_view outputName) + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings) { + xpSettings.require(Xp::CaDerivations); auto drvNameWithExtension = drvPath.name(); auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); @@ -36,4 +38,19 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( }; } +DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & b) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return DownstreamPlaceholder::unknownCaOutput(o.path, b.output); + }, + [&](const SingleDerivedPath::Built & b2) { + return DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2), + b.output); + }, + }, b.drvPath->raw()); +} + } diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index f0c0dee77..9372dcd58 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -3,6 +3,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" namespace nix { @@ -52,10 +53,13 @@ public: * * The derivation itself is known (we have a store path for it), but * the output doesn't yet have a known store path. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DownstreamPlaceholder unknownCaOutput( const StorePath & drvPath, - std::string_view outputName); + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Create a placehold for the output of an unknown derivation. @@ -70,6 +74,17 @@ public: const DownstreamPlaceholder & drvPlaceholder, std::string_view outputName, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convenience constructor that handles both cases (unknown + * content-addressed output and unknown derivation), delegating as + * needed to `unknownCaOutput` and `unknownDerivation`. + * + * Recursively builds up a placeholder from a + * `SingleDerivedPath::Built.drvPath` chain. + */ + static DownstreamPlaceholder fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & built); }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fa17d606d..78b05031a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -358,6 +358,9 @@ public: [&](const StorePath & drvPath) { throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, }, sOrDrvPath); } conn->to << ss; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f78bd44ca..17b4ecc73 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1196,6 +1196,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (checkSigs && pathInfoIsUntrusted(info)) throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path)); + /* In case we are not interested in reading the NAR: discard it. */ + bool narRead = false; + Finally cleanup = [&]() { + if (!narRead) { + ParseSink sink; + parseDump(sink, source); + } + }; + addTempRoot(info.path); if (repair || !isValidPath(info.path)) { @@ -1220,6 +1229,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, TeeSource wrapperSource { source, hashSink }; + narRead = true; restorePath(realPath, wrapperSource); auto hashResult = hashSink.finish(); @@ -1499,17 +1509,33 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StringSet store; - for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); + StorePathSet validPaths; + + { + StorePathSet storePathsInStoreDir; + /* Why aren't we using `queryAllValidPaths`? Because that would + tell us about all the paths than the database knows about. Here we + want to know about all the store paths in the store directory, + regardless of what the database thinks. + + We will end up cross-referencing these two sources of truth (the + database and the filesystem) in the loop below, in order to catch + invalid states. + */ + for (auto & i : readDirectory(realStoreDir)) { + try { + storePathsInStoreDir.insert({i.name}); + } catch (BadStorePath &) { } + } - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); - StorePathSet validPaths; - StorePathSet done; + StorePathSet done; - for (auto & i : queryAllValidPaths()) - verifyPath(i, store, done, validPaths, repair, errors); + for (auto & i : queryAllValidPaths()) + verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); + } /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1595,21 +1621,21 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const StorePath & path, const StringSet & store, +void LocalStore::verifyPath(const StorePath & path, const StorePathSet & storePathsInStoreDir, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); if (!done.insert(path).second) return; - if (!store.count(std::string(path.to_string()))) { + if (!storePathsInStoreDir.count(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(i, store, done, validPaths, repair, errors); + verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c9b570eaa..e97195f5b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -314,7 +314,7 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const StorePath & path, const StringSet & store, + void verifyPath(const StorePath & path, const StorePathSet & store, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); std::shared_ptr<const ValidPathInfo> queryPathInfoInternal(State & state, const StorePath & path); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 14160dc8b..185d61c15 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -132,7 +132,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); + pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second })); }; auto checkOutput = [&]( @@ -176,10 +176,18 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - if (!isValidPath(bfd.drvPath)) { + auto drvPathP = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath); + if (!drvPathP) { + // TODO make work in this case. + warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(*this)); + return; + } + auto & drvPath = drvPathP->path; + + if (!isValidPath(drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(bfd.drvPath); + state->unknown.insert(drvPath); return; } @@ -187,7 +195,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; @@ -197,15 +205,45 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref<Derivation>(derivationFromPath(bfd.drvPath)); - ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); + auto drv = make_ref<Derivation>(derivationFromPath(drvPath)); + ParsedDerivation parsedDrv(StorePath(drvPath), *drv); + + if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + experimentalFeatureSettings.require(Xp::CaDerivations); + + // If there are unknown output paths, attempt to find if the + // paths are known to substituters through a realisation. + auto outputHashes = staticOutputHashes(*this, *drv); + knownOutputPaths = true; + + for (auto [outputName, hash] : outputHashes) { + if (!bfd.outputs.contains(outputName)) + continue; + + bool found = false; + for (auto &sub : getDefaultSubstituters()) { + auto realisation = sub->queryRealisation({hash, outputName}); + if (!realisation) + continue; + found = true; + if (!isValidPath(realisation->outPath)) + invalid.insert(realisation->outPath); + break; + } + if (!found) { + // Some paths did not have a realisation, this must be built. + knownOutputPaths = false; + break; + } + } + } if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState)); } else - mustBuildDrv(bfd.drvPath, *drv); + mustBuildDrv(drvPath, *drv); }, [&](const DerivedPath::Opaque & bo) { @@ -310,7 +348,9 @@ std::map<DrvOutput, StorePath> drvOutputReferences( OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { - auto outputsOpt_ = store.queryPartialDerivationOutputMap(bfd.drvPath, evalStore_); + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + + auto outputsOpt_ = store.queryPartialDerivationOutputMap(drvPath, evalStore_); auto outputsOpt = std::visit(overloaded { [&](const OutputsSpec::All &) { @@ -325,7 +365,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, if (!pOutputPathOpt) throw Error( "the derivation '%s' doesn't have an output named '%s'", - store.printStorePath(bfd.drvPath), output); + bfd.drvPath->to_string(store), output); outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt)); } return outputsOpt; @@ -335,11 +375,64 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { if (!outputPathOpt) - throw MissingRealisation(store.printStorePath(bfd.drvPath), outputName); + throw MissingRealisation(bfd.drvPath->to_string(store), outputName); auto & outputPath = *outputPathOpt; outputs.insert_or_assign(outputName, outputPath); } return outputs; } + +StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_); + if (outputPaths.count(bfd.output) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store.printStorePath(drvPath), bfd.output); + auto & optPath = outputPaths.at(bfd.output); + if (!optPath) + throw Error("'%s' does not yet map to a known concrete store path", + bfd.to_string(store)); + return *optPath; + }, + }, req.raw()); +} + + +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) +{ + auto drvPath = resolveDerivedPath(store, *bfd.drvPath); + auto outputMap = store.queryDerivationOutputMap(drvPath); + auto outputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast<StringSet>(names); + }, + }, bfd.outputs.raw()); + for (auto iter = outputMap.begin(); iter != outputMap.end();) { + auto & outputName = iter->first; + if (bfd.outputs.contains(outputName)) { + outputsLeft.erase(outputName); + ++iter; + } else { + iter = outputMap.erase(iter); + } + } + if (!outputsLeft.empty()) + throw Error("derivation '%s' does not have an outputs %s", + store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(std::get<OutputsSpec::Names>(bfd.outputs)))); + return outputMap; +} + } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 869b490ad..81f360f3a 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -16,10 +16,16 @@ std::string StorePathWithOutputs::to_string(const Store & store) const DerivedPath StorePathWithOutputs::toDerivedPath() const { if (!outputs.empty()) { - return DerivedPath::Built { path, OutputsSpec::Names { outputs } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::Names { outputs }, + }; } else if (path.isDerivation()) { assert(outputs.empty()); - return DerivedPath::Built { path, OutputsSpec::All { } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::All { }, + }; } else { return DerivedPath::Opaque { path }; } @@ -34,29 +40,36 @@ std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> } -std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) +StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) -> std::variant<StorePathWithOutputs, StorePath> { + [&](const DerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> { - return StorePathWithOutputs { - .path = bfd.drvPath, - // Use legacy encoding of wildcard as empty set - .outputs = std::visit(overloaded { - [&](const OutputsSpec::All &) -> StringSet { - return {}; - }, - [&](const OutputsSpec::Names & outputs) { - return static_cast<StringSet>(outputs); - }, - }, bfd.outputs.raw()), - }; + [&](const DerivedPath::Built & bfd) -> StorePathWithOutputs::ParseResult { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { + return StorePathWithOutputs { + .path = bo.path, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast<StringSet>(outputs); + }, + }, bfd.outputs.raw()), + }; + }, + [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult { + return std::monostate {}; + }, + }, bfd.drvPath->raw()); }, }, p.raw()); } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index d75850868..57e03252d 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -23,7 +23,9 @@ struct StorePathWithOutputs DerivedPath toDerivedPath() const; - static std::variant<StorePathWithOutputs, StorePath> tryFromDerivedPath(const DerivedPath &); + typedef std::variant<StorePathWithOutputs, StorePath, std::monostate> ParseResult; + + static StorePathWithOutputs::ParseResult tryFromDerivedPath(const DerivedPath &); }; std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 21258daec..58f72beb9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -656,6 +656,9 @@ static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & con GET_PROTOCOL_MAJOR(conn.daemonVersion), GET_PROTOCOL_MINOR(conn.daemonVersion)); }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'"); + }, }, sOrDrvPath); } conn.to << ss; @@ -670,9 +673,16 @@ void RemoteStore::copyDrvsFromEvalStore( /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ RealisedPath::Set drvPaths2; - for (auto & i : paths) - if (auto p = std::get_if<DerivedPath::Built>(&i)) - drvPaths2.insert(p->drvPath); + for (const auto & i : paths) { + std::visit(overloaded { + [&](const DerivedPath::Opaque & bp) { + // Do nothing, path is hopefully there already + }, + [&](const DerivedPath::Built & bp) { + drvPaths2.insert(bp.drvPath->getBaseStorePath()); + }, + }, i.raw()); + } copyClosure(*evalStore, *this, drvPaths2); } } @@ -742,7 +752,8 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults( }; OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); + auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); + auto drv = evalStore->readDerivation(drvPath); const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { @@ -750,7 +761,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults( if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", - printStorePath(bfd.drvPath), output); + printStorePath(drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4ea16a1c0..28689e100 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1496,6 +1496,7 @@ ref<Store> openStore(const std::string & uri_, if (implem.uriSchemes.count(parsedUri.scheme)) { auto store = implem.create(parsedUri.scheme, baseURI, params); if (store) { + experimentalFeatureSettings.require(store->experimentalFeature()); store->init(); store->warnUnknownSettings(); return ref<Store>(store); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3758c730f..da1a3eefb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -109,13 +109,28 @@ struct StoreConfig : public Config virtual ~StoreConfig() { } + /** + * The name of this type of store. + */ virtual const std::string name() = 0; + /** + * Documentation for this type of store. + */ virtual std::string doc() { return ""; } + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ + virtual std::optional<ExperimentalFeature> experimentalFeature() const + { + return std::nullopt; + } + const PathSetting storeDir_{this, settings.nixStore, "store", R"( @@ -930,6 +945,7 @@ void removeTempRoots(); * Resolve the derived path completely, failing if any derivation output * is unknown. */ +StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr); OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index 160443ec1..d6549f66f 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -17,14 +17,34 @@ Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary() }); } +Gen<SingleDerivedPath::Built> Arbitrary<SingleDerivedPath::Built>::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()), + .output = (*gen::arbitrary<StorePathName>()).name, + }); +} + Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary() { return gen::just(DerivedPath::Built { - .drvPath = *gen::arbitrary<StorePath>(), + .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()), .outputs = *gen::arbitrary<OutputsSpec>(), }); } +Gen<SingleDerivedPath> Arbitrary<SingleDerivedPath>::arbitrary() +{ + switch (*gen::inRange<uint8_t>(0, std::variant_size_v<SingleDerivedPath::Raw>)) { + case 0: + return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Opaque>()); + case 1: + return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Built>()); + default: + assert(false); + } +} + Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary() { switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) { @@ -45,12 +65,69 @@ class DerivedPathTest : public LibStoreTest { }; -// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is -// no a real fixture. -// -// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args -TEST_F(DerivedPathTest, force_init) -{ +/** + * Round trip (string <-> data structure) test for + * `DerivedPath::Opaque`. + */ +TEST_F(DerivedPathTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = DerivedPath::parse(*store, opaque); + auto * p = std::get_if<DerivedPath::Opaque>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store->parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(*store), opaque); +} + +/** + * Round trip (string <-> data structure) test for a simpler + * `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_opaque) { + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^bar,foo"; + auto elem = DerivedPath::parse(*store, built); + auto * p = std::get_if<DerivedPath::Built>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "foo", "bar" })); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"; + auto elem = DerivedPath::parse(*store, built, mockXpSettings); + auto * p = std::get_if<DerivedPath::Built>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "bar", "baz" })); + auto * drvPath = std::get_if<SingleDerivedPath::Built>(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "foo"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive derived path. + */ +TEST_F(DerivedPathTest, built_built_xp) { + ASSERT_THROW( + DerivedPath::parse(*store, "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"), + MissingExperimentalFeature); } RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 506f3ccb1..98d61f228 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -12,8 +12,18 @@ namespace rc { using namespace nix; template<> -struct Arbitrary<DerivedPath::Opaque> { - static Gen<DerivedPath::Opaque> arbitrary(); +struct Arbitrary<SingleDerivedPath::Opaque> { + static Gen<SingleDerivedPath::Opaque> arbitrary(); +}; + +template<> +struct Arbitrary<SingleDerivedPath::Built> { + static Gen<SingleDerivedPath::Built> arbitrary(); +}; + +template<> +struct Arbitrary<SingleDerivedPath> { + static Gen<SingleDerivedPath> arbitrary(); }; template<> diff --git a/src/libstore/tests/downstream-placeholder.cc b/src/libstore/tests/downstream-placeholder.cc index ec3e1000f..fd29530ac 100644 --- a/src/libstore/tests/downstream-placeholder.cc +++ b/src/libstore/tests/downstream-placeholder.cc @@ -5,17 +5,24 @@ namespace nix { TEST(DownstreamPlaceholder, unknownCaOutput) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + ASSERT_EQ( DownstreamPlaceholder::unknownCaOutput( StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, - "out").render(), + "out", + mockXpSettings).render(), "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz"); } TEST(DownstreamPlaceholder, unknownDerivation) { /** - * We set these in tests rather than the regular globals so we don't have - * to worry about race conditions if the tests run concurrently. + * Same reason as above */ ExperimentalFeatureSettings mockXpSettings; mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); @@ -24,7 +31,8 @@ TEST(DownstreamPlaceholder, unknownDerivation) { DownstreamPlaceholder::unknownDerivation( DownstreamPlaceholder::unknownCaOutput( StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" }, - "out"), + "out", + mockXpSettings), "out", mockXpSettings).render(), "/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w"); diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 9f661c5c3..7982fdc5e 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -2,6 +2,22 @@ ///@file /** + * Declare comparison methods without defining them. + */ +#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \ + bool operator COMPARATOR(const MY_TYPE & other) const; +#define DECLARE_EQUAL(my_type) \ + DECLARE_ONE_CMP(==, my_type) +#define DECLARE_LEQ(my_type) \ + DECLARE_ONE_CMP(<, my_type) +#define DECLARE_NEQ(my_type) \ + DECLARE_ONE_CMP(!=, my_type) +#define DECLARE_CMP(my_type) \ + DECLARE_EQUAL(my_type) \ + DECLARE_LEQ(my_type) \ + DECLARE_NEQ(my_type) + +/** * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. * diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 7c4112d32..782331283 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array<ExperimentalFeatureDetails, 15> xpFeatureDetails = {{ +constexpr std::array<ExperimentalFeatureDetails, 14> xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -183,15 +183,6 @@ constexpr std::array<ExperimentalFeatureDetails, 15> xpFeatureDetails = {{ )", }, { - .tag = Xp::DiscardReferences, - .name = "discard-references", - .description = R"( - Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations - that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for - runtime dependencies. - )", - }, - { .tag = Xp::DaemonTrustOverride, .name = "daemon-trust-override", .description = R"( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index faf2e9398..add592ae6 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -27,7 +27,6 @@ enum struct ExperimentalFeature ReplFlake, AutoAllocateUids, Cgroups, - DiscardReferences, DaemonTrustOverride, DynamicDerivations, ParseTomlTimestamps, diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index d7220e71d..61cef743d 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -1,4 +1,5 @@ #include "json-utils.hh" +#include "error.hh" namespace nix { @@ -16,4 +17,27 @@ nlohmann::json * get(nlohmann::json & map, const std::string & key) return &*i; } +const nlohmann::json & valueAt( + const nlohmann::json & map, + const std::string & key) +{ + if (!map.contains(key)) + throw Error("Expected JSON object to contain key '%s' but it doesn't", key); + + return map[key]; +} + +const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType + ) +{ + if (value.type() != expectedType) + throw Error( + "Expected JSON value to be of type '%s' but it is of type '%s'", + nlohmann::json(expectedType).type_name(), + value.type_name()); + + return value; +} } diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 5e63c1af4..77c63595c 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -11,6 +11,28 @@ const nlohmann::json * get(const nlohmann::json & map, const std::string & key); nlohmann::json * get(nlohmann::json & map, const std::string & key); /** + * Get the value of a json object at a key safely, failing + * with a Nix Error if the key does not exist. + * + * Use instead of nlohmann::json::at() to avoid ugly exceptions. + * + * _Does not check whether `map` is an object_, use `ensureType` for that. + */ +const nlohmann::json & valueAt( + const nlohmann::json & map, + const std::string & key); + +/** + * Ensure the type of a json object is what you expect, failing + * with a Nix Error if it isn't. + * + * Use before type conversions and element access to avoid ugly exceptions. + */ +const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType); + +/** * For `adl_serializer<std::optional<T>>` below, we need to track what * types are not already using `null`. Only for them can we use `null` * to represent `std::nullopt`. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6510df8f0..66f319c3e 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -393,7 +393,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { - .drvPath = bashDrv, + .drvPath = makeConstantStorePathRef(bashDrv), .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); @@ -417,7 +417,7 @@ static void main_nix_build(int argc, char * * argv) })) { pathsToBuild.push_back(DerivedPath::Built { - .drvPath = inputDrv, + .drvPath = makeConstantStorePathRef(inputDrv), .outputs = OutputsSpec::Names { inputOutputs }, }); pathsToCopy.insert(inputDrv); @@ -590,7 +590,10 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); + pathsToBuild.push_back(DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::Names{outputName}, + }); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 3dc3db0fc..b112e8cb3 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -482,7 +482,7 @@ static void printMissing(EvalState & state, DrvInfos & elems) for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) targets.push_back(DerivedPath::Built{ - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }); else @@ -760,7 +760,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) std::vector<DerivedPath> paths { drvPath ? (DerivedPath) (DerivedPath::Built { - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }) : (DerivedPath) (DerivedPath::Opaque { diff --git a/src/nix/app.cc b/src/nix/app.cc index e678b54f0..16a921194 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -22,11 +22,13 @@ StringPairs resolveRewrites( StringPairs res; for (auto & dep : dependencies) if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path)) - for (auto & [ outputName, outputPath ] : drvDep->outputs) - res.emplace( - DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), - store.printStorePath(outputPath) - ); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + for (auto & [ outputName, outputPath ] : drvDep->outputs) + res.emplace( + DownstreamPlaceholder::unknownCaOutput( + drvDep->drvPath->outPath(), outputName).render(), + store.printStorePath(outputPath) + ); return res; } @@ -64,7 +66,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { /* We want all outputs of the drv */ return DerivedPath::Built { - .drvPath = d.drvPath, + .drvPath = makeConstantStorePathRef(d.drvPath), .outputs = OutputsSpec::All {}, }; }, @@ -105,7 +107,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { .context = { DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::Names { outputName }, } }, .program = program, diff --git a/src/nix/build.cc b/src/nix/build.cc index ad1842a4e..479100186 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -9,18 +9,18 @@ using namespace nix; -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store) +static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store) { auto res = nlohmann::json::array(); for (auto & t : paths) { - std::visit([&res, store](const auto & t) { + std::visit([&](const auto & t) { res.push_back(t.toJSON(store)); }, t.raw()); } return res; } -nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult> & buildables, ref<Store> store) +static nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult> & buildables, const Store & store) { auto res = nlohmann::json::array(); for (auto & b : buildables) { @@ -125,7 +125,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile printMissing(store, pathsToBuild, lvlError); if (json) - logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + logger->cout("%s", derivedPathsToJSON(pathsToBuild, *store).dump()); return; } @@ -136,7 +136,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile installables, repair ? bmRepair : buildMode); - if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, *store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index bcc00d490..5a80f0308 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -109,7 +109,7 @@ struct CmdBundle : InstallableValueCommand store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 195eeaa21..c033804e4 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -235,7 +235,7 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore /* Build the derivation. */ store->buildPaths( { DerivedPath::Built { - .drvPath = shellDrvPath, + .drvPath = makeConstantStorePathRef(shellDrvPath), .outputs = OutputsSpec::All { }, }}, bmNormal, evalStore); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3ce1de44a..83b74c8ca 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -544,9 +544,9 @@ struct CmdFlakeCheck : FlakeCommand *attr2.value, attr2.pos); if (drvPath && attr_name == settings.thisSystem.get()) { drvPaths.push_back(DerivedPath::Built { - .drvPath = *drvPath, - .outputs = OutputsSpec::All { }, - }); + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All { }, + }); } } } diff --git a/src/nix/log.cc b/src/nix/log.cc index aaf829764..9a9bd30f9 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -33,6 +33,17 @@ struct CmdLog : InstallableCommand auto b = installable->toDerivedPath(); + // For compat with CLI today, TODO revisit + auto oneUp = std::visit(overloaded { + [&](const DerivedPath::Opaque & bo) { + return make_ref<SingleDerivedPath>(bo); + }, + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + }, b.path.raw()); + auto path = resolveDerivedPath(*store, *oneUp); + RunPager pager; for (auto & sub : subs) { auto * logSubP = dynamic_cast<LogStore *>(&*sub); @@ -42,14 +53,7 @@ struct CmdLog : InstallableCommand } auto & logSub = *logSubP; - auto log = std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) { - return logSub.getBuildLog(bo.path); - }, - [&](const DerivedPath::Built & bfd) { - return logSub.getBuildLog(bfd.drvPath); - }, - }, b.path.raw()); + auto log = logSub.getBuildLog(path); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); diff --git a/src/nix/main.cc b/src/nix/main.cc index df66beb8c..c5a9c8b33 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -180,8 +180,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs for (auto & implem : *Implementations::registered) { auto storeConfig = implem.getConfig(); auto storeName = storeConfig->name(); - stores[storeName]["doc"] = storeConfig->doc(); - stores[storeName]["settings"] = storeConfig->toJSON(); + auto & j = stores[storeName]; + j["doc"] = storeConfig->doc(); + j["settings"] = storeConfig->toJSON(); + j["experimentalFeature"] = storeConfig->experimentalFeature(); } res["stores"] = std::move(stores); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index a3a9dc698..592de773c 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -239,7 +239,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (pos != std::string::npos) { size_t margin = 32; auto pos2 = pos >= margin ? pos - margin : 0; - hits[hash].emplace_back(fmt("%s: …%s…\n", + hits[hash].emplace_back(fmt("%s: …%s…", p2, hilite(filterPrintable( std::string(contents, pos2, pos - pos2 + hash.size() + margin)), @@ -255,7 +255,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & hash : hashes) { auto pos = target.find(hash); if (pos != std::string::npos) - hits[hash].emplace_back(fmt("%s -> %s\n", p2, + hits[hash].emplace_back(fmt("%s -> %s", p2, hilite(target, pos, StorePath::HashLen, getColour(hash)))); } } @@ -272,9 +272,9 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & hit : hits[hash]) { bool first = hit == *hits[hash].begin(); - std::cout << tailPad - << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)) - << hit; + logger->cout("%s%s%s", tailPad, + (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)), + hit); if (!all) break; } |