diff options
| author | John Ericson <John.Ericson@Obsidian.Systems> | 2023-01-15 17:39:04 -0500 |
|---|---|---|
| committer | John Ericson <John.Ericson@Obsidian.Systems> | 2023-08-10 00:08:32 -0400 |
| commit | 60b7121d2c6d4322b7c2e8e7acfec7b701b2d3a1 (patch) | |
| tree | c07508902903edf2d1a11b135ddd2bb512819ea6 /src | |
| parent | d00fe5f22559efc6f8b4b92eab537b08c0e43dee (diff) | |
Make the Derived Path family of types inductive for dynamic derivations
We want to be able to write down `foo.drv^bar.drv^baz`:
`foo.drv^bar.drv` is the dynamic derivation (since it is itself a
derivation output, `bar.drv` from `foo.drv`).
To that end, we create `Single{Derivation,BuiltPath}` types, that are
very similar except instead of having multiple outputs (in a set or
map), they have a single one. This is for everything to the left of the
rightmost `^`.
`NixStringContextElem` has an analogous change, and now can reuse
`SingleDerivedPath` at the top level. In fact, if we ever get rid of
`DrvDeep`, `NixStringContextElem` could be replaced with
`SingleDerivedPath` entirely!
Important note: some JSON formats have changed.
We already can *produce* dynamic derivations, but we can't refer to them
directly. Today, we can merely express building or example at the top
imperatively over time by building `foo.drv^bar.drv`, and then with a
second nix invocation doing `<result-from-first>^baz`, but this is not
declarative. The ethos of Nix of being able to write down the full plan
everything you want to do, and then execute than plan with a single
command, and for that we need the new inductive form of these types.
Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>
Diffstat (limited to 'src')
43 files changed, 1096 insertions, 252 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 2fb17d06f..c1f03a8ef 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -322,7 +322,12 @@ connected: throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); - auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } }); + auto res = sshStore->buildPathsWithResults({ + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All {}, + } + }); // One path to build should produce exactly one build result assert(res.size() == 1); optResult = std::move(res[0]); diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index db9c440e3..c6cc7fa9c 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -8,13 +8,39 @@ namespace nix { -nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = store->printStorePath(path); +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ } - return res; +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::pair<std::string, StorePath> +CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE std::map<std::string, StorePath> +CMP(SingleBuiltPath, BuiltPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +StorePath SingleBuiltPath::outPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) { return p.path; }, + [](const SingleBuiltPath::Built & b) { return b.output.second; }, + }, raw() + ); } StorePathSet BuiltPath::outPaths() const @@ -32,6 +58,40 @@ StorePathSet BuiltPath::outPaths() const ); } +nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + for (const auto & [outputName, outputPath] : outputs) { + res["outputs"][outputName] = store.printStorePath(outputPath); + } + return res; +} + +nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + auto & [outputName, outputPath] = output; + res["output"] = outputName; + res["outputPath"] = store.printStorePath(outputPath); + return res; +} + +nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json BuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const { RealisedPath::Set res; @@ -40,7 +100,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, [&](const BuiltPath::Built & p) { auto drvHashes = - staticOutputHashes(store, store.readDerivation(p.drvPath)); + staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); for (auto& [outputName, outputPath] : p.outputs) { if (experimentalFeatureSettings.isEnabled( Xp::CaDerivations)) { @@ -48,7 +108,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const if (!drvOutput) throw Error( "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath), outputName); + store.printStorePath(p.drvPath->outPath()), outputName); auto thisRealisation = store.queryRealisation( DrvOutput{*drvOutput, outputName}); assert(thisRealisation); // We’ve built it, so we must diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 744e8090b..747bcc440 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -3,19 +3,60 @@ namespace nix { +struct SingleBuiltPath; + +struct SingleBuiltPathBuilt { + ref<SingleBuiltPath> drvPath; + std::pair<std::string, StorePath> output; + + std::string to_string(const Store & store) const; + static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; + + DECLARE_CMP(SingleBuiltPathBuilt); +}; + +using _SingleBuiltPathRaw = std::variant< + DerivedPathOpaque, + SingleBuiltPathBuilt +>; + +struct SingleBuiltPath : _SingleBuiltPathRaw { + using Raw = _SingleBuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleBuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } + + StorePath outPath() const; + + static SingleBuiltPath parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; +}; + +static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath) +{ + return make_ref<SingleBuiltPath>(SingleBuiltPath::Opaque { drvPath }); +} + /** * A built derived path with hints in the form of optional concrete output paths. * * See 'BuiltPath' for more an explanation. */ struct BuiltPathBuilt { - StorePath drvPath; + ref<SingleBuiltPath> drvPath; std::map<std::string, StorePath> outputs; - nlohmann::json toJSON(ref<Store> store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view); + std::string to_string(const Store & store) const; + static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; - GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -41,6 +82,7 @@ struct BuiltPath : _BuiltPathRaw { StorePathSet outPaths() const; RealisedPath::Set toRealisedPaths(Store & store) const; + nlohmann::json toJSON(const Store & store) const; }; typedef std::vector<BuiltPath> BuiltPaths; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index b35ca2910..2f89eee02 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -92,7 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() for (auto & [drvPath, outputs] : byDrvPath) res.push_back({ .path = DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = outputs, }, .info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value { diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 6ecf54b7c..b45641e8a 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -18,14 +18,7 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() std::optional<StorePath> InstallableDerivedPath::getStorePath() { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; - }, - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - }, derivedPath.raw()); + return derivedPath.getBaseStorePath(); } InstallableDerivedPath InstallableDerivedPath::parse( @@ -42,7 +35,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( // Remove this prior to stabilizing the new CLI. if (storePath.isDerivation()) { auto oldDerivedPath = DerivedPath::Built { - .drvPath = storePath, + .drvPath = makeConstantStorePathRef(storePath), .outputs = OutputsSpec::All { }, }; warn( @@ -55,8 +48,10 @@ InstallableDerivedPath InstallableDerivedPath::parse( }, // If the user did use ^, we just do exactly what is written. [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + auto drv = make_ref<SingleDerivedPath>(SingleDerivedPath::parse(*store, prefix)); + drvRequireExperiment(*drv); return DerivedPath::Built { - .drvPath = store->parseStorePath(prefix), + .drvPath = std::move(drv), .outputs = outputSpec, }; }, diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 4da9b131b..1b169c3bd 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -118,7 +118,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() return {{ .path = DerivedPath::Built { - .drvPath = std::move(drvPath), + .drvPath = makeConstantStorePathRef(std::move(drvPath)), .outputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { std::set<std::string> outputsToInstall; diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 1eff293cc..08ad35105 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -55,7 +55,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths else if (v.type() == nString) { return {{ - .path = state->coerceToDerivedPath(pos, v, errorCtx), + .path = DerivedPath::fromSingle( + state->coerceToSingleDerivedPath(pos, v, errorCtx)), .info = make_ref<ExtraPathInfo>(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 9d593a01f..4c7d134ec 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -515,6 +515,30 @@ ref<Installable> SourceExprCommand::parseInstallable( return installables.front(); } +static SingleBuiltPath getBuiltPath(ref<Store> evalStore, ref<Store> store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { + return SingleBuiltPath::Opaque { bo.path }; + }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. + SingleDerivedPath::Built truncatedBfd { + .drvPath = makeConstantStorePathRef(drvPath.outPath()), + .output = bfd.output, + }; + auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); + return SingleBuiltPath::Built { + .drvPath = make_ref<SingleBuiltPath>(std::move(drvPath)), + .output = { bfd.output, outputPath }, + }; + }, + }, + b.raw()); +} + std::vector<BuiltPathWithResult> Installable::build( ref<Store> evalStore, ref<Store> store, @@ -568,7 +592,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build [&](const DerivedPath::Built & bfd) { auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { @@ -597,7 +624,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build for (auto & [outputName, realisation] : buildResult.builtOutputs) outputs.emplace(outputName, realisation.outPath); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info, .result = buildResult}}); }, @@ -691,7 +721,7 @@ StorePathSet Installable::toDerivations( : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { - drvPaths.insert(bfd.drvPath); + drvPaths.insert(resolveDerivedPath(*store, *bfd.drvPath)); }, }, b.path.raw()); diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index d15162e76..ea7d0e2ec 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -648,7 +648,7 @@ bool NixRepl::processLine(std::string line) if (command == ":b" || command == ":bl") { state->store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 9e734e654..6a60ba87b 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -599,7 +599,7 @@ string_t AttrCursor::getStringWithContext() return d.drvPath; }, [&](const NixStringContextElem::Built & b) -> const StorePath & { - return b.drvPath; + return b.drvPath->getBaseStorePath(); }, [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a8e6baea6..3e521b732 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1042,7 +1042,7 @@ void EvalState::mkOutputString( : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(), NixStringContext { NixStringContextElem::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .output = outputName, } }); @@ -2299,7 +2299,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon } -std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) +std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) { NixStringContext context; auto s = forceString(v, context, pos, errorCtx); @@ -2310,21 +2310,16 @@ std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked s, csize) .withTrace(pos, errorCtx).debugThrow<EvalError>(); auto derivedPath = std::visit(overloaded { - [&](NixStringContextElem::Opaque && o) -> DerivedPath { - return DerivedPath::Opaque { - .path = std::move(o.path), - }; + [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { + return std::move(o); }, - [&](NixStringContextElem::DrvDeep &&) -> DerivedPath { + [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath { error( "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", s).withTrace(pos, errorCtx).debugThrow<EvalError>(); }, - [&](NixStringContextElem::Built && b) -> DerivedPath { - return DerivedPath::Built { - .drvPath = std::move(b.drvPath), - .outputs = OutputsSpec::Names { std::move(b.output) }, - }; + [&](NixStringContextElem::Built && b) -> SingleDerivedPath { + return std::move(b); }, }, ((NixStringContextElem &&) *context.begin()).raw()); return { @@ -2334,12 +2329,12 @@ std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked } -DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) +SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) { - auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx); + auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx); auto s = s_; std::visit(overloaded { - [&](const DerivedPath::Opaque & o) { + [&](const SingleDerivedPath::Opaque & o) { auto sExpected = store->printStorePath(o.path); if (s != sExpected) error( @@ -2347,25 +2342,27 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str s, sExpected) .withTrace(pos, errorCtx).debugThrow<EvalError>(); }, - [&](const DerivedPath::Built & b) { - // TODO need derived path with single output to make this - // total. Will add as part of RFC 92 work and then this is - // cleaned up. - auto output = *std::get<OutputsSpec::Names>(b.outputs).begin(); - - auto drv = store->readDerivation(b.drvPath); - auto i = drv.outputs.find(output); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output); - auto optOutputPath = i->second.path(*store, drv.name, output); - // This is testing for the case of CA derivations - auto sExpected = optOutputPath - ? store->printStorePath(*optOutputPath) - : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render(); + [&](const SingleDerivedPath::Built & b) { + auto sExpected = std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + auto drv = store->readDerivation(o.path); + auto i = drv.outputs.find(b.output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); + auto optOutputPath = i->second.path(*store, drv.name, b.output); + // This is testing for the case of CA derivations + return optOutputPath + ? store->printStorePath(*optOutputPath) + : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); + }, + [&](const SingleDerivedPath::Built & o) { + return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); + }, + }, b.drvPath->raw()); if (s != sExpected) error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", - s, output, store->printStorePath(b.drvPath), sExpected) + s, b.output, b.drvPath->to_string(*store), sExpected) .withTrace(pos, errorCtx).debugThrow<EvalError>(); } }, derivedPath.raw()); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 29d0f05a1..0268a2a12 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -22,7 +22,7 @@ namespace nix { class Store; class EvalState; class StorePath; -struct DerivedPath; +struct SingleDerivedPath; enum RepairFlag : bool; @@ -532,12 +532,12 @@ public: StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** - * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only. + * Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only. */ - std::pair<DerivedPath, std::string_view> coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); + std::pair<SingleDerivedPath, std::string_view> coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); /** - * Coerce to `DerivedPath`. + * Coerce to `SingleDerivedPath`. * * Must be a string which is either a literal store path or a * "placeholder (see `DownstreamPlaceholder`). @@ -551,7 +551,7 @@ public: * source of truth, and ultimately tells us what we want, and then * we ensure the string corresponds to it. */ - DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); + SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); public: diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 430607214..54943b481 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -56,7 +56,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) .drvPath = b.drvPath, .outputs = OutputsSpec::Names { b.output }, }); - ensureValid(b.drvPath); + ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { auto ctxS = store->printStorePath(o.path); @@ -77,7 +77,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) if (!evalSettings.enableImportFromDerivation) debugThrowLastTrace(Error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - store->printStorePath(drvs.begin()->drvPath))); + drvs.begin()->to_string(*store))); /* Build/substitute the context. */ std::vector<DerivedPath> buildReqs; @@ -95,7 +95,11 @@ StringMap EvalState::realiseContext(const NixStringContext & context) /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { res.insert_or_assign( - DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = drv.drvPath, + .output = outputName, + }).render(), store->printStorePath(outputPath) ); } @@ -1251,7 +1255,10 @@ drvName, Bindings * attrs, Value & v) } }, [&](const NixStringContextElem::Built & b) { - drv.inputDrvs[b.drvPath].insert(b.output); + if (auto * p = std::get_if<DerivedPath::Opaque>(&*b.drvPath)) + drv.inputDrvs[p->path].insert(b.output); + else + throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported"); }, [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 8b3468009..bfc731744 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -106,7 +106,10 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, contextInfos[std::move(d.drvPath)].allOutputs = true; }, [&](NixStringContextElem::Built && b) { - contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); + // FIXME should eventually show string context as is, no + // resolving here. + auto drvPath = resolveDerivedPath(*state.store, *b.drvPath); + contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output)); }, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; @@ -222,7 +225,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); context.emplace(NixStringContextElem::Built { - .drvPath = namePath, + .drvPath = makeConstantStorePathRef(namePath), .output = std::string { outputName }, }); } diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index c713fe28a..2a5ca64f6 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -21,12 +21,12 @@ TEST_F(DerivedPathExpressionTest, force_init) RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, prop_opaque_path_round_trip, - (const DerivedPath::Opaque & o)) + (const SingleDerivedPath::Opaque & o)) { auto * v = state.allocValue(); state.mkStorePathString(o.path, *v); - auto d = state.coerceToDerivedPath(noPos, *v, ""); - RC_ASSERT(DerivedPath { o } == d); + auto d = state.coerceToSingleDerivedPath(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { o } == d); } // TODO use DerivedPath::Built for parameter once it supports a single output @@ -46,12 +46,12 @@ RC_GTEST_FIXTURE_PROP( auto * v = state.allocValue(); state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = makeConstantStorePathRef(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } RC_GTEST_FIXTURE_PROP( @@ -61,12 +61,12 @@ RC_GTEST_FIXTURE_PROP( { auto * v = state.allocValue(); state.mkOutputString(*v, drvPath, outputName.name, outPath); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = makeConstantStorePathRef(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } } /* namespace nix */ diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 0d9381577..c56b50b59 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -8,6 +8,8 @@ namespace nix { +// Test a few cases of invalid string context elements. + TEST(NixStringContextElemTest, empty_invalid) { EXPECT_THROW( NixStringContextElem::parse(""), @@ -38,6 +40,10 @@ TEST(NixStringContextElemTest, slash_invalid) { BadStorePath); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::Opaque`. + */ TEST(NixStringContextElemTest, opaque) { std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; auto elem = NixStringContextElem::parse(opaque); @@ -47,6 +53,10 @@ TEST(NixStringContextElemTest, opaque) { ASSERT_EQ(elem.to_string(), opaque); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::DrvDeep`. + */ TEST(NixStringContextElemTest, drvDeep) { std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(drvDeep); @@ -56,28 +66,62 @@ TEST(NixStringContextElemTest, drvDeep) { ASSERT_EQ(elem.to_string(), drvDeep); } -TEST(NixStringContextElemTest, built) { +/** + * Round trip (string <-> data structure) test for a simpler + * `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_opaque) { std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(built); auto * p = std::get_if<NixStringContextElem::Built>(&elem); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(5) }, + })); ASSERT_EQ(elem.to_string(), built); } +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built, mockXpSettings); + auto * p = std::get_if<NixStringContextElem::Built>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + auto * drvPath = std::get_if<SingleDerivedPath::Built>(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "bar"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(9) }, + })); + ASSERT_EQ(elem.to_string(), built); } -namespace rc { -using namespace nix; +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive string context element. + */ +TEST(NixStringContextElemTest, built_built_xp) { + ASSERT_THROW( + NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); +} -Gen<NixStringContextElem::Opaque> Arbitrary<NixStringContextElem::Opaque>::arbitrary() -{ - return gen::just(NixStringContextElem::Opaque { - .path = *gen::arbitrary<StorePath>(), - }); } +namespace rc { +using namespace nix; + Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary() { return gen::just(NixStringContextElem::DrvDeep { @@ -85,14 +129,6 @@ Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arb }); } -Gen<NixStringContextElem::Built> Arbitrary<NixStringContextElem::Built>::arbitrary() -{ - return gen::just(NixStringContextElem::Built { - .drvPath = *gen::arbitrary<StorePath>(), - .output = (*gen::arbitrary<StorePathName>()).name, - }); -} - Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary() { switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) { diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index f76fc76e4..d8116011e 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -4,29 +4,52 @@ namespace nix { -NixStringContextElem NixStringContextElem::parse(std::string_view s0) +NixStringContextElem NixStringContextElem::parse( + std::string_view s0, + const ExperimentalFeatureSettings & xpSettings) { std::string_view s = s0; + std::function<SingleDerivedPath()> parseRest; + parseRest = [&]() -> SingleDerivedPath { + // Case on whether there is a '!' + size_t index = s.find("!"); + if (index == std::string_view::npos) { + return SingleDerivedPath::Opaque { + .path = StorePath { s }, + }; + } else { + std::string output { s.substr(0, index) }; + // Advance string to parse after the '!' + s = s.substr(index + 1); + auto drv = make_ref<SingleDerivedPath>(parseRest()); + drvRequireExperiment(*drv, xpSettings); + return SingleDerivedPath::Built { + .drvPath = std::move(drv), + .output = std::move(output), + }; + } + }; + if (s.size() == 0) { throw BadNixStringContextElem(s0, "String context element should never be an empty string"); } + switch (s.at(0)) { case '!': { - s = s.substr(1); // advance string to parse after first ! - size_t index = s.find("!"); - // This makes index + 1 safe. Index can be the length (one after index - // of last character), so given any valid character index --- a - // successful find --- we can add one. - if (index == std::string_view::npos) { + // Advance string to parse after the '!' + s = s.substr(1); + + // Find *second* '!' + if (s.find("!") == std::string_view::npos) { throw BadNixStringContextElem(s0, "String content element beginning with '!' should have a second '!'"); } - return NixStringContextElem::Built { - .drvPath = StorePath { s.substr(index + 1) }, - .output = std::string(s.substr(0, index)), - }; + + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } case '=': { return NixStringContextElem::DrvDeep { @@ -34,33 +57,51 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0) }; } default: { - return NixStringContextElem::Opaque { - .path = StorePath { s }, - }; + // Ensure no '!' + if (s.find("!") != std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element not beginning with '!' should not have a second '!'"); + } + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } } } -std::string NixStringContextElem::to_string() const { - return std::visit(overloaded { +std::string NixStringContextElem::to_string() const +{ + std::string res; + + std::function<void(const SingleDerivedPath &)> toStringRest; + toStringRest = [&](auto & p) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + res += o.path.to_string(); + }, + [&](const SingleDerivedPath::Built & o) { + res += o.output; + res += '!'; + toStringRest(*o.drvPath); + }, + }, p.raw()); + }; + + std::visit(overloaded { [&](const NixStringContextElem::Built & b) { - std::string res; - res += '!'; - res += b.output; res += '!'; - res += b.drvPath.to_string(); - return res; + toStringRest(b); + }, + [&](const NixStringContextElem::Opaque & o) { + toStringRest(o); }, [&](const NixStringContextElem::DrvDeep & d) { - std::string res; res += '='; res += d.drvPath.to_string(); - return res; - }, - [&](const NixStringContextElem::Opaque & o) { - return std::string { o.path.to_string() }; }, }, raw()); + + return res; } } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 287ae08a9..a1b71695b 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -3,7 +3,7 @@ #include "util.hh" #include "comparator.hh" -#include "path.hh" +#include "derived-path.hh" #include <variant> @@ -31,11 +31,7 @@ public: * * Encoded as just the path: ‘<path>’. */ -struct NixStringContextElem_Opaque { - StorePath path; - - GENERATE_CMP(NixStringContextElem_Opaque, me->path); -}; +typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque; /** * Path to a derivation and its entire build closure. @@ -57,12 +53,7 @@ struct NixStringContextElem_DrvDeep { * * Encoded in the form ‘!<output>!<drvPath>’. */ -struct NixStringContextElem_Built { - StorePath drvPath; - std::string output; - - GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); -}; +typedef SingleDerivedPath::Built NixStringContextElem_Built; using _NixStringContextElem_Raw = std::variant< NixStringContextElem_Opaque, @@ -93,8 +84,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw { * - ‘<path>’ * - ‘=<path>’ * - ‘!<name>!<path>’ + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static NixStringContextElem parse(std::string_view s); + static NixStringContextElem parse( + std::string_view s, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); std::string to_string() const; }; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5e37f7ecb..8bdf2f367 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -65,7 +65,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -74,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -84,7 +84,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -95,7 +95,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -1490,7 +1490,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) for (auto & outputName : outputs->second) { auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = dg->drvPath, + .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 4aa4d6dca..e941b4e65 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -77,7 +77,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); return goal->getBuildResult(DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); } catch (Error & e) { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 8b7fbdcda..920097680 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1172,6 +1172,19 @@ void LocalDerivationGoal::writeStructuredAttrs() } +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { @@ -1179,7 +1192,7 @@ static StorePath pathPartOfReq(const DerivedPath & req) return bo.path; }, [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; + return pathPartOfReq(*bfd.drvPath); }, }, req.raw()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index a9ca9cbbc..6779dbcf3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -111,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -265,7 +268,7 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) { - topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built{makeConstantStorePathRef(goal->drvPath), goal->wantedOutputs}); } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 52d073f81..3594b7570 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -7,52 +7,133 @@ namespace nix { -nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const { +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ + } +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::string +CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE OutputsSpec +CMP(SingleDerivedPath, DerivedPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +{ + return store.printStorePath(path); +} + +nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["path"] = store->printStorePath(path); + res["drvPath"] = drvPath->toJSON(store); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); + res["output"] = output; + auto outputPathIter = outputMap.find(output); + if (outputPathIter == outputMap.end()) + res["outputPath"] = nullptr; + else if (std::optional p = outputPathIter->second) + res["outputPath"] = store.printStorePath(*p); + else + res["outputPath"] = nullptr; return res; } -nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const { +nlohmann::json DerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); + res["drvPath"] = drvPath->toJSON(store); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - const auto outputMap = store->queryPartialDerivationOutputMap(drvPath); + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); for (const auto & [output, outputPathOpt] : outputMap) { if (!outputs.contains(output)) continue; if (outputPathOpt) - res["outputs"][output] = store->printStorePath(*outputPathOpt); + res["outputs"][output] = store.printStorePath(*outputPathOpt); else res["outputs"][output] = nullptr; } return res; } +nlohmann::json SingleDerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json DerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); } +std::string SingleDerivedPath::Built::to_string(const Store & store) const +{ + return drvPath->to_string(store) + "^" + output; +} + +std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +{ + return drvPath->to_string(store) + "!" + output; +} + std::string DerivedPath::Built::to_string(const Store & store) const { - return store.printStorePath(drvPath) + return drvPath->to_string(store) + '^' + outputs.to_string(); } std::string DerivedPath::Built::to_string_legacy(const Store & store) const { - return store.printStorePath(drvPath) - + '!' + return drvPath->to_string_legacy(store) + + "!" + outputs.to_string(); } +std::string SingleDerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + std::string DerivedPath::to_string(const Store & store) const { + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + +std::string SingleDerivedPath::to_string_legacy(const Store & store) const +{ return std::visit(overloaded { - [&](const DerivedPath::Built & req) { return req.to_string(store); }, - [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const SingleDerivedPath::Opaque & req) { return req.to_string(store); }, }, this->raw()); } @@ -70,30 +151,156 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_ return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings) +{ + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque &) { + // plain drv path; no experimental features required. + }, + [&](const SingleDerivedPath::Built &) { + xpSettings.require(Xp::DynamicDerivations); + }, + }, drv.raw()); +} + +SingleDerivedPath::Built SingleDerivedPath::Built::parse( + const Store & store, ref<SingleDerivedPath> drv, + std::string_view output, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); + return { + .drvPath = drv, + .output = std::string { output }, + }; +} + +DerivedPath::Built DerivedPath::Built::parse( + const Store & store, ref<SingleDerivedPath> drv, + std::string_view outputsS, + const ExperimentalFeatureSettings & xpSettings) { + drvRequireExperiment(*drv, xpSettings); return { - .drvPath = store.parseStorePath(drvS), + .drvPath = drv, .outputs = OutputsSpec::parse(outputsS), }; } -static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) +static SingleDerivedPath parseWithSingle( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) { - size_t n = s.find(separator); + size_t n = s.rfind(separator); + return n == s.npos + ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s) + : (SingleDerivedPath) SingleDerivedPath::Built::parse(store, + make_ref<SingleDerivedPath>(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "^", xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "!", xpSettings); +} + +static DerivedPath parseWith( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) +{ + size_t n = s.rfind(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); + : (DerivedPath) DerivedPath::Built::parse(store, + make_ref<SingleDerivedPath>(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +DerivedPath DerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWith(store, s, "^", xpSettings); +} + +DerivedPath DerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWith(store, s, "!", xpSettings); +} + +DerivedPath DerivedPath::fromSingle(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) -> DerivedPath { + return o; + }, + [&](const SingleDerivedPath::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + }, req.raw()); +} + +const StorePath & SingleDerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +const StorePath & DerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +template<typename DP> +static inline const StorePath & getBaseStorePath_(const DP & derivedPath) +{ + return std::visit(overloaded { + [&](const typename DP::Built & bfd) -> auto & { + return bfd.drvPath->getBaseStorePath(); + }, + [&](const typename DP::Opaque & bo) -> auto & { + return bo.path; + }, + }, derivedPath.raw()); } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +const StorePath & SingleDerivedPath::getBaseStorePath() const { - return parseWith(store, s, "^"); + return getBaseStorePath_(*this); } -DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +const StorePath & DerivedPath::getBaseStorePath() const { - return parseWith(store, s, "!"); + return getBaseStorePath_(*this); } } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 7a4261ce0..ec30dac61 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -24,15 +24,135 @@ class Store; struct DerivedPathOpaque { StorePath path; - nlohmann::json toJSON(ref<Store> store) const; std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; +struct SingleDerivedPath; + +/** + * A single derived path that is built from a derivation + * + * Built derived paths are pair of a derivation and an output name. They are + * evaluated by building the derivation, and then taking the resulting output + * path of the given output name. + */ +struct SingleDerivedPathBuilt { + ref<SingleDerivedPath> drvPath; + std::string output; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPathBuilt parse( + const Store & store, ref<SingleDerivedPath> drvPath, + std::string_view outputs, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; + + DECLARE_CMP(SingleDerivedPathBuilt); +}; + +using _SingleDerivedPathRaw = std::variant< + DerivedPathOpaque, + SingleDerivedPathBuilt +>; + /** - * A derived path that is built from a derivation + * A "derived path" is a very simple sort of expression (not a Nix + * language expression! But an expression in a the general sense) that + * evaluates to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ +struct SingleDerivedPath : _SingleDerivedPathRaw { + using Raw = _SingleDerivedPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleDerivedPathBuilt; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + /** + * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; +}; + +static inline ref<SingleDerivedPath> makeConstantStorePathRef(StorePath drvPath) +{ + return make_ref<SingleDerivedPath>(SingleDerivedPath::Opaque { drvPath }); +} + +/** + * A set of derived paths that are built from a derivation * * Built derived paths are pair of a derivation and some output names. * They are evaluated by building the derivation, and then replacing the @@ -44,10 +164,22 @@ struct DerivedPathOpaque { * output name. */ struct DerivedPathBuilt { - StorePath drvPath; + ref<SingleDerivedPath> drvPath; OutputsSpec outputs; /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** * Uses `^` as the separator */ std::string to_string(const Store & store) const; @@ -57,11 +189,16 @@ struct DerivedPathBuilt { std::string to_string_legacy(const Store & store) const; /** * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); - nlohmann::json toJSON(ref<Store> store) const; + static DerivedPathBuilt parse( + const Store & store, ref<SingleDerivedPath>, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; - GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(DerivedPathBuilt); }; using _DerivedPathRaw = std::variant< @@ -71,13 +208,13 @@ using _DerivedPathRaw = std::variant< /** * A "derived path" is a very simple sort of expression that evaluates - * to (concrete) store path. It is either: + * to one or more (concrete) store paths. It is either: * - * - opaque, in which case it is just a concrete store path with + * - opaque, in which case it is just a single concrete store path with * possibly no known derivation * - * - built, in which case it is a pair of a derivation path and an - * output name. + * - built, in which case it is a pair of a derivation path and some + * output names. */ struct DerivedPath : _DerivedPathRaw { using Raw = _DerivedPathRaw; @@ -91,6 +228,18 @@ struct DerivedPath : _DerivedPathRaw { } /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** * Uses `^` as the separator */ std::string to_string(const Store & store) const; @@ -100,14 +249,43 @@ struct DerivedPath : _DerivedPathRaw { std::string to_string_legacy(const Store & store) const; /** * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parse(const Store & store, std::string_view); + static DerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parseLegacy(const Store & store, std::string_view); + static DerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convert a `SingleDerivedPath` to a `DerivedPath`. + */ + static DerivedPath fromSingle(const SingleDerivedPath &); + + nlohmann::json toJSON(Store & store) const; }; typedef std::vector<DerivedPath> DerivedPaths; +/** + * Used by various parser functions to require experimental features as + * needed. + * + * Somewhat unfortunate this cannot just be an implementation detail for + * this module. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index d623c05e2..885b3604d 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -38,4 +38,19 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( }; } +DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & b) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return DownstreamPlaceholder::unknownCaOutput(o.path, b.output); + }, + [&](const SingleDerivedPath::Built & b2) { + return DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2), + b.output); + }, + }, b.drvPath->raw()); +} + } diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index 97f77e6b8..9372dcd58 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -3,6 +3,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" namespace nix { @@ -73,6 +74,17 @@ public: const DownstreamPlaceholder & drvPlaceholder, std::string_view outputName, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convenience constructor that handles both cases (unknown + * content-addressed output and unknown derivation), delegating as + * needed to `unknownCaOutput` and `unknownDerivation`. + * + * Recursively builds up a placeholder from a + * `SingleDerivedPath::Built.drvPath` chain. + */ + static DownstreamPlaceholder fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & built); }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fa17d606d..78b05031a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -358,6 +358,9 @@ public: [&](const StorePath & drvPath) { throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, }, sOrDrvPath); } conn->to << ss; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 14160dc8b..1ece7a4c0 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -132,7 +132,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); + pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second })); }; auto checkOutput = [&]( @@ -176,10 +176,18 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - if (!isValidPath(bfd.drvPath)) { + auto drvPathP = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath); + if (!drvPathP) { + // TODO make work in this case. + warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(*this)); + return; + } + auto & drvPath = drvPathP->path; + + if (!isValidPath(drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(bfd.drvPath); + state->unknown.insert(drvPath); return; } @@ -187,7 +195,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; @@ -197,15 +205,15 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref<Derivation>(derivationFromPath(bfd.drvPath)); - ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); + auto drv = make_ref<Derivation>(derivationFromPath(drvPath)); + ParsedDerivation parsedDrv(StorePath(drvPath), *drv); if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState)); } else - mustBuildDrv(bfd.drvPath, *drv); + mustBuildDrv(drvPath, *drv); }, [&](const DerivedPath::Opaque & bo) { @@ -310,7 +318,9 @@ std::map<DrvOutput, StorePath> drvOutputReferences( OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { - auto outputsOpt_ = store.queryPartialDerivationOutputMap(bfd.drvPath, evalStore_); + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + + auto outputsOpt_ = store.queryPartialDerivationOutputMap(drvPath, evalStore_); auto outputsOpt = std::visit(overloaded { [&](const OutputsSpec::All &) { @@ -325,7 +335,7 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, if (!pOutputPathOpt) throw Error( "the derivation '%s' doesn't have an output named '%s'", - store.printStorePath(bfd.drvPath), output); + bfd.drvPath->to_string(store), output); outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt)); } return outputsOpt; @@ -335,11 +345,64 @@ OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, OutputPathMap outputs; for (auto & [outputName, outputPathOpt] : outputsOpt) { if (!outputPathOpt) - throw MissingRealisation(store.printStorePath(bfd.drvPath), outputName); + throw MissingRealisation(bfd.drvPath->to_string(store), outputName); auto & outputPath = *outputPathOpt; outputs.insert_or_assign(outputName, outputPath); } return outputs; } + +StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_); + if (outputPaths.count(bfd.output) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store.printStorePath(drvPath), bfd.output); + auto & optPath = outputPaths.at(bfd.output); + if (!optPath) + throw Error("'%s' does not yet map to a known concrete store path", + bfd.to_string(store)); + return *optPath; + }, + }, req.raw()); +} + + +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) +{ + auto drvPath = resolveDerivedPath(store, *bfd.drvPath); + auto outputMap = store.queryDerivationOutputMap(drvPath); + auto outputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast<StringSet>(names); + }, + }, bfd.outputs.raw()); + for (auto iter = outputMap.begin(); iter != outputMap.end();) { + auto & outputName = iter->first; + if (bfd.outputs.contains(outputName)) { + outputsLeft.erase(outputName); + ++iter; + } else { + iter = outputMap.erase(iter); + } + } + if (!outputsLeft.empty()) + throw Error("derivation '%s' does not have an outputs %s", + store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(std::get<OutputsSpec::Names>(bfd.outputs)))); + return outputMap; +} + } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 869b490ad..81f360f3a 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -16,10 +16,16 @@ std::string StorePathWithOutputs::to_string(const Store & store) const DerivedPath StorePathWithOutputs::toDerivedPath() const { if (!outputs.empty()) { - return DerivedPath::Built { path, OutputsSpec::Names { outputs } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::Names { outputs }, + }; } else if (path.isDerivation()) { assert(outputs.empty()); - return DerivedPath::Built { path, OutputsSpec::All { } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::All { }, + }; } else { return DerivedPath::Opaque { path }; } @@ -34,29 +40,36 @@ std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> } -std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) +StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) -> std::variant<StorePathWithOutputs, StorePath> { + [&](const DerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> { - return StorePathWithOutputs { - .path = bfd.drvPath, - // Use legacy encoding of wildcard as empty set - .outputs = std::visit(overloaded { - [&](const OutputsSpec::All &) -> StringSet { - return {}; - }, - [&](const OutputsSpec::Names & outputs) { - return static_cast<StringSet>(outputs); - }, - }, bfd.outputs.raw()), - }; + [&](const DerivedPath::Built & bfd) -> StorePathWithOutputs::ParseResult { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { + return StorePathWithOutputs { + .path = bo.path, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast<StringSet>(outputs); + }, + }, bfd.outputs.raw()), + }; + }, + [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult { + return std::monostate {}; + }, + }, bfd.drvPath->raw()); }, }, p.raw()); } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index d75850868..57e03252d 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -23,7 +23,9 @@ struct StorePathWithOutputs DerivedPath toDerivedPath() const; - static std::variant<StorePathWithOutputs, StorePath> tryFromDerivedPath(const DerivedPath &); + typedef std::variant<StorePathWithOutputs, StorePath, std::monostate> ParseResult; + + static StorePathWithOutputs::ParseResult tryFromDerivedPath(const DerivedPath &); }; std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 21258daec..58f72beb9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -656,6 +656,9 @@ static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & con GET_PROTOCOL_MAJOR(conn.daemonVersion), GET_PROTOCOL_MINOR(conn.daemonVersion)); }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'"); + }, }, sOrDrvPath); } conn.to << ss; @@ -670,9 +673,16 @@ void RemoteStore::copyDrvsFromEvalStore( /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ RealisedPath::Set drvPaths2; - for (auto & i : paths) - if (auto p = std::get_if<DerivedPath::Built>(&i)) - drvPaths2.insert(p->drvPath); + for (const auto & i : paths) { + std::visit(overloaded { + [&](const DerivedPath::Opaque & bp) { + // Do nothing, path is hopefully there already + }, + [&](const DerivedPath::Built & bp) { + drvPaths2.insert(bp.drvPath->getBaseStorePath()); + }, + }, i.raw()); + } copyClosure(*evalStore, *this, drvPaths2); } } @@ -742,7 +752,8 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults( }; OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); + auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); + auto drv = evalStore->readDerivation(drvPath); const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { @@ -750,7 +761,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults( if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", - printStorePath(bfd.drvPath), output); + printStorePath(drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f9029ade1..da1a3eefb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -945,6 +945,7 @@ void removeTempRoots(); * Resolve the derived path completely, failing if any derivation output * is unknown. */ +StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr); OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index 160443ec1..d6549f66f 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -17,14 +17,34 @@ Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary() }); } +Gen<SingleDerivedPath::Built> Arbitrary<SingleDerivedPath::Built>::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()), + .output = (*gen::arbitrary<StorePathName>()).name, + }); +} + Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary() { return gen::just(DerivedPath::Built { - .drvPath = *gen::arbitrary<StorePath>(), + .drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()), .outputs = *gen::arbitrary<OutputsSpec>(), }); } +Gen<SingleDerivedPath> Arbitrary<SingleDerivedPath>::arbitrary() +{ + switch (*gen::inRange<uint8_t>(0, std::variant_size_v<SingleDerivedPath::Raw>)) { + case 0: + return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Opaque>()); + case 1: + return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Built>()); + default: + assert(false); + } +} + Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary() { switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) { @@ -45,12 +65,69 @@ class DerivedPathTest : public LibStoreTest { }; -// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is -// no a real fixture. -// -// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args -TEST_F(DerivedPathTest, force_init) -{ +/** + * Round trip (string <-> data structure) test for + * `DerivedPath::Opaque`. + */ +TEST_F(DerivedPathTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = DerivedPath::parse(*store, opaque); + auto * p = std::get_if<DerivedPath::Opaque>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store->parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(*store), opaque); +} + +/** + * Round trip (string <-> data structure) test for a simpler + * `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_opaque) { + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^bar,foo"; + auto elem = DerivedPath::parse(*store, built); + auto * p = std::get_if<DerivedPath::Built>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "foo", "bar" })); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"; + auto elem = DerivedPath::parse(*store, built, mockXpSettings); + auto * p = std::get_if<DerivedPath::Built>(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "bar", "baz" })); + auto * drvPath = std::get_if<SingleDerivedPath::Built>(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "foo"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive derived path. + */ +TEST_F(DerivedPathTest, built_built_xp) { + ASSERT_THROW( + DerivedPath::parse(*store, "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"), + MissingExperimentalFeature); } RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 506f3ccb1..98d61f228 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -12,8 +12,18 @@ namespace rc { using namespace nix; template<> -struct Arbitrary<DerivedPath::Opaque> { - static Gen<DerivedPath::Opaque> arbitrary(); +struct Arbitrary<SingleDerivedPath::Opaque> { + static Gen<SingleDerivedPath::Opaque> arbitrary(); +}; + +template<> +struct Arbitrary<SingleDerivedPath::Built> { + static Gen<SingleDerivedPath::Built> arbitrary(); +}; + +template<> +struct Arbitrary<SingleDerivedPath> { + static Gen<SingleDerivedPath> arbitrary(); }; template<> diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 9f661c5c3..7982fdc5e 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -2,6 +2,22 @@ ///@file /** + * Declare comparison methods without defining them. + */ +#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \ + bool operator COMPARATOR(const MY_TYPE & other) const; +#define DECLARE_EQUAL(my_type) \ + DECLARE_ONE_CMP(==, my_type) +#define DECLARE_LEQ(my_type) \ + DECLARE_ONE_CMP(<, my_type) +#define DECLARE_NEQ(my_type) \ + DECLARE_ONE_CMP(!=, my_type) +#define DECLARE_CMP(my_type) \ + DECLARE_EQUAL(my_type) \ + DECLARE_LEQ(my_type) \ + DECLARE_NEQ(my_type) + +/** * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. * diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6510df8f0..66f319c3e 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -393,7 +393,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { - .drvPath = bashDrv, + .drvPath = makeConstantStorePathRef(bashDrv), .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); @@ -417,7 +417,7 @@ static void main_nix_build(int argc, char * * argv) })) { pathsToBuild.push_back(DerivedPath::Built { - .drvPath = inputDrv, + .drvPath = makeConstantStorePathRef(inputDrv), .outputs = OutputsSpec::Names { inputOutputs }, }); pathsToCopy.insert(inputDrv); @@ -590,7 +590,10 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); + pathsToBuild.push_back(DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::Names{outputName}, + }); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 91b073b49..8dd071aa6 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -481,7 +481,7 @@ static void printMissing(EvalState & state, DrvInfos & elems) for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) targets.push_back(DerivedPath::Built{ - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }); else @@ -759,7 +759,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) std::vector<DerivedPath> paths { drvPath ? (DerivedPath) (DerivedPath::Built { - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }) : (DerivedPath) (DerivedPath::Opaque { diff --git a/src/nix/app.cc b/src/nix/app.cc index e0f68b4fc..16a921194 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -25,7 +25,8 @@ StringPairs resolveRewrites( if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) for (auto & [ outputName, outputPath ] : drvDep->outputs) res.emplace( - DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), + DownstreamPlaceholder::unknownCaOutput( + drvDep->drvPath->outPath(), outputName).render(), store.printStorePath(outputPath) ); return res; @@ -65,7 +66,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { /* We want all outputs of the drv */ return DerivedPath::Built { - .drvPath = d.drvPath, + .drvPath = makeConstantStorePathRef(d.drvPath), .outputs = OutputsSpec::All {}, }; }, @@ -106,7 +107,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { .context = { DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::Names { outputName }, } }, .program = program, diff --git a/src/nix/build.cc b/src/nix/build.cc index ad1842a4e..479100186 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -9,18 +9,18 @@ using namespace nix; -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store) +static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store) { auto res = nlohmann::json::array(); for (auto & t : paths) { - std::visit([&res, store](const auto & t) { + std::visit([&](const auto & t) { res.push_back(t.toJSON(store)); }, t.raw()); } return res; } -nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult> & buildables, ref<Store> store) +static nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult> & buildables, const Store & store) { auto res = nlohmann::json::array(); for (auto & b : buildables) { @@ -125,7 +125,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile printMissing(store, pathsToBuild, lvlError); if (json) - logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + logger->cout("%s", derivedPathsToJSON(pathsToBuild, *store).dump()); return; } @@ -136,7 +136,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile installables, repair ? bmRepair : buildMode); - if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, *store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index bcc00d490..5a80f0308 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -109,7 +109,7 @@ struct CmdBundle : InstallableValueCommand store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 195eeaa21..c033804e4 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -235,7 +235,7 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore /* Build the derivation. */ store->buildPaths( { DerivedPath::Built { - .drvPath = shellDrvPath, + .drvPath = makeConstantStorePathRef(shellDrvPath), .outputs = OutputsSpec::All { }, }}, bmNormal, evalStore); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3ce1de44a..83b74c8ca 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -544,9 +544,9 @@ struct CmdFlakeCheck : FlakeCommand *attr2.value, attr2.pos); if (drvPath && attr_name == settings.thisSystem.get()) { drvPaths.push_back(DerivedPath::Built { - .drvPath = *drvPath, - .outputs = OutputsSpec::All { }, - }); + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All { }, + }); } } } diff --git a/src/nix/log.cc b/src/nix/log.cc index aaf829764..9a9bd30f9 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -33,6 +33,17 @@ struct CmdLog : InstallableCommand auto b = installable->toDerivedPath(); + // For compat with CLI today, TODO revisit + auto oneUp = std::visit(overloaded { + [&](const DerivedPath::Opaque & bo) { + return make_ref<SingleDerivedPath>(bo); + }, + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + }, b.path.raw()); + auto path = resolveDerivedPath(*store, *oneUp); + RunPager pager; for (auto & sub : subs) { auto * logSubP = dynamic_cast<LogStore *>(&*sub); @@ -42,14 +53,7 @@ struct CmdLog : InstallableCommand } auto & logSub = *logSubP; - auto log = std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) { - return logSub.getBuildLog(bo.path); - }, - [&](const DerivedPath::Built & bfd) { - return logSub.getBuildLog(bfd.drvPath); - }, - }, b.path.raw()); + auto log = logSub.getBuildLog(path); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); |