diff options
Diffstat (limited to 'src/libstore/derivations.cc')
| -rw-r--r-- | src/libstore/derivations.cc | 637 |
1 files changed, 525 insertions, 112 deletions
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 05dc9a3cc..67069c3c9 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1,15 +1,18 @@ #include "derivations.hh" +#include "downstream-placeholder.hh" #include "store-api.hh" #include "globals.hh" #include "util.hh" +#include "split.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "fs-accessor.hh" #include <boost/container/small_vector.hpp> #include <nlohmann/json.hpp> namespace nix { -std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const +std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const { return std::visit(overloaded { [](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> { @@ -29,15 +32,15 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string [](const DerivationOutput::Impure &) -> std::optional<StorePath> { return std::nullopt; }, - }, raw()); + }, raw); } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const +StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const { - return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + return store.makeFixedOutputPathFromCA( + outputPathName(drvName, outputName), + ContentAddressWithReferences::withoutRefs(ca)); } @@ -57,7 +60,7 @@ bool DerivationType::isCA() const [](const Impure &) { return true; }, - }, raw()); + }, raw); } bool DerivationType::isFixed() const @@ -72,7 +75,7 @@ bool DerivationType::isFixed() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } bool DerivationType::hasKnownOutputPaths() const @@ -87,7 +90,7 @@ bool DerivationType::hasKnownOutputPaths() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } @@ -103,7 +106,7 @@ bool DerivationType::isSandboxed() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } @@ -119,7 +122,7 @@ bool DerivationType::isPure() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } @@ -133,7 +136,7 @@ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) { auto references = drv.inputSrcs; - for (auto & i : drv.inputDrvs) + for (auto & i : drv.inputDrvs.map) references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be @@ -151,8 +154,9 @@ static void expect(std::istream & str, std::string_view s) { char s2[s.size()]; str.read(s2, s.size()); - if (std::string(s2, s.size()) != s) - throw FormatError("expected string '%1%'", s); + std::string_view s2View { s2, s.size() }; + if (s2View != s) + throw FormatError("expected string '%s', got '%s'", s, s2View); } @@ -204,40 +208,44 @@ static bool endOfList(std::istream & str) static StringSet parseStrings(std::istream & str, bool arePaths) { StringSet res; + expect(str, "["); while (!endOfList(str)) res.insert(arePaths ? parsePath(str) : parseString(str)); return res; } -static DerivationOutput parseDerivationOutput(const Store & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hash) +static DerivationOutput parseDerivationOutput( + const Store & store, + std::string_view pathS, std::string_view hashAlgo, std::string_view hashS, + const ExperimentalFeatureSettings & xpSettings) { if (hashAlgo != "") { - auto method = FileIngestionMethod::Flat; - if (hashAlgo.substr(0, 2) == "r:") { - method = FileIngestionMethod::Recursive; - hashAlgo = hashAlgo.substr(2); - } + ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo); + if (method == TextIngestionMethod {}) + xpSettings.require(Xp::DynamicDerivations); const auto hashType = parseHashType(hashAlgo); - if (hash == "impure") { - settings.requireExperimentalFeature(Xp::ImpureDerivations); - assert(pathS == ""); + if (hashS == "impure") { + xpSettings.require(Xp::ImpureDerivations); + if (pathS != "") + throw FormatError("impure derivation output should not specify output path"); return DerivationOutput::Impure { .method = std::move(method), .hashType = std::move(hashType), }; - } else if (hash != "") { + } else if (hashS != "") { validatePath(pathS); + auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); return DerivationOutput::CAFixed { - .hash = FixedOutputHash { + .ca = ContentAddress { .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(hash, hashType), + .hash = std::move(hash), }, }; } else { - settings.requireExperimentalFeature(Xp::CaDerivations); - assert(pathS == ""); + xpSettings.require(Xp::CaDerivations); + if (pathS != "") + throw FormatError("content-addressed derivation output should not specify output path"); return DerivationOutput::CAFloating { .method = std::move(method), .hashType = std::move(hashType), @@ -254,29 +262,116 @@ static DerivationOutput parseDerivationOutput(const Store & store, } } -static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) +static DerivationOutput parseDerivationOutput( + const Store & store, std::istringstream & str, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); expect(str, ","); const auto hashAlgo = parseString(str); expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - return parseDerivationOutput(store, pathS, hashAlgo, hash); + return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings); } +/** + * All ATerm Derivation format versions currently known. + * + * Unknown versions are rejected at the parsing stage. + */ +enum struct DerivationATermVersion { + /** + * Older unversioned form + */ + Traditional, + + /** + * Newer versioned form; only this version so far. + */ + DynamicDerivations, +}; + +static DerivedPathMap<StringSet>::ChildNode parseDerivedPathMapNode( + const Store & store, + std::istringstream & str, + DerivationATermVersion version) +{ + DerivedPathMap<StringSet>::ChildNode node; + + auto parseNonDynamic = [&]() { + node.value = parseStrings(str, false); + }; -Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) + // Older derivation should never use new form, but newer + // derivaiton can use old form. + switch (version) { + case DerivationATermVersion::Traditional: + parseNonDynamic(); + break; + case DerivationATermVersion::DynamicDerivations: + switch (str.peek()) { + case '[': + parseNonDynamic(); + break; + case '(': + expect(str, "("); + node.value = parseStrings(str, false); + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + auto outputName = parseString(str); + expect(str, ","); + node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version)); + expect(str, ")"); + } + expect(str, ")"); + break; + default: + throw FormatError("invalid inputDrvs entry in derivation"); + } + break; + default: + // invalid format, not a parse error but internal error + assert(false); + } + return node; +} + + +Derivation parseDerivation( + const Store & store, std::string && s, std::string_view name, + const ExperimentalFeatureSettings & xpSettings) { Derivation drv; drv.name = name; std::istringstream str(std::move(s)); - expect(str, "Derive(["); + expect(str, "D"); + DerivationATermVersion version; + switch (str.peek()) { + case 'e': + expect(str, "erive("); + version = DerivationATermVersion::Traditional; + break; + case 'r': + expect(str, "rvWithVersion("); + auto versionS = parseString(str); + if (versionS == "xp-dyn-drv") { + // Only verison we have so far + version = DerivationATermVersion::DynamicDerivations; + xpSettings.require(Xp::DynamicDerivations); + } else { + throw FormatError("Unknown derivation ATerm format version '%s'", versionS); + } + expect(str, ","); + break; + } /* Parse the list of outputs. */ + expect(str, "["); while (!endOfList(str)) { expect(str, "("); std::string id = parseString(str); - auto output = parseDerivationOutput(store, str); + auto output = parseDerivationOutput(store, str, xpSettings); drv.outputs.emplace(std::move(id), std::move(output)); } @@ -285,12 +380,12 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi while (!endOfList(str)) { expect(str, "("); Path drvPath = parsePath(str); - expect(str, ",["); - drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false)); + expect(str, ","); + drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version)); expect(str, ")"); } - expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); + expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); expect(str, ","); drv.platform = parseString(str); expect(str, ","); drv.builder = parseString(str); @@ -313,6 +408,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi } +/** + * Print a derivation string literal to an `std::string`. + * + * This syntax does not generalize to the expression language, which needs to + * escape `$`. + * + * @param res Where to print to + * @param s Which logical string to print + */ static void printString(std::string & res, std::string_view s) { boost::container::small_vector<char, 64 * 1024> buffer; @@ -365,14 +469,67 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt } +static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap<StringSet>::ChildNode & node) +{ + s += ','; + if (node.childMap.empty()) { + printUnquotedStrings(s, node.value.begin(), node.value.end()); + } else { + s += "("; + printUnquotedStrings(s, node.value.begin(), node.value.end()); + s += ",["; + bool first = true; + for (auto & [outputName, childNode] : node.childMap) { + if (first) first = false; else s += ','; + s += '('; printUnquotedString(s, outputName); + unparseDerivedPathMapNode(store, s, childNode); + s += ')'; + } + s += "])"; + } +} + + +/** + * Does the derivation have a dependency on the output of a dynamic + * derivation? + * + * In other words, does it on the output of derivation that is itself an + * ouput of a derivation? This corresponds to a dependency that is an + * inductive derived path with more than one layer of + * `DerivedPath::Built`. + */ +static bool hasDynamicDrvDep(const Derivation & drv) +{ + return + std::find_if( + drv.inputDrvs.map.begin(), + drv.inputDrvs.map.end(), + [](auto & kv) { return !kv.second.childMap.empty(); }) + != drv.inputDrvs.map.end(); +} + + std::string Derivation::unparse(const Store & store, bool maskOutputs, - std::map<std::string, StringSet> * actualInputs) const + DerivedPathMap<StringSet>::ChildNode::Map * actualInputs) const { std::string s; s.reserve(65536); - s += "Derive(["; + + /* Use older unversioned form if possible, for wider compat. Use + newer form only if we need it, which we do for + `Xp::DynamicDerivations`. */ + if (hasDynamicDrvDep(*this)) { + s += "DrvWithVersion("; + // Only version we have so far + printUnquotedString(s, "xp-dyn-drv"); + s += ","; + } else { + s += "Derive("; + } bool first = true; + s += "["; for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); @@ -384,12 +541,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, }, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); - s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, [&](const DerivationOutput::Deferred &) { @@ -397,30 +554,30 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); }, - [&](const DerivationOutputImpure & doi) { + [&](const DerivationOutput::Impure & doi) { // FIXME s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); + s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType)); s += ','; printUnquotedString(s, "impure"); } - }, i.second.raw()); + }, i.second.raw); s += ')'; } s += "],["; first = true; if (actualInputs) { - for (auto & i : *actualInputs) { + for (auto & [drvHashModulo, childMap] : *actualInputs) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, drvHashModulo); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } else { - for (auto & i : inputDrvs) { + for (auto & [drvPath, childMap] : inputDrvs.map) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, store.printStorePath(i.first)); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, store.printStorePath(drvPath)); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } @@ -455,7 +612,7 @@ bool isDerivation(std::string_view fileName) } -std::string outputPathName(std::string_view drvName, std::string_view outputName) { +std::string outputPathName(std::string_view drvName, OutputNameView outputName) { std::string res { drvName }; if (outputName != "out") { res += "-"; @@ -498,7 +655,7 @@ DerivationType BasicDerivation::type() const [&](const DerivationOutput::Impure &) { impureOutputs.insert(i.first); }, - }, i.second.raw()); + }, i.second.raw); } if (inputAddressedOutputs.empty() @@ -615,10 +772,10 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut if (type.isFixed()) { std::map<std::string, Hash> outputHashes; for (const auto & i : drv.outputs) { - auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw()); + auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw); auto hash = hashString(htSHA256, "fixed:out:" - + dof.hash.printMethodAlgo() + ":" - + dof.hash.hash.to_string(Base16, false) + ":" + + dof.ca.printMethodAlgo() + ":" + + dof.ca.hash.to_string(Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -652,20 +809,18 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut [](const DerivationType::Impure &) -> DrvHash::Kind { assert(false); } - }, drv.type().raw()); + }, drv.type().raw); - std::map<std::string, StringSet> inputs2; - for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { - // Avoid lambda capture restriction with standard / Clang - auto & inputOutputs = inputOutputs0; + DerivedPathMap<StringSet>::ChildNode::Map inputs2; + for (auto & [drvPath, node] : drv.inputDrvs.map) { const auto & res = pathDerivationModulo(store, drvPath); if (res.kind == DrvHash::Kind::Deferred) kind = DrvHash::Kind::Deferred; - for (auto & outputName : inputOutputs) { + for (auto & outputName : node.value) { const auto h = get(res.hashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(Base16, false)].insert(outputName); + inputs2[h->to_string(Base16, false)].value.insert(outputName); } } @@ -695,7 +850,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) const auto hashAlgo = readString(in); const auto hash = readString(in); - return parseDerivationOutput(store, pathS, hashAlgo, hash); + return parseDerivationOutput(store, pathS, hashAlgo, hash, experimentalFeatureSettings); } StringSet BasicDerivation::outputNames() const @@ -709,10 +864,10 @@ StringSet BasicDerivation::outputNames() const DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { DerivationOutputsAndOptPaths outsAndOptPaths; - for (auto output : outputs) + for (auto & [outputName, output] : outputs) outsAndOptPaths.insert(std::make_pair( - output.first, - std::make_pair(output.second, output.second.path(store, name, output.first)) + outputName, + std::make_pair(output, output.path(store, name, outputName)) ) ); return outsAndOptPaths; @@ -740,7 +895,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {}); + drv.inputSrcs = WorkerProto::Serialise<StorePathSet>::read(store, + WorkerProto::ReadConn { .from = in }); in >> drv.platform >> drv.builder; drv.args = readStrings<Strings>(in); @@ -768,12 +924,12 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) - << dof.hash.printMethodAlgo() - << dof.hash.hash.to_string(Base16, false); + << dof.ca.printMethodAlgo() + << dof.ca.hash.to_string(Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" - << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) + << (dof.method.renderPrefix() + printHashType(dof.hashType)) << ""; }, [&](const DerivationOutput::Deferred &) { @@ -783,12 +939,14 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, [&](const DerivationOutput::Impure & doi) { out << "" - << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)) + << (doi.method.renderPrefix() + printHashType(doi.hashType)) << "impure"; }, - }, i.second.raw()); + }, i.second.raw); } - worker_proto::write(store, out, drv.inputSrcs); + WorkerProto::write(store, + WorkerProto::WriteConn { .to = out }, + drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) @@ -796,23 +954,19 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr } -std::string hashPlaceholder(const std::string_view outputName) +std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); } -std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName) -{ - auto drvNameWithExtension = drvPath.name(); - auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); - auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); - return "/" + hashString(htSHA256, clearText).to_string(Base32, false); -} + static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { + debug("Rewriting the derivation"); + for (auto & rewrite : rewrites) { debug("rewriting %s as %s", rewrite.first, rewrite.second); } @@ -832,7 +986,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) { + if (std::holds_alternative<DerivationOutput::Deferred>(output.raw)) { auto h = get(hashModulo.hashes, outputName); if (!h) throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", @@ -851,14 +1005,70 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const { std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs; - for (auto & input : inputDrvs) - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first)) - if (outputPath) - inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath); + std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accum; + accum = [&](auto & inputDrv, auto & node) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) { + if (outputPath) { + inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); + if (auto p = get(node.childMap, outputName)) + accum(*outputPath, *p); + } + } + }; + + for (auto & [inputDrv, node] : inputDrvs.map) + accum(inputDrv, node); return tryResolve(store, inputDrvOutputs); } +static bool tryResolveInput( + Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, + const DownstreamPlaceholder * placeholderOpt, + const StorePath & inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode, + const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) +{ + auto getOutput = [&](const std::string & outputName) { + auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName }); + if (!actualPathOpt) + warn("output %s of input %s missing, aborting the resolving", + outputName, + store.printStorePath(inputDrv) + ); + return actualPathOpt; + }; + + auto getPlaceholder = [&](const std::string & outputName) { + return placeholderOpt + ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName) + : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName); + }; + + for (auto & outputName : inputNode.value) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + inputRewrites.emplace( + getPlaceholder(outputName).render(), + store.printStorePath(actualPath)); + } + inputSrcs.insert(std::move(actualPath)); + } + + for (auto & [outputName, childNode] : inputNode.childMap) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + auto nextPlaceholder = getPlaceholder(outputName); + if (!tryResolveInput(store, inputSrcs, inputRewrites, + &nextPlaceholder, actualPath, childNode, + inputDrvOutputs)) + return false; + } + return true; +} + std::optional<BasicDerivation> Derivation::tryResolve( Store & store, const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const @@ -868,31 +1078,81 @@ std::optional<BasicDerivation> Derivation::tryResolve( // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & [inputDrv, inputOutputs] : inputDrvs) { - for (auto & outputName : inputOutputs) { - if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { - inputRewrites.emplace( - downstreamPlaceholder(store, inputDrv, outputName), - store.printStorePath(*actualPath)); - resolved.inputSrcs.insert(*actualPath); - } else { - warn("output '%s' of input '%s' missing, aborting the resolving", - outputName, - store.printStorePath(inputDrv)); - return {}; - } - } - } + for (auto & [inputDrv, inputNode] : inputDrvs.map) + if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, + nullptr, inputDrv, inputNode, inputDrvOutputs)) + return std::nullopt; rewriteDerivation(store, resolved, inputRewrites); return resolved; } + +void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +{ + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); + drvName = drvName.substr(0, drvName.size() - drvExtension.size()); + + if (drvName != name) { + throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name); + } + + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) + { + auto j = env.find(varName); + if (j == env.end() || store.parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + store.printStorePath(drvPath), varName, store.printStorePath(actual)); + }; + + + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + type(); + + std::optional<DrvHash> hashesModulo; + for (auto & i : outputs) { + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doia) { + if (!hashesModulo) { + // somewhat expensive so we do lazily + hashesModulo = hashDerivationModulo(store, *this, true); + } + auto currentOutputHash = get(hashesModulo->hashes, i.first); + if (!currentOutputHash) + throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), i.first); + StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](const DerivationOutput::CAFixed & dof) { + auto path = dof.path(store, drvName, i.first); + envHasRightPath(path, i.first); + }, + [&](const DerivationOutput::CAFloating &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, + }, i.second.raw); + } +} + + const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( - const Store & store, std::string_view drvName, std::string_view outputName) const + const Store & store, std::string_view drvName, OutputNameView outputName) const { nlohmann::json res = nlohmann::json::object(); std::visit(overloaded { @@ -901,25 +1161,99 @@ nlohmann::json DerivationOutput::toJSON( }, [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); - res["hashAlgo"] = dof.hash.printMethodAlgo(); - res["hash"] = dof.hash.hash.to_string(Base16, false); + res["hashAlgo"] = dof.ca.printMethodAlgo(); + res["hash"] = dof.ca.hash.to_string(Base16, false); + // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { - res["hashAlgo"] = makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType); + res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType); }, [&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Impure & doi) { - res["hashAlgo"] = makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType); + res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType); res["impure"] = true; }, - }, raw()); + }, raw); return res; } + +DerivationOutput DerivationOutput::fromJSON( + const Store & store, std::string_view drvName, OutputNameView outputName, + const nlohmann::json & _json, + const ExperimentalFeatureSettings & xpSettings) +{ + std::set<std::string_view> keys; + ensureType(_json, nlohmann::detail::value_t::object); + auto json = (std::map<std::string, nlohmann::json>) _json; + + for (const auto & [key, _] : json) + keys.insert(key); + + auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashType> { + std::string hashAlgo = json["hashAlgo"]; + // remaining to parse, will be mutated by parsers + std::string_view s = hashAlgo; + ContentAddressMethod method = ContentAddressMethod::parsePrefix(s); + if (method == TextIngestionMethod {}) + xpSettings.require(Xp::DynamicDerivations); + auto hashType = parseHashType(s); + return { std::move(method), std::move(hashType) }; + }; + + if (keys == (std::set<std::string_view> { "path" })) { + return DerivationOutput::InputAddressed { + .path = store.parseStorePath((std::string) json["path"]), + }; + } + + else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) { + auto [method, hashType] = methodAlgo(); + auto dof = DerivationOutput::CAFixed { + .ca = ContentAddress { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + }, + }; + if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) + throw Error("Path doesn't match derivation output"); + return dof; + } + + else if (keys == (std::set<std::string_view> { "hashAlgo" })) { + xpSettings.require(Xp::CaDerivations); + auto [method, hashType] = methodAlgo(); + return DerivationOutput::CAFloating { + .method = std::move(method), + .hashType = std::move(hashType), + }; + } + + else if (keys == (std::set<std::string_view> { })) { + return DerivationOutput::Deferred {}; + } + + else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) { + xpSettings.require(Xp::ImpureDerivations); + auto [method, hashType] = methodAlgo(); + return DerivationOutput::Impure { + .method = std::move(method), + .hashType = hashType, + }; + } + + else { + throw Error("invalid JSON for derivation output"); + } +} + + nlohmann::json Derivation::toJSON(const Store & store) const { nlohmann::json res = nlohmann::json::object(); + res["name"] = name; + { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); @@ -936,10 +1270,25 @@ nlohmann::json Derivation::toJSON(const Store & store) const } { - auto& inputDrvsObj = res["inputDrvs"]; - inputDrvsObj = nlohmann::json ::object(); - for (auto & input : inputDrvs) - inputDrvsObj[store.printStorePath(input.first)] = input.second; + std::function<nlohmann::json(const DerivedPathMap<StringSet>::ChildNode &)> doInput; + doInput = [&](const auto & inputNode) { + auto value = nlohmann::json::object(); + value["outputs"] = inputNode.value; + { + auto next = nlohmann::json::object(); + for (auto & [outputId, childNode] : inputNode.childMap) + next[outputId] = doInput(childNode); + value["dynamicOutputs"] = std::move(next); + } + return value; + }; + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : inputDrvs.map) { + inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode); + } + } } res["system"] = platform; @@ -950,4 +1299,68 @@ nlohmann::json Derivation::toJSON(const Store & store) const return res; } + +Derivation Derivation::fromJSON( + const Store & store, + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings) +{ + using nlohmann::detail::value_t; + + Derivation res; + + ensureType(json, value_t::object); + + res.name = ensureType(valueAt(json, "name"), value_t::string); + + try { + auto & outputsObj = ensureType(valueAt(json, "outputs"), value_t::object); + for (auto & [outputName, output] : outputsObj.items()) { + res.outputs.insert_or_assign( + outputName, + DerivationOutput::fromJSON(store, res.name, outputName, output)); + } + } catch (Error & e) { + e.addTrace({}, "while reading key 'outputs'"); + throw; + } + + try { + auto & inputsList = ensureType(valueAt(json, "inputSrcs"), value_t::array); + for (auto & input : inputsList) + res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputSrcs'"); + throw; + } + + try { + std::function<DerivedPathMap<StringSet>::ChildNode(const nlohmann::json &)> doInput; + doInput = [&](const auto & json) { + DerivedPathMap<StringSet>::ChildNode node; + node.value = static_cast<const StringSet &>( + ensureType(valueAt(json, "outputs"), value_t::array)); + for (auto & [outputId, childNode] : ensureType(valueAt(json, "dynamicOutputs"), value_t::object).items()) { + xpSettings.require(Xp::DynamicDerivations); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; + auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object); + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = + doInput(inputOutputs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputDrvs'"); + throw; + } + + res.platform = ensureType(valueAt(json, "system"), value_t::string); + res.builder = ensureType(valueAt(json, "builder"), value_t::string); + res.args = ensureType(valueAt(json, "args"), value_t::array); + res.env = ensureType(valueAt(json, "env"), value_t::object); + + return res; +} + } |