Merge remote-tracking branch 'upstream/master' into path-info

1 files changed, 198 insertions, 116 deletions

diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 1b64819f6..08d5e5c5e 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -25,26 +25,42 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
         [](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
             return std::nullopt;
         },
+        [](const DerivationOutput::Impure &) -> std::optional<StorePath> {
+            return std::nullopt;
+        },
     }, raw());
 }
 
 
-StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
+StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
+{
     return store.makeFixedOutputPath(
         outputPathName(drvName, outputName),
         { hash, {} });
 }
 
 
-bool DerivationType::isCA() const {
+bool DerivationType::isCA() const
+{
     /* Normally we do the full `std::visit` to make sure we have
        exhaustively handled all variants, but so long as there is a
        variant called `ContentAddressed`, it must be the only one for
        which `isCA` is true for this to make sense!. */
-    return std::holds_alternative<ContentAddressed>(raw());
+    return std::visit(overloaded {
+        [](const InputAddressed & ia) {
+            return false;
+        },
+        [](const ContentAddressed & ca) {
+            return true;
+        },
+        [](const Impure &) {
+            return true;
+        },
+    }, raw());
 }
 
-bool DerivationType::isFixed() const {
+bool DerivationType::isFixed() const
+{
     return std::visit(overloaded {
         [](const InputAddressed & ia) {
             return false;
@@ -52,10 +68,14 @@ bool DerivationType::isFixed() const {
         [](const ContentAddressed & ca) {
             return ca.fixed;
         },
+        [](const Impure &) {
+            return false;
+        },
     }, raw());
 }
 
-bool DerivationType::hasKnownOutputPaths() const {
+bool DerivationType::hasKnownOutputPaths() const
+{
     return std::visit(overloaded {
         [](const InputAddressed & ia) {
             return !ia.deferred;
@@ -63,17 +83,40 @@ bool DerivationType::hasKnownOutputPaths() const {
         [](const ContentAddressed & ca) {
             return ca.fixed;
         },
+        [](const Impure &) {
+            return false;
+        },
     }, raw());
 }
 
 
-bool DerivationType::isImpure() const {
+bool DerivationType::isSandboxed() const
+{
     return std::visit(overloaded {
         [](const InputAddressed & ia) {
+            return true;
+        },
+        [](const ContentAddressed & ca) {
+            return ca.sandboxed;
+        },
+        [](const Impure &) {
             return false;
         },
+    }, raw());
+}
+
+
+bool DerivationType::isPure() const
+{
+    return std::visit(overloaded {
+        [](const InputAddressed & ia) {
+            return true;
+        },
         [](const ContentAddressed & ca) {
-            return !ca.pure;
+            return true;
+        },
+        [](const Impure &) {
+            return false;
         },
     }, raw());
 }
@@ -176,7 +219,14 @@ static DerivationOutput parseDerivationOutput(const Store & store,
             hashAlgo = hashAlgo.substr(2);
         }
         const auto hashType = parseHashType(hashAlgo);
-        if (hash != "") {
+        if (hash == "impure") {
+            settings.requireExperimentalFeature(Xp::ImpureDerivations);
+            assert(pathS == "");
+            return DerivationOutput::Impure {
+                .method = std::move(method),
+                .hashType = std::move(hashType),
+            };
+        } else if (hash != "") {
             validatePath(pathS);
             return DerivationOutput::CAFixed {
                 .hash = FixedOutputHash {
@@ -345,6 +395,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
                 s += ','; printUnquotedString(s, "");
+            },
+            [&](const DerivationOutputImpure & doi) {
+                // FIXME
+                s += ','; printUnquotedString(s, "");
+                s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
+                s += ','; printUnquotedString(s, "impure");
             }
         }, i.second.raw());
         s += ')';
@@ -410,8 +466,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
 
 DerivationType BasicDerivation::type() const
 {
-    std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
+    std::set<std::string_view>
+        inputAddressedOutputs,
+        fixedCAOutputs,
+        floatingCAOutputs,
+        deferredIAOutputs,
+        impureOutputs;
     std::optional<HashType> floatingHashType;
+
     for (auto & i : outputs) {
         std::visit(overloaded {
             [&](const DerivationOutput::InputAddressed &) {
@@ -426,43 +488,78 @@ DerivationType BasicDerivation::type() const
                     floatingHashType = dof.hashType;
                 } else {
                     if (*floatingHashType != dof.hashType)
-                        throw Error("All floating outputs must use the same hash type");
+                        throw Error("all floating outputs must use the same hash type");
                 }
             },
             [&](const DerivationOutput::Deferred &) {
-               deferredIAOutputs.insert(i.first);
+                deferredIAOutputs.insert(i.first);
+            },
+            [&](const DerivationOutput::Impure &) {
+                impureOutputs.insert(i.first);
             },
         }, i.second.raw());
     }
 
-    if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
-        throw Error("Must have at least one output");
-    } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
+    if (inputAddressedOutputs.empty()
+        && fixedCAOutputs.empty()
+        && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty()
+        && impureOutputs.empty())
+        throw Error("must have at least one output");
+
+    if (!inputAddressedOutputs.empty()
+        && fixedCAOutputs.empty()
+        && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty()
+        && impureOutputs.empty())
         return DerivationType::InputAddressed {
             .deferred = false,
         };
-    } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
+
+    if (inputAddressedOutputs.empty()
+        && !fixedCAOutputs.empty()
+        && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty()
+        && impureOutputs.empty())
+    {
         if (fixedCAOutputs.size() > 1)
             // FIXME: Experimental feature?
-            throw Error("Only one fixed output is allowed for now");
+            throw Error("only one fixed output is allowed for now");
         if (*fixedCAOutputs.begin() != "out")
-            throw Error("Single fixed output must be named \"out\"");
+            throw Error("single fixed output must be named \"out\"");
         return DerivationType::ContentAddressed {
-            .pure = false,
+            .sandboxed = false,
             .fixed = true,
         };
-    } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
+    }
+
+    if (inputAddressedOutputs.empty()
+        && fixedCAOutputs.empty()
+        && !floatingCAOutputs.empty()
+        && deferredIAOutputs.empty()
+        && impureOutputs.empty())
         return DerivationType::ContentAddressed {
-            .pure = true,
+            .sandboxed = true,
             .fixed = false,
         };
-    } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
+
+    if (inputAddressedOutputs.empty()
+        && fixedCAOutputs.empty()
+        && floatingCAOutputs.empty()
+        && !deferredIAOutputs.empty()
+        && impureOutputs.empty())
         return DerivationType::InputAddressed {
             .deferred = true,
         };
-    } else {
-        throw Error("Can't mix derivation output types");
-    }
+
+    if (inputAddressedOutputs.empty()
+        && fixedCAOutputs.empty()
+        && floatingCAOutputs.empty()
+        && deferredIAOutputs.empty()
+        && !impureOutputs.empty())
+        return DerivationType::Impure { };
+
+    throw Error("can't mix derivation output types");
 }
 
 
@@ -474,7 +571,7 @@ Sync<DrvHashes> drvHashes;
 /* Look up the derivation by value and memoize the
    `hashDerivationModulo` call.
  */
-static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
+static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath)
 {
     {
         auto hashes = drvHashes.lock();
@@ -509,7 +606,7 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
    don't leak the provenance of fixed outputs, reducing pointless cache
    misses as the build itself won't know this.
  */
-DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
+DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
 {
     auto type = drv.type();
 
@@ -524,7 +621,20 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
                 + store.printStorePath(dof.path(store, drv.name, i.first)));
             outputHashes.insert_or_assign(i.first, std::move(hash));
         }
-        return outputHashes;
+        return DrvHash {
+            .hashes = outputHashes,
+            .kind = DrvHash::Kind::Regular,
+        };
+    }
+
+    if (!type.isPure()) {
+        std::map<std::string, Hash> outputHashes;
+        for (const auto & [outputName, _] : drv.outputs)
+            outputHashes.insert_or_assign(outputName, impureOutputHash);
+        return DrvHash {
+            .hashes = outputHashes,
+            .kind = DrvHash::Kind::Deferred,
+        };
     }
 
     auto kind = std::visit(overloaded {
@@ -538,67 +648,41 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
                 ? DrvHash::Kind::Regular
                 : DrvHash::Kind::Deferred;
         },
+        [](const DerivationType::Impure &) -> DrvHash::Kind {
+            assert(false);
+        }
     }, drv.type().raw());
 
-    /* For other derivations, replace the inputs paths with recursive
-       calls to this function. */
     std::map<std::string, StringSet> inputs2;
     for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
         // Avoid lambda capture restriction with standard / Clang
         auto & inputOutputs = inputOutputs0;
         const auto & res = pathDerivationModulo(store, drvPath);
-        std::visit(overloaded {
-            // Regular non-CA derivation, replace derivation
-            [&](const DrvHash & drvHash) {
-                kind |= drvHash.kind;
-                inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs);
-            },
-            // CA derivation's output hashes
-            [&](const CaOutputHashes & outputHashes) {
-                std::set<std::string> justOut = { "out" };
-                for (auto & output : inputOutputs) {
-                    /* Put each one in with a single "out" output.. */
-                    const auto h = outputHashes.at(output);
-                    inputs2.insert_or_assign(
-                        h.to_string(Base16, false),
-                        justOut);
-                }
-            },
-        }, res.raw());
+        if (res.kind == DrvHash::Kind::Deferred)
+            kind = DrvHash::Kind::Deferred;
+        for (auto & outputName : inputOutputs) {
+            const auto h = res.hashes.at(outputName);
+            inputs2[h.to_string(Base16, false)].insert(outputName);
+        }
     }
 
     auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
 
-    return DrvHash { .hash = hash, .kind = kind };
-}
-
-
-void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept
-{
-    switch (other) {
-    case DrvHash::Kind::Regular:
-        break;
-    case DrvHash::Kind::Deferred:
-        self = other;
-        break;
+    std::map<std::string, Hash> outputHashes;
+    for (const auto & [outputName, _] : drv.outputs) {
+        outputHashes.insert_or_assign(outputName, hash);
     }
+
+    return DrvHash {
+        .hashes = outputHashes,
+        .kind = kind,
+    };
 }
 
 
 std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv)
 {
-    std::map<std::string, Hash> res;
-    std::visit(overloaded {
-        [&](const DrvHash & drvHash) {
-            for (auto & outputName : drv.outputNames()) {
-                res.insert({outputName, drvHash.hash});
-            }
-        },
-        [&](const CaOutputHashes & outputHashes) {
-            res = outputHashes;
-        },
-    }, hashDerivationModulo(store, drv, true).raw());
-    return res;
+    return hashDerivationModulo(store, drv, true).hashes;
 }
 
 
@@ -625,7 +709,8 @@ StringSet BasicDerivation::outputNames() const
     return names;
 }
 
-DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
+DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
+{
     DerivationOutputsAndOptPaths outsAndOptPaths;
     for (auto output : outputs)
         outsAndOptPaths.insert(std::make_pair(
@@ -636,7 +721,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s
     return outsAndOptPaths;
 }
 
-std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
+std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
+{
     auto nameWithSuffix = drvPath.name();
     constexpr std::string_view extension = ".drv";
     assert(hasSuffix(nameWithSuffix, extension));
@@ -698,6 +784,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
                     << ""
                     << "";
             },
+            [&](const DerivationOutput::Impure & doi) {
+                out << ""
+                    << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
+                    << "impure";
+            },
         }, i.second.raw());
     }
     worker_proto::write(store, out, drv.inputSrcs);
@@ -723,21 +814,19 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
 }
 
 
-static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
-
-    debug("Rewriting the derivation");
-
-    for (auto &rewrite: rewrites) {
+static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
+{
+    for (auto & rewrite : rewrites) {
         debug("rewriting %s as %s", rewrite.first, rewrite.second);
     }
 
     drv.builder = rewriteStrings(drv.builder, rewrites);
-    for (auto & arg: drv.args) {
+    for (auto & arg : drv.args) {
         arg = rewriteStrings(arg, rewrites);
     }
 
     StringPairs newEnv;
-    for (auto & envVar: drv.env) {
+    for (auto & envVar : drv.env) {
         auto envName = rewriteStrings(envVar.first, rewrites);
         auto envValue = rewriteStrings(envVar.second, rewrites);
         newEnv.emplace(envName, envValue);
@@ -747,7 +836,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
     auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
     for (auto & [outputName, output] : drv.outputs) {
         if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
-            auto & h = hashModulo.requireNoFixedNonDeferred();
+            auto & h = hashModulo.hashes.at(outputName);
             auto outPath = store.makeOutputPath(outputName, h, drv.name);
             drv.env[outputName] = store.printStorePath(outPath);
             output = DerivationOutput::InputAddressed {
@@ -758,55 +847,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
 
 }
 
-const Hash & DrvHashModulo::requireNoFixedNonDeferred() const {
-    auto * drvHashOpt = std::get_if<DrvHash>(&raw());
-    assert(drvHashOpt);
-    assert(drvHashOpt->kind == DrvHash::Kind::Regular);
-    return drvHashOpt->hash;
-}
-
-static bool tryResolveInput(
-    Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
-    const StorePath & inputDrv, const StringSet & inputOutputs)
+std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
 {
-    auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
-
-    auto getOutput = [&](const std::string & outputName) {
-        auto & actualPathOpt = inputDrvOutputs.at(outputName);
-        if (!actualPathOpt)
-            warn("output %s of input %s missing, aborting the resolving",
-                outputName,
-                store.printStorePath(inputDrv)
-            );
-        return actualPathOpt;
-    };
+    std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
 
-    for (auto & outputName : inputOutputs) {
-        auto actualPathOpt = getOutput(outputName);
-        if (!actualPathOpt) return false;
-        auto actualPath = *actualPathOpt;
-        inputRewrites.emplace(
-            downstreamPlaceholder(store, inputDrv, outputName),
-            store.printStorePath(actualPath));
-        inputSrcs.insert(std::move(actualPath));
-    }
+    for (auto & input : inputDrvs)
+        for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
+            if (outputPath)
+                inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
 
-    return true;
+    return tryResolve(store, inputDrvOutputs);
 }
 
-std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
+std::optional<BasicDerivation> Derivation::tryResolve(
+    Store & store,
+    const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
+{
     BasicDerivation resolved { *this };
 
     // Input paths that we'll want to rewrite in the derivation
     StringMap inputRewrites;
 
-    for (auto & [inputDrv, inputOutputs] : inputDrvs)
-        if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
-            return std::nullopt;
+    for (auto & [inputDrv, inputOutputs] : inputDrvs) {
+        for (auto & outputName : inputOutputs) {
+            if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
+                inputRewrites.emplace(
+                    downstreamPlaceholder(store, inputDrv, outputName),
+                    store.printStorePath(*actualPath));
+                resolved.inputSrcs.insert(*actualPath);
+            } else {
+                warn("output '%s' of input '%s' missing, aborting the resolving",
+                    outputName,
+                    store.printStorePath(inputDrv));
+                return {};
+            }
+        }
+    }
 
     rewriteDerivation(store, resolved, inputRewrites);
 
     return resolved;
 }
 
+const Hash impureOutputHash = hashString(htSHA256, "impure");
+
 }