aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/libexpr/eval.cc1
-rw-r--r--src/libexpr/eval.hh2
-rw-r--r--src/libexpr/primops.cc32
-rw-r--r--src/libstore/build/derivation-goal.cc178
-rw-r--r--src/libstore/build/derivation-goal.hh7
-rw-r--r--src/libstore/build/local-derivation-goal.cc21
-rw-r--r--src/libstore/derivations.cc235
-rw-r--r--src/libstore/derivations.hh52
-rw-r--r--src/libstore/local-store.cc3
-rw-r--r--src/libstore/path.cc9
-rw-r--r--src/libstore/path.hh2
-rw-r--r--src/libutil/experimental-features.cc1
-rw-r--r--src/libutil/experimental-features.hh1
-rw-r--r--src/nix/show-derivation.cc4
-rw-r--r--tests/impure-derivations.nix46
-rw-r--r--tests/impure-derivations.sh39
-rw-r--r--tests/local.mk3
17 files changed, 484 insertions, 152 deletions
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 437c7fc53..b87e06ef5 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -436,6 +436,7 @@ EvalState::EvalState(
, sBuilder(symbols.create("builder"))
, sArgs(symbols.create("args"))
, sContentAddressed(symbols.create("__contentAddressed"))
+ , sImpure(symbols.create("__impure"))
, sOutputHash(symbols.create("outputHash"))
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
, sOutputHashMode(symbols.create("outputHashMode"))
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index e7915dd99..7ed376e8d 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -78,7 +78,7 @@ public:
sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls,
sFile, sLine, sColumn, sFunctor, sToString,
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
- sContentAddressed,
+ sContentAddressed, sImpure,
sOutputHash, sOutputHashAlgo, sOutputHashMode,
sRecurseForDerivations,
sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath,
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 9f549e52f..eaf04320e 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -989,6 +989,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
PathSet context;
bool contentAddressed = false;
+ bool isImpure = false;
std::optional<std::string> outputHash;
std::string outputHashAlgo;
auto ingestionMethod = FileIngestionMethod::Flat;
@@ -1051,6 +1052,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
settings.requireExperimentalFeature(Xp::CaDerivations);
}
+ else if (i->name == state.sImpure) {
+ isImpure = state.forceBool(*i->value, pos);
+ if (isImpure)
+ settings.requireExperimentalFeature(Xp::ImpureDerivations);
+ }
+
/* The `args' attribute is special: it supplies the
command-line arguments to the builder. */
else if (i->name == state.sArgs) {
@@ -1197,15 +1204,28 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
});
}
- else if (contentAddressed) {
+ else if (contentAddressed || isImpure) {
+ if (contentAddressed && isImpure)
+ throw EvalError({
+ .msg = hintfmt("derivation cannot be both content-addressed and impure"),
+ .errPos = posDrvName
+ });
+
HashType ht = parseHashType(outputHashAlgo);
for (auto & i : outputs) {
drv.env[i] = hashPlaceholder(i);
- drv.outputs.insert_or_assign(i,
- DerivationOutput::CAFloating {
- .method = ingestionMethod,
- .hashType = ht,
- });
+ if (isImpure)
+ drv.outputs.insert_or_assign(i,
+ DerivationOutput::Impure {
+ .method = ingestionMethod,
+ .hashType = ht,
+ });
+ else
+ drv.outputs.insert_or_assign(i,
+ DerivationOutput::CAFloating {
+ .method = ingestionMethod,
+ .hashType = ht,
+ });
}
}
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 3d1c4fbc1..2f3490829 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -204,9 +204,34 @@ void DerivationGoal::haveDerivation()
{
trace("have derivation");
+ parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
+
if (!drv->type().hasKnownOutputPaths())
settings.requireExperimentalFeature(Xp::CaDerivations);
+ if (!drv->type().isPure()) {
+ settings.requireExperimentalFeature(Xp::ImpureDerivations);
+
+ for (auto & [outputName, output] : drv->outputs) {
+ auto randomPath = StorePath::random(outputPathName(drv->name, outputName));
+ assert(!worker.store.isValidPath(randomPath));
+ initialOutputs.insert({
+ outputName,
+ InitialOutput {
+ .wanted = true,
+ .outputHash = impureOutputHash,
+ .known = InitialOutputStatus {
+ .path = randomPath,
+ .status = PathStatus::Absent
+ }
+ }
+ });
+ }
+
+ gaveUpOnSubstitution();
+ return;
+ }
+
for (auto & i : drv->outputsAndOptPaths(worker.store))
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
@@ -230,9 +255,6 @@ void DerivationGoal::haveDerivation()
return;
}
- parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv);
-
-
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
@@ -266,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried()
{
trace("all outputs substituted (maybe)");
+ assert(drv->type().isPure());
+
if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) {
done(BuildResult::TransientFailure, {},
Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ",
@@ -315,9 +339,21 @@ void DerivationGoal::outputsSubstitutionTried()
void DerivationGoal::gaveUpOnSubstitution()
{
/* The inputs must be built before we can build this goal. */
+ inputDrvOutputs.clear();
if (useDerivation)
- for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
+ for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
+ /* Ensure that pure derivations don't depend on impure
+ derivations. */
+ if (drv->type().isPure()) {
+ auto inputDrv = worker.evalStore.readDerivation(i.first);
+ if (!inputDrv.type().isPure())
+ throw Error("pure derivation '%s' depends on impure derivation '%s'",
+ worker.store.printStorePath(drvPath),
+ worker.store.printStorePath(i.first));
+ }
+
addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal));
+ }
/* Copy the input sources from the eval store to the build
store. */
@@ -345,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution()
void DerivationGoal::repairClosure()
{
+ assert(drv->type().isPure());
+
/* If we're repairing, we now know that our own outputs are valid.
Now check whether the other paths in the outputs closure are
good. If not, then start derivation goals for the derivations
@@ -452,22 +490,24 @@ void DerivationGoal::inputsRealised()
drvs. */
: true);
},
+ [&](const DerivationType::Impure &) {
+ return true;
+ }
}, drvType.raw());
- if (resolveDrv)
- {
+ if (resolveDrv && !fullDrv.inputDrvs.empty()) {
settings.requireExperimentalFeature(Xp::CaDerivations);
/* We are be able to resolve this derivation based on the
- now-known results of dependencies. If so, we become a stub goal
- aliasing that resolved derivation goal */
- std::optional attempt = fullDrv.tryResolve(worker.store);
+ now-known results of dependencies. If so, we become a
+ stub goal aliasing that resolved derivation goal. */
+ std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs);
assert(attempt);
Derivation drvResolved { *std::move(attempt) };
auto pathResolved = writeDerivation(worker.store, drvResolved);
- auto msg = fmt("Resolved derivation: '%s' -> '%s'",
+ auto msg = fmt("resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath),
worker.store.printStorePath(pathResolved));
act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
@@ -488,21 +528,13 @@ void DerivationGoal::inputsRealised()
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
- assert(worker.evalStore.isValidPath(drvPath));
- auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath);
- for (auto & j : wantedDepOutputs) {
- if (outputs.count(j) > 0) {
- auto optRealizedInput = outputs.at(j);
- if (!optRealizedInput)
- throw Error(
- "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
- worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
- worker.store.computeFSClosure(*optRealizedInput, inputPaths);
- } else
+ for (auto & j : wantedDepOutputs)
+ if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
+ worker.store.computeFSClosure(*outPath, inputPaths);
+ else
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
- }
}
}
@@ -923,7 +955,7 @@ void DerivationGoal::buildDone()
st =
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected :
- derivationType.isImpure() || diskFull ? BuildResult::TransientFailure :
+ derivationType.needsNetworkAccess() || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure;
}
@@ -934,60 +966,52 @@ void DerivationGoal::buildDone()
void DerivationGoal::resolvedFinished()
{
+ trace("resolved derivation finished");
+
assert(resolvedDrvGoal);
auto resolvedDrv = *resolvedDrvGoal->drv;
+ auto & resolvedResult = resolvedDrvGoal->buildResult;
- auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
+ DrvOutputs builtOutputs;
- StorePathSet outputPaths;
+ if (resolvedResult.success()) {
+ auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv);
- // `wantedOutputs` might be empty, which means “all the outputs”
- auto realWantedOutputs = wantedOutputs;
- if (realWantedOutputs.empty())
- realWantedOutputs = resolvedDrv.outputNames();
+ StorePathSet outputPaths;
- DrvOutputs builtOutputs;
+ // `wantedOutputs` might be empty, which means “all the outputs”
+ auto realWantedOutputs = wantedOutputs;
+ if (realWantedOutputs.empty())
+ realWantedOutputs = resolvedDrv.outputNames();
+
+ for (auto & wantedOutput : realWantedOutputs) {
+ assert(initialOutputs.count(wantedOutput) != 0);
+ assert(resolvedHashes.count(wantedOutput) != 0);
+ auto realisation = resolvedResult.builtOutputs.at(
+ DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput });
+ if (drv->type().isPure()) {
+ auto newRealisation = realisation;
+ newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput };
+ newRealisation.signatures.clear();
+ newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath);
+ signRealisation(newRealisation);
+ worker.store.registerDrvOutput(newRealisation);
+ }
+ outputPaths.insert(realisation.outPath);
+ builtOutputs.emplace(realisation.id, realisation);
+ }
- for (auto & wantedOutput : realWantedOutputs) {
- assert(initialOutputs.count(wantedOutput) != 0);
- assert(resolvedHashes.count(wantedOutput) != 0);
- auto realisation = worker.store.queryRealisation(
- DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
+ runPostBuildHook(
+ worker.store,
+ *logger,
+ drvPath,
+ outputPaths
);
- // We've just built it, but maybe the build failed, in which case the
- // realisation won't be there
- if (realisation) {
- auto newRealisation = *realisation;
- newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
- newRealisation.signatures.clear();
- newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath);
- signRealisation(newRealisation);
- worker.store.registerDrvOutput(newRealisation);
- outputPaths.insert(realisation->outPath);
- builtOutputs.emplace(realisation->id, *realisation);
- } else {
- // If we don't have a realisation, then it must mean that something
- // failed when building the resolved drv
- assert(!buildResult.success());
- }
}
- runPostBuildHook(
- worker.store,
- *logger,
- drvPath,
- outputPaths
- );
-
- auto status = [&]() {
- auto & resolvedResult = resolvedDrvGoal->buildResult;
- switch (resolvedResult.status) {
- case BuildResult::AlreadyValid:
- return BuildResult::ResolvesToAlreadyValid;
- default:
- return resolvedResult.status;
- }
- }();
+ auto status = resolvedResult.status;
+ if (status == BuildResult::AlreadyValid)
+ status = BuildResult::ResolvesToAlreadyValid;
done(status, std::move(builtOutputs));
}
@@ -1236,6 +1260,7 @@ void DerivationGoal::flushLine()
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
{
+ assert(drv->type().isPure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
std::map<std::string, std::optional<StorePath>> res;
for (auto & [name, output] : drv->outputs)
@@ -1248,6 +1273,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
OutputPathMap DerivationGoal::queryDerivationOutputMap()
{
+ assert(drv->type().isPure());
if (!useDerivation || drv->type().hasKnownOutputPaths()) {
OutputPathMap res;
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
@@ -1261,6 +1287,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
{
+ if (!drv->type().isPure()) return { false, {} };
+
bool checkHash = buildMode == bmRepair;
auto wantedOutputsLeft = wantedOutputs;
DrvOutputs validOutputs;
@@ -1304,6 +1332,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity()
if (info.wanted && info.known && info.known->isValid())
validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path });
}
+
// If we requested all the outputs via the empty set, we are always fine.
// If we requested specific elements, the loop above removes all the valid
// ones, so any that are left must be invalid.
@@ -1343,7 +1372,6 @@ void DerivationGoal::done(
if (ex)
// FIXME: strip: "error: "
buildResult.errorMsg = ex->what();
- amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
if (buildResult.status == BuildResult::TimedOut)
worker.timedOut = true;
if (buildResult.status == BuildResult::PermanentFailure)
@@ -1370,7 +1398,21 @@ void DerivationGoal::done(
fs.open(traceBuiltOutputsFile, std::fstream::out);
fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl;
}
+
+ amDone(buildResult.success() ? ecSuccess : ecFailed, ex);
}
+void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
+{
+ Goal::waiteeDone(waitee, result);
+
+ if (waitee->buildResult.success())
+ if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path))
+ for (auto & [output, realisation] : waitee->buildResult.builtOutputs)
+ inputDrvOutputs.insert_or_assign(
+ { bfd->drvPath, output.outputName },
+ realisation.outPath);
+}
+
}
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index f556b6f25..2d8bfd592 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -57,6 +57,11 @@ struct DerivationGoal : public Goal
them. */
StringSet wantedOutputs;
+ /* Mapping from input derivations + output names to actual store
+ paths. This is filled in by waiteeDone() as each dependency
+ finishes, before inputsRealised() is reached, */
+ std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
+
/* Whether additional wanted outputs have been added. */
bool needRestart = false;
@@ -224,6 +229,8 @@ struct DerivationGoal : public Goal
DrvOutputs builtOutputs = {},
std::optional<Error> ex = {});
+ void waiteeDone(GoalPtr waitee, ExitCode result) override;
+
StorePathSet exportReferences(const StorePathSet & storePaths);
};
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index b176f318b..a6c07314f 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
- useChroot = !(derivationType.isImpure()) && !noChroot;
+ useChroot = !derivationType.needsNetworkAccess() && !noChroot;
}
auto & localStore = getLocalStore();
@@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
"nogroup:x:65534:\n", sandboxGid()));
/* Create /etc/hosts with localhost entry. */
- if (!(derivationType.isImpure()))
+ if (!derivationType.needsNetworkAccess())
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot,
@@ -796,7 +796,7 @@ void LocalDerivationGoal::startBuilder()
us.
*/
- if (!(derivationType.isImpure()))
+ if (!derivationType.needsNetworkAccess())
privateNetwork = true;
userNamespaceSync.create();
@@ -1060,7 +1060,7 @@ void LocalDerivationGoal::initEnv()
to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */
- if (derivationType.isImpure()) {
+ if (derivationType.needsNetworkAccess()) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or("");
}
@@ -1674,7 +1674,7 @@ void LocalDerivationGoal::runChild()
/* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so
on. */
- if (derivationType.isImpure()) {
+ if (derivationType.needsNetworkAccess()) {
// Only use nss functions to resolve hosts and
// services. Don’t use it for anything else that may
// be configured for this system. This limits the
@@ -2399,6 +2399,13 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
assert(false);
},
+ [&](const DerivationOutput::Impure & doi) {
+ return newInfoFromCA(DerivationOutput::CAFloating {
+ .method = doi.method,
+ .hashType = doi.hashType,
+ });
+ },
+
}, output.raw());
/* FIXME: set proper permissions in restorePath() so
@@ -2609,7 +2616,9 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
},
.outPath = newInfo.path
};
- if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
+ if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
+ && drv->type().isPure())
+ {
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);
}
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 85d75523f..75e0178bb 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -25,26 +25,42 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
[](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
return std::nullopt;
},
+ [](const DerivationOutput::Impure &) -> std::optional<StorePath> {
+ return std::nullopt;
+ },
}, raw());
}
-StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
+StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const
+{
return store.makeFixedOutputPath(
hash.method, hash.hash,
outputPathName(drvName, outputName));
}
-bool DerivationType::isCA() const {
+bool DerivationType::isCA() const
+{
/* Normally we do the full `std::visit` to make sure we have
exhaustively handled all variants, but so long as there is a
variant called `ContentAddressed`, it must be the only one for
which `isCA` is true for this to make sense!. */
- return std::holds_alternative<ContentAddressed>(raw());
+ return std::visit(overloaded {
+ [](const InputAddressed & ia) {
+ return false;
+ },
+ [](const ContentAddressed & ca) {
+ return true;
+ },
+ [](const Impure &) {
+ return true;
+ },
+ }, raw());
}
-bool DerivationType::isFixed() const {
+bool DerivationType::isFixed() const
+{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
@@ -52,10 +68,14 @@ bool DerivationType::isFixed() const {
[](const ContentAddressed & ca) {
return ca.fixed;
},
+ [](const Impure &) {
+ return false;
+ },
}, raw());
}
-bool DerivationType::hasKnownOutputPaths() const {
+bool DerivationType::hasKnownOutputPaths() const
+{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return !ia.deferred;
@@ -63,11 +83,15 @@ bool DerivationType::hasKnownOutputPaths() const {
[](const ContentAddressed & ca) {
return ca.fixed;
},
+ [](const Impure &) {
+ return false;
+ },
}, raw());
}
-bool DerivationType::isImpure() const {
+bool DerivationType::needsNetworkAccess() const
+{
return std::visit(overloaded {
[](const InputAddressed & ia) {
return false;
@@ -75,6 +99,25 @@ bool DerivationType::isImpure() const {
[](const ContentAddressed & ca) {
return !ca.pure;
},
+ [](const Impure &) {
+ return true;
+ },
+ }, raw());
+}
+
+
+bool DerivationType::isPure() const
+{
+ return std::visit(overloaded {
+ [](const InputAddressed & ia) {
+ return true;
+ },
+ [](const ContentAddressed & ca) {
+ return true;
+ },
+ [](const Impure &) {
+ return false;
+ },
}, raw());
}
@@ -176,7 +219,16 @@ static DerivationOutput parseDerivationOutput(const Store & store,
hashAlgo = hashAlgo.substr(2);
}
const auto hashType = parseHashType(hashAlgo);
- if (hash != "") {
+ if (hash == "impure") {
+ settings.requireExperimentalFeature(Xp::ImpureDerivations);
+ assert(pathS == "");
+ return DerivationOutput {
+ .output = DerivationOutputImpure {
+ .method = std::move(method),
+ .hashType = std::move(hashType),
+ },
+ };
+ } else if (hash != "") {
validatePath(pathS);
return DerivationOutput::CAFixed {
.hash = FixedOutputHash {
@@ -345,6 +397,12 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
+ },
+ [&](const DerivationOutputImpure & doi) {
+ // FIXME
+ s += ','; printUnquotedString(s, "");
+ s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
+ s += ','; printUnquotedString(s, "impure");
}
}, i.second.raw());
s += ')';
@@ -410,8 +468,14 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
DerivationType BasicDerivation::type() const
{
- std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
+ std::set<std::string_view>
+ inputAddressedOutputs,
+ fixedCAOutputs,
+ floatingCAOutputs,
+ deferredIAOutputs,
+ impureOutputs;
std::optional<HashType> floatingHashType;
+
for (auto & i : outputs) {
std::visit(overloaded {
[&](const DerivationOutput::InputAddressed &) {
@@ -426,43 +490,78 @@ DerivationType BasicDerivation::type() const
floatingHashType = dof.hashType;
} else {
if (*floatingHashType != dof.hashType)
- throw Error("All floating outputs must use the same hash type");
+ throw Error("all floating outputs must use the same hash type");
}
},
[&](const DerivationOutput::Deferred &) {
- deferredIAOutputs.insert(i.first);
+ deferredIAOutputs.insert(i.first);
+ },
+ [&](const DerivationOutput::Impure &) {
+ impureOutputs.insert(i.first);
},
}, i.second.raw());
}
- if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
- throw Error("Must have at least one output");
- } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
+ if (inputAddressedOutputs.empty()
+ && fixedCAOutputs.empty()
+ && floatingCAOutputs.empty()
+ && deferredIAOutputs.empty()
+ && impureOutputs.empty())
+ throw Error("must have at least one output");
+
+ if (!inputAddressedOutputs.empty()
+ && fixedCAOutputs.empty()
+ && floatingCAOutputs.empty()
+ && deferredIAOutputs.empty()
+ && impureOutputs.empty())
return DerivationType::InputAddressed {
.deferred = false,
};
- } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
+
+ if (inputAddressedOutputs.empty()
+ && !fixedCAOutputs.empty()
+ && floatingCAOutputs.empty()
+ && deferredIAOutputs.empty()
+ && impureOutputs.empty())
+ {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
- throw Error("Only one fixed output is allowed for now");
+ throw Error("only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
- throw Error("Single fixed output must be named \"out\"");
+ throw Error("single fixed output must be named \"out\"");
return DerivationType::ContentAddressed {
.pure = false,
.fixed = true,
};
- } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
+ }
+
+ if (inputAddressedOutputs.empty()
+ && fixedCAOutputs.empty()
+ && !floatingCAOutputs.empty()
+ && deferredIAOutputs.empty()
+ && impureOutputs.empty())
return DerivationType::ContentAddressed {
.pure = true,
.fixed = false,
};
- } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
+
+ if (inputAddressedOutputs.empty()
+ && fixedCAOutputs.empty()
+ && floatingCAOutputs.empty()
+ && !deferredIAOutputs.empty()
+ && impureOutputs.empty())
return DerivationType::InputAddressed {
.deferred = true,
};
- } else {
- throw Error("Can't mix derivation output types");
- }
+
+ if (inputAddressedOutputs.empty()
+ && fixedCAOutputs.empty()
+ && floatingCAOutputs.empty()
+ && deferredIAOutputs.empty()
+ && !impureOutputs.empty())
+ return DerivationType::Impure { };
+
+ throw Error("can't mix derivation output types");
}
@@ -524,12 +623,22 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
+ store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
- return DrvHash{
+ return DrvHash {
.hashes = outputHashes,
.kind = DrvHash::Kind::Regular,
};
}
+ if (!type.isPure()) {
+ std::map<std::string, Hash> outputHashes;
+ for (const auto & [outputName, _] : drv.outputs)
+ outputHashes.insert_or_assign(outputName, impureOutputHash);
+ return DrvHash {
+ .hashes = outputHashes,
+ .kind = DrvHash::Kind::Deferred,
+ };
+ }
+
auto kind = std::visit(overloaded {
[](const DerivationType::InputAddressed & ia) {
/* This might be a "pesimistically" deferred output, so we don't
@@ -541,6 +650,9 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
? DrvHash::Kind::Regular
: DrvHash::Kind::Deferred;
},
+ [](const DerivationType::Impure &) -> DrvHash::Kind {
+ assert(false);
+ }
}, drv.type().raw());
std::map<std::string, StringSet> inputs2;
@@ -599,7 +711,8 @@ StringSet BasicDerivation::outputNames() const
return names;
}
-DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const {
+DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const
+{
DerivationOutputsAndOptPaths outsAndOptPaths;
for (auto output : outputs)
outsAndOptPaths.insert(std::make_pair(
@@ -610,7 +723,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s
return outsAndOptPaths;
}
-std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
+std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath)
+{
auto nameWithSuffix = drvPath.name();
constexpr std::string_view extension = ".drv";
assert(hasSuffix(nameWithSuffix, extension));
@@ -672,6 +786,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
<< ""
<< "";
},
+ [&](const DerivationOutput::Impure & doi) {
+ out << ""
+ << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType))
+ << "impure";
+ },
}, i.second.raw());
}
worker_proto::write(store, out, drv.inputSrcs);
@@ -697,21 +816,19 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
}
-static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
-
- debug("Rewriting the derivation");
-
- for (auto &rewrite: rewrites) {
+static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites)
+{
+ for (auto & rewrite : rewrites) {
debug("rewriting %s as %s", rewrite.first, rewrite.second);
}
drv.builder = rewriteStrings(drv.builder, rewrites);
- for (auto & arg: drv.args) {
+ for (auto & arg : drv.args) {
arg = rewriteStrings(arg, rewrites);
}
StringPairs newEnv;
- for (auto & envVar: drv.env) {
+ for (auto & envVar : drv.env) {
auto envName = rewriteStrings(envVar.first, rewrites);
auto envValue = rewriteStrings(envVar.second, rewrites);
newEnv.emplace(envName, envValue);
@@ -732,48 +849,48 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
}
-static bool tryResolveInput(
- Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
- const StorePath & inputDrv, const StringSet & inputOutputs)
+std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const
{
- auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
-
- auto getOutput = [&](const std::string & outputName) {
- auto & actualPathOpt = inputDrvOutputs.at(outputName);
- if (!actualPathOpt)
- warn("output %s of input %s missing, aborting the resolving",
- outputName,
- store.printStorePath(inputDrv)
- );
- return actualPathOpt;
- };
+ std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs;
- for (auto & outputName : inputOutputs) {
- auto actualPathOpt = getOutput(outputName);
- if (!actualPathOpt) return false;
- auto actualPath = *actualPathOpt;
- inputRewrites.emplace(
- downstreamPlaceholder(store, inputDrv, outputName),
- store.printStorePath(actualPath));
- inputSrcs.insert(std::move(actualPath));
- }
+ for (auto & input : inputDrvs)
+ for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first))
+ if (outputPath)
+ inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath);
- return true;
+ return tryResolve(store, inputDrvOutputs);
}
-std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
+std::optional<BasicDerivation> Derivation::tryResolve(
+ Store & store,
+ const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const
+{
BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation
StringMap inputRewrites;
- for (auto & [inputDrv, inputOutputs] : inputDrvs)
- if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
- return std::nullopt;
+ for (auto & [inputDrv, inputOutputs] : inputDrvs) {
+ for (auto & outputName : inputOutputs) {
+ if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
+ inputRewrites.emplace(
+ downstreamPlaceholder(store, inputDrv, outputName),
+ store.printStorePath(*actualPath));
+ resolved.inputSrcs.insert(*actualPath);
+ } else {
+ warn("output '%s' of input '%s' missing, aborting the resolving",
+ outputName,
+ store.printStorePath(inputDrv));
+ return {};
+ }
+ }
+ }
rewriteDerivation(store, resolved, inputRewrites);
return resolved;
}
+const Hash impureOutputHash = hashString(htSHA256, "impure");
+
}
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 63ea5ef76..b62e40786 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -41,15 +41,26 @@ struct DerivationOutputCAFloating
};
/* Input-addressed output which depends on a (CA) derivation whose hash isn't
- * known atm
+ * known yet.
*/
struct DerivationOutputDeferred {};
+/* Impure output which is moved to a content-addressed location (like
+ CAFloating) but isn't registered as a realization.
+ */
+struct DerivationOutputImpure
+{
+ /* information used for expected hash computation */
+ FileIngestionMethod method;
+ HashType hashType;
+};
+
typedef std::variant<
DerivationOutputInputAddressed,
DerivationOutputCAFixed,
DerivationOutputCAFloating,
- DerivationOutputDeferred
+ DerivationOutputDeferred,
+ DerivationOutputImpure
> _DerivationOutputRaw;
struct DerivationOutput : _DerivationOutputRaw
@@ -61,6 +72,7 @@ struct DerivationOutput : _DerivationOutputRaw
using CAFixed = DerivationOutputCAFixed;
using CAFloating = DerivationOutputCAFloating;
using Deferred = DerivationOutputDeferred;
+ using Impure = DerivationOutputImpure;
/* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer
@@ -94,9 +106,13 @@ struct DerivationType_ContentAddressed {
bool fixed;
};
+struct DerivationType_Impure {
+};
+
typedef std::variant<
DerivationType_InputAddressed,
- DerivationType_ContentAddressed
+ DerivationType_ContentAddressed,
+ DerivationType_Impure
> _DerivationTypeRaw;
struct DerivationType : _DerivationTypeRaw {
@@ -104,7 +120,7 @@ struct DerivationType : _DerivationTypeRaw {
using Raw::Raw;
using InputAddressed = DerivationType_InputAddressed;
using ContentAddressed = DerivationType_ContentAddressed;
-
+ using Impure = DerivationType_Impure;
/* Do the outputs of the derivation have paths calculated from their content,
or from the derivation itself? */
@@ -114,10 +130,13 @@ struct DerivationType : _DerivationTypeRaw {
non-CA derivations. */
bool isFixed() const;
- /* Is the derivation impure and needs to access non-deterministic resources, or
- pure and can be sandboxed? Note that whether or not we actually sandbox the
- derivation is controlled separately. Never true for non-CA derivations. */
- bool isImpure() const;
+ /* Whether the derivation needs to access the network. Note that
+ whether or not we actually sandbox the derivation is controlled
+ separately. Never true for non-CA derivations. */
+ bool needsNetworkAccess() const;
+
+ /* FIXME */
+ bool isPure() const;
/* Does the derivation knows its own output paths?
Only true when there's no floating-ca derivation involved in the
@@ -173,7 +192,14 @@ struct Derivation : BasicDerivation
added directly to input sources.
2. Input placeholders are replaced with realized input store paths. */
- std::optional<BasicDerivation> tryResolve(Store & store);
+ std::optional<BasicDerivation> tryResolve(Store & store) const;
+
+ /* Like the above, but instead of querying the Nix database for
+ realisations, uses a given mapping from input derivation paths
+ + output names to actual output store paths. */
+ std::optional<BasicDerivation> tryResolve(
+ Store & store,
+ const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const;
Derivation() = default;
Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { }
@@ -211,7 +237,7 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
struct DrvHash {
std::map<std::string, Hash> hashes;
- enum struct Kind: bool {
+ enum struct Kind : bool {
// Statically determined derivations.
// This hash will be directly used to compute the output paths
Regular,
@@ -252,8 +278,10 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
/*
Return a map associating each output to a hash that uniquely identifies its
derivation (modulo the self-references).
+
+ FIXME: what is the Hash in this map?
*/
-std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
+std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv);
/* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, DrvHash> DrvHashes;
@@ -286,4 +314,6 @@ std::string hashPlaceholder(const std::string_view outputName);
dependency which is a CA derivation. */
std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName);
+extern const Hash impureOutputHash;
+
}
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 60fe53af1..d77fff963 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -719,6 +719,9 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
[&](const DerivationOutput::Deferred &) {
/* Nothing to check */
},
+ [&](const DerivationOutput::Impure &) {
+ /* Nothing to check */
+ },
}, i.second.raw());
}
}
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index e642abcd5..392db225e 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -1,5 +1,7 @@
#include "store-api.hh"
+#include <sodium.h>
+
namespace nix {
static void checkName(std::string_view path, std::string_view name)
@@ -41,6 +43,13 @@ bool StorePath::isDerivation() const
StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x");
+StorePath StorePath::random(std::string_view name)
+{
+ Hash hash(htSHA1);
+ randombytes_buf(hash.hash, hash.hashSize);
+ return StorePath(hash, name);
+}
+
StorePath Store::parseStorePath(std::string_view path) const
{
auto p = canonPath(std::string(path));
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index e65fee622..77fd0f8dc 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -58,6 +58,8 @@ public:
}
static StorePath dummy;
+
+ static StorePath random(std::string_view name);
};
typedef std::set<StorePath> StorePathSet;
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index 01f318fa3..e033a4116 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -7,6 +7,7 @@ namespace nix {
std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::CaDerivations, "ca-derivations" },
+ { Xp::ImpureDerivations, "impure-derivations" },
{ Xp::Flakes, "flakes" },
{ Xp::NixCommand, "nix-command" },
{ Xp::RecursiveNix, "recursive-nix" },
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index b5140dcfe..3a254b423 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -16,6 +16,7 @@ namespace nix {
enum struct ExperimentalFeature
{
CaDerivations,
+ ImpureDerivations,
Flakes,
NixCommand,
RecursiveNix,
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 0d9655732..fb46b4dbf 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -77,6 +77,10 @@ struct CmdShowDerivation : InstallablesCommand
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
},
[&](const DerivationOutput::Deferred &) {},
+ [&](const DerivationOutput::Impure & doi) {
+ outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType));
+ outputObj.attr("impure", true);
+ },
}, output.raw());
}
}
diff --git a/tests/impure-derivations.nix b/tests/impure-derivations.nix
new file mode 100644
index 000000000..ba7d53146
--- /dev/null
+++ b/tests/impure-derivations.nix
@@ -0,0 +1,46 @@
+with import ./config.nix;
+
+rec {
+
+ impure = mkDerivation {
+ name = "impure";
+ outputs = [ "out" "stuff" ];
+ buildCommand =
+ ''
+ x=$(< $TEST_ROOT/counter)
+ mkdir $out $stuff
+ echo $x > $out/n
+ ln -s $out/n $stuff/bla
+ printf $((x + 1)) > $TEST_ROOT/counter
+ '';
+ __impure = true;
+ outputHashAlgo = "sha256";
+ outputHashMode = "recursive";
+ impureEnvVars = [ "TEST_ROOT" ];
+ };
+
+ impureOnImpure = mkDerivation {
+ name = "impure-on-impure";
+ buildCommand =
+ ''
+ x=$(< ${impure}/n)
+ mkdir $out
+ printf X$x > $out/n
+ ln -s ${impure.stuff} $out/symlink
+ ln -s $out $out/self
+ '';
+ __impure = true;
+ outputHashAlgo = "sha256";
+ outputHashMode = "recursive";
+ };
+
+ # This is not allowed.
+ inputAddressed = mkDerivation {
+ name = "input-addressed";
+ buildCommand =
+ ''
+ cat ${impure} > $out
+ '';
+ };
+
+}
diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh
new file mode 100644
index 000000000..cb1eaf84a
--- /dev/null
+++ b/tests/impure-derivations.sh
@@ -0,0 +1,39 @@
+source common.sh
+
+requireDaemonNewerThan "2.8pre20220311"
+
+enableFeatures "ca-derivations ca-references impure-derivations"
+
+clearStore
+
+# Basic test of impure derivations: building one a second time should not use the previous result.
+printf 0 > $TEST_ROOT/counter
+
+json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure)
+path1=$(echo $json | jq -r .[].outputs.out)
+path1_stuff=$(echo $json | jq -r .[].outputs.stuff)
+[[ $(< $path1/n) = 0 ]]
+[[ $(< $path1_stuff/bla) = 0 ]]
+
+[[ $(nix path-info --json $path1 | jq .[].ca) =~ fixed:r:sha256: ]]
+
+path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out)
+[[ $(< $path2/n) = 1 ]]
+
+# Test impure derivations that depend on impure derivations.
+path3=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure -vvvvv | jq -r .[].outputs.out)
+[[ $(< $path3/n) = X2 ]]
+
+path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure -vvvvv | jq -r .[].outputs.out)
+[[ $(< $path4/n) = X3 ]]
+
+# Test that (self-)references work.
+[[ $(< $path4/symlink/bla) = 3 ]]
+[[ $(< $path4/self/n) = X3 ]]
+
+# Input-addressed derivations cannot depend on impure derivations directly.
+nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1 | grep 'depends on impure derivation'
+
+drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .)
+[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]]
+[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]]
diff --git a/tests/local.mk b/tests/local.mk
index 97971dd76..668b34500 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -97,7 +97,8 @@ nix_tests = \
nix-profile.sh \
suggestions.sh \
store-ping.sh \
- fetchClosure.sh
+ fetchClosure.sh \
+ impure-derivations.sh
ifeq ($(HAVE_LIBCPUID), 1)
nix_tests += compute-levels.sh
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 02:06:48 +0000