aboutsummaryrefslogtreecommitdiff
path: root/src/libstore
diff options
context:
space:
mode:
authorJohn Ericson <John.Ericson@Obsidian.Systems>2022-03-25 19:40:52 +0000
committerJohn Ericson <John.Ericson@Obsidian.Systems>2022-03-25 19:40:52 +0000
commitff2a8ccfe176fa3e075ed8925a371eeb17e627e6 (patch)
tree7c20ce8a8ae5370f6fc078fa26888c7417f47c7a /src/libstore
parent938650700fafe76e3755982d670855fed3db35c6 (diff)
parent0dc2974930df57cac6673c02e9bc6eb6fd16ba48 (diff)
Merge branch 'path-info' into ca-drv-exotic
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/binary-cache-store.hh5
-rw-r--r--src/libstore/build/derivation-goal.cc45
-rw-r--r--src/libstore/build/derivation-goal.hh8
-rw-r--r--src/libstore/build/goal.cc2
-rw-r--r--src/libstore/build/goal.hh13
-rw-r--r--src/libstore/build/local-derivation-goal.cc35
-rw-r--r--src/libstore/builtins/buildenv.cc10
-rw-r--r--src/libstore/daemon.cc19
-rw-r--r--src/libstore/derivations.cc299
-rw-r--r--src/libstore/derivations.hh131
-rw-r--r--src/libstore/derived-path.cc32
-rw-r--r--src/libstore/derived-path.hh2
-rw-r--r--src/libstore/gc-store.cc13
-rw-r--r--src/libstore/gc-store.hh4
-rw-r--r--src/libstore/gc.cc6
-rw-r--r--src/libstore/local-fs-store.hh6
-rw-r--r--src/libstore/local-store.cc15
-rw-r--r--src/libstore/log-store.hh21
-rw-r--r--src/libstore/make-content-addressed.cc83
-rw-r--r--src/libstore/make-content-addressed.hh12
-rw-r--r--src/libstore/misc.cc2
-rw-r--r--src/libstore/parsed-derivations.cc2
-rw-r--r--src/libstore/parsed-derivations.hh1
-rw-r--r--src/libstore/remote-store.hh6
-rw-r--r--src/libstore/repair-flag.hh7
-rw-r--r--src/libstore/ssh-store.cc4
-rw-r--r--src/libstore/store-api.cc1
-rw-r--r--src/libstore/store-api.hh12
-rw-r--r--src/libstore/store-cast.hh16
29 files changed, 547 insertions, 265 deletions
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 9603a8caa..ca538b3cb 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -2,6 +2,7 @@
#include "crypto.hh"
#include "store-api.hh"
+#include "log-store.hh"
#include "pool.hh"
@@ -28,7 +29,9 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
"other than -1 which we reserve to indicate Nix defaults should be used"};
};
-class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store
+class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
+ public virtual Store,
+ public virtual LogStore
{
private:
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index afed9bf16..3d1c4fbc1 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -204,11 +204,9 @@ void DerivationGoal::haveDerivation()
{
trace("have derivation");
- if (drv->type() == DerivationType::CAFloating)
+ if (!drv->type().hasKnownOutputPaths())
settings.requireExperimentalFeature(Xp::CaDerivations);
- retrySubstitution = false;
-
for (auto & i : drv->outputsAndOptPaths(worker.store))
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
@@ -311,14 +309,11 @@ void DerivationGoal::outputsSubstitutionTried()
gaveUpOnSubstitution();
}
+
/* At least one of the output paths could not be
produced using a substitute. So we have to build instead. */
void DerivationGoal::gaveUpOnSubstitution()
{
- /* Make sure checkPathValidity() from now on checks all
- outputs. */
- wantedOutputs.clear();
-
/* The inputs must be built before we can build this goal. */
if (useDerivation)
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
@@ -426,7 +421,8 @@ void DerivationGoal::inputsRealised()
return;
}
- if (retrySubstitution) {
+ if (retrySubstitution && !retriedSubstitution) {
+ retriedSubstitution = true;
haveDerivation();
return;
}
@@ -440,9 +436,28 @@ void DerivationGoal::inputsRealised()
if (useDerivation) {
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
- if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) &&
- ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
- || fullDrv.type() == DerivationType::DeferredInputAddressed)) {
+ auto drvType = fullDrv.type();
+ bool resolveDrv = std::visit(overloaded {
+ [&](const DerivationType::InputAddressed & ia) {
+ /* must resolve if deferred. */
+ return ia.deferred;
+ },
+ [&](const DerivationType::ContentAddressed & ca) {
+ return !fullDrv.inputDrvs.empty() && (
+ ca.fixed
+ /* Can optionally resolve if fixed, which is good
+ for avoiding unnecessary rebuilds. */
+ ? settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
+ /* Must resolve if floating and there are any inputs
+ drvs. */
+ : true);
+ },
+ }, drvType.raw());
+
+ if (resolveDrv)
+ {
+ settings.requireExperimentalFeature(Xp::CaDerivations);
+
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal
aliasing that resolved derivation goal */
@@ -501,7 +516,7 @@ void DerivationGoal::inputsRealised()
/* Don't repeat fixed-output derivations since they're already
verified by their output hash.*/
- nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
+ nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1;
/* Okay, try to build. Note that here we don't wait for a build
slot to become available, since we don't need one if there is a
@@ -908,7 +923,7 @@ void DerivationGoal::buildDone()
st =
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected :
- derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
+ derivationType.isImpure() || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure;
}
@@ -1221,7 +1236,7 @@ void DerivationGoal::flushLine()
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
{
- if (!useDerivation || drv->type() != DerivationType::CAFloating) {
+ if (!useDerivation || drv->type().hasKnownOutputPaths()) {
std::map<std::string, std::optional<StorePath>> res;
for (auto & [name, output] : drv->outputs)
res.insert_or_assign(name, output.path(worker.store, drv->name, name));
@@ -1233,7 +1248,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
OutputPathMap DerivationGoal::queryDerivationOutputMap()
{
- if (!useDerivation || drv->type() != DerivationType::CAFloating) {
+ if (!useDerivation || drv->type().hasKnownOutputPaths()) {
OutputPathMap res;
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
res.insert_or_assign(name, *output.second);
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index ea2db89b2..f556b6f25 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -61,8 +61,12 @@ struct DerivationGoal : public Goal
bool needRestart = false;
/* Whether to retry substituting the outputs after building the
- inputs. */
- bool retrySubstitution;
+ inputs. This is done in case of an incomplete closure. */
+ bool retrySubstitution = false;
+
+ /* Whether we've retried substitution, in which case we won't try
+ again. */
+ bool retriedSubstitution = false;
/* The derivation stored at drvPath. */
std::unique_ptr<Derivation> drv;
diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
index d2420b107..58e805f55 100644
--- a/src/libstore/build/goal.cc
+++ b/src/libstore/build/goal.cc
@@ -28,7 +28,7 @@ void Goal::addWaitee(GoalPtr waitee)
void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
{
- assert(waitees.find(waitee) != waitees.end());
+ assert(waitees.count(waitee));
waitees.erase(waitee);
trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index 07c752bb9..35121c5d9 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -40,21 +40,21 @@ struct Goal : public std::enable_shared_from_this<Goal>
WeakGoals waiters;
/* Number of goals we are/were waiting for that have failed. */
- unsigned int nrFailed;
+ size_t nrFailed = 0;
/* Number of substitution goals we are/were waiting for that
failed because there are no substituters. */
- unsigned int nrNoSubstituters;
+ size_t nrNoSubstituters = 0;
/* Number of substitution goals we are/were waiting for that
failed because they had unsubstitutable references. */
- unsigned int nrIncompleteClosure;
+ size_t nrIncompleteClosure = 0;
/* Name of this goal for debugging purposes. */
std::string name;
/* Whether the goal is finished. */
- ExitCode exitCode;
+ ExitCode exitCode = ecBusy;
/* Build result. */
BuildResult buildResult;
@@ -65,10 +65,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
Goal(Worker & worker, DerivedPath path)
: worker(worker)
, buildResult { .path = std::move(path) }
- {
- nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
- exitCode = ecBusy;
- }
+ { }
virtual ~Goal()
{
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 14b1582f1..412c3df29 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
- useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
+ useChroot = !(derivationType.isImpure()) && !noChroot;
}
auto & localStore = getLocalStore();
@@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
"nogroup:x:65534:\n", sandboxGid()));
/* Create /etc/hosts with localhost entry. */
- if (!(derivationIsImpure(derivationType)))
+ if (!(derivationType.isImpure()))
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot,
@@ -796,7 +796,7 @@ void LocalDerivationGoal::startBuilder()
us.
*/
- if (!(derivationIsImpure(derivationType)))
+ if (!(derivationType.isImpure()))
privateNetwork = true;
userNamespaceSync.create();
@@ -1049,7 +1049,7 @@ void LocalDerivationGoal::initEnv()
derivation, tell the builder, so that for instance `fetchurl'
can skip checking the output. On older Nixes, this environment
variable won't be set, so `fetchurl' will do the check. */
- if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
+ if (derivationType.isFixed()) env["NIX_OUTPUT_CHECKED"] = "1";
/* *Only* if this is a fixed-output derivation, propagate the
values of the environment variables specified in the
@@ -1060,7 +1060,7 @@ void LocalDerivationGoal::initEnv()
to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */
- if (derivationIsImpure(derivationType)) {
+ if (derivationType.isImpure()) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or("");
}
@@ -1340,6 +1340,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
next->queryMissing(allowed, willBuild, willSubstitute,
unknown, downloadSize, narSize);
}
+
+ virtual std::optional<std::string> getBuildLog(const StorePath & path) override
+ { return std::nullopt; }
+
+ virtual void addBuildLog(const StorePath & path, std::string_view log) override
+ { unsupported("addBuildLog"); }
};
@@ -1668,7 +1674,7 @@ void LocalDerivationGoal::runChild()
/* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so
on. */
- if (derivationIsImpure(derivationType)) {
+ if (derivationType.isImpure()) {
// Only use nss functions to resolve hosts and
// services. Don’t use it for anything else that may
// be configured for this system. This limits the
@@ -1912,7 +1918,7 @@ void LocalDerivationGoal::runChild()
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
- if (derivationIsImpure(derivationType))
+ if (derivationType.isImpure())
sandboxProfile += "(import \"sandbox-network.sb\")\n";
/* Add the output paths we'll use at build-time to the chroot */
@@ -2272,7 +2278,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
return res;
};
- auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
+ auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo {
auto & st = outputStats.at(outputName);
if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
outputHash.method == ContentAddressMethod { TextHashMethod {} })
@@ -2340,7 +2346,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
ValidPathInfo newInfo = std::visit(overloaded {
- [&](const DerivationOutputInputAddressed & output) {
+ [&](const DerivationOutput::InputAddressed & output) {
/* input-addressed case */
auto requiredFinalPath = output.path;
/* Preemptively add rewrite rule for final hash, as that is
@@ -2357,7 +2363,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
return newInfo0;
},
- [&](const DerivationOutputCAFixed & dof) {
+ [&](const DerivationOutput::CAFixed & dof) {
auto wanted = getContentAddressHash(dof.ca);
auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
@@ -2386,17 +2392,17 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
return newInfo0;
},
- [&](DerivationOutputCAFloating & dof) {
+ [&](const DerivationOutput::CAFloating & dof) {
return newInfoFromCA(dof);
},
- [&](DerivationOutputDeferred) -> ValidPathInfo {
+ [&](const DerivationOutput::Deferred &) -> ValidPathInfo {
// No derivation should reach that point without having been
// rewritten first
assert(false);
},
- }, output.output);
+ }, output.raw());
/* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */
@@ -2610,7 +2616,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);
}
- builtOutputs.emplace(thisRealisation.id, thisRealisation);
+ if (wantOutput(outputName, wantedOutputs))
+ builtOutputs.emplace(thisRealisation.id, thisRealisation);
}
return builtOutputs;
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 25d015cb9..6f6ad57cb 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -47,9 +47,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
throw;
}
- /* The files below are special-cased to that they don't show up
- * in user profiles, either because they are useless, or
- * because they would cauase pointless collisions (e.g., each
+ /* The files below are special-cased to that they don't show
+ * up in user profiles, either because they are useless, or
+ * because they would cause pointless collisions (e.g., each
* Python package brings its own
* `$out/lib/pythonX.Y/site-packages/easy-install.pth'.)
*/
@@ -57,7 +57,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
hasSuffix(srcFile, "/nix-support") ||
hasSuffix(srcFile, "/perllocal.pod") ||
hasSuffix(srcFile, "/info/dir") ||
- hasSuffix(srcFile, "/log"))
+ hasSuffix(srcFile, "/log") ||
+ hasSuffix(srcFile, "/manifest.nix") ||
+ hasSuffix(srcFile, "/manifest.json"))
continue;
else if (S_ISDIR(srcSt.st_mode)) {
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 4b5a57ff1..8271bc8d6 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -3,7 +3,9 @@
#include "worker-protocol.hh"
#include "build-result.hh"
#include "store-api.hh"
+#include "store-cast.hh"
#include "gc-store.hh"
+#include "log-store.hh"
#include "path-with-outputs.hh"
#include "finally.hh"
#include "archive.hh"
@@ -562,6 +564,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
BuildMode buildMode = (BuildMode) readInt(from);
logger->startWork();
+ auto drvType = drv.type();
+
/* Content-addressed derivations are trustless because their output paths
are verified by their content alone, so any derivation is free to
try to produce such a path.
@@ -594,12 +598,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
derivations, we throw out the precomputed output paths and just
store the hashes, so there aren't two competing sources of truth an
attacker could exploit. */
- if (drv.type() == DerivationType::InputAddressed && !trusted)
+ if (!(drvType.isCA() || trusted))
throw Error("you are not privileged to build input-addressed derivations");
/* Make sure that the non-input-addressed derivations that got this far
are in fact content-addressed if we don't trust them. */
- assert(derivationIsCA(drv.type()) || trusted);
+ assert(drvType.isCA() || trusted);
/* Recompute the derivation path when we cannot trust the original. */
if (!trusted) {
@@ -608,7 +612,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
original not-necessarily-resolved derivation to verify the drv
derivation as adequate claim to the input-addressed output
paths. */
- assert(derivationIsCA(drv.type()));
+ assert(drvType.isCA());
Derivation drv2;
static_cast<BasicDerivation &>(drv2) = drv;
@@ -649,7 +653,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
Path path = absPath(readString(from));
logger->startWork();
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
gcStore.addIndirectRoot(path);
logger->stopWork();
@@ -667,7 +671,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopFindRoots: {
logger->startWork();
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
Roots roots = gcStore.findRoots(!trusted);
logger->stopWork();
@@ -699,7 +703,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
if (options.ignoreLiveness)
throw Error("you are not allowed to ignore liveness");
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
gcStore.collectGarbage(options, results);
logger->stopWork();
@@ -957,11 +961,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
if (!trusted)
throw Error("you are not privileged to add logs");
+ auto & logStore = require<LogStore>(*store);
{
FramedSource source(from);
StringSink sink;
source.drainInto(sink);
- store->addBuildLog(path, sink.s);
+ logStore.addBuildLog(path, sink.s);
}
logger->stopWork();
to << 1;
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index f6f41361b..00757abc8 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -12,25 +12,25 @@ namespace nix {
std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{
return std::visit(overloaded {
- [](const DerivationOutputInputAddressed & doi) -> std::optional<StorePath> {
+ [](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
return { doi.path };
},
- [&](const DerivationOutputCAFixed & dof) -> std::optional<StorePath> {
+ [&](const DerivationOutput::CAFixed & dof) -> std::optional<StorePath> {
return {
dof.path(store, drvName, outputName)
};
},
- [](const DerivationOutputCAFloating & dof) -> std::optional<StorePath> {
+ [](const DerivationOutput::CAFloating & dof) -> std::optional<StorePath> {
return std::nullopt;
},
- [](const DerivationOutputDeferred &) -> std::optional<StorePath> {
+ [](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
return std::nullopt;
},
- }, output);
+ }, raw());
}
-StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
+StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPathFromCA(StorePathDescriptor {
.name = outputPathName(drvName, outputName),
.info = ca,
@@ -38,47 +38,46 @@ StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view dr
}
-bool derivationIsCA(DerivationType dt) {
- switch (dt) {
- case DerivationType::InputAddressed: return false;
- case DerivationType::CAFixed: return true;
- case DerivationType::CAFloating: return true;
- case DerivationType::DeferredInputAddressed: return false;
- };
- // Since enums can have non-variant values, but making a `default:` would
- // disable exhaustiveness warnings.
- assert(false);
+bool DerivationType::isCA() const {
+ /* Normally we do the full `std::visit` to make sure we have
+ exhaustively handled all variants, but so long as there is a
+ variant called `ContentAddressed`, it must be the only one for
+ which `isCA` is true for this to make sense!. */
+ return std::holds_alternative<ContentAddressed>(raw());
}
-bool derivationIsFixed(DerivationType dt) {
- switch (dt) {
- case DerivationType::InputAddressed: return false;
- case DerivationType::CAFixed: return true;
- case DerivationType::CAFloating: return false;
- case DerivationType::DeferredInputAddressed: return false;
- };
- assert(false);
+bool DerivationType::isFixed() const {
+ return std::visit(overloaded {
+ [](const InputAddressed & ia) {
+ return false;
+ },
+ [](const ContentAddressed & ca) {
+ return ca.fixed;
+ },
+ }, raw());
}
-bool derivationHasKnownOutputPaths(DerivationType dt) {
- switch (dt) {
- case DerivationType::InputAddressed: return true;
- case DerivationType::CAFixed: return true;
- case DerivationType::CAFloating: return false;
- case DerivationType::DeferredInputAddressed: return false;
- };
- assert(false);
+bool DerivationType::hasKnownOutputPaths() const {
+ return std::visit(overloaded {
+ [](const InputAddressed & ia) {
+ return !ia.deferred;
+ },
+ [](const ContentAddressed & ca) {
+ return ca.fixed;
+ },
+ }, raw());
}
-bool derivationIsImpure(DerivationType dt) {
- switch (dt) {
- case DerivationType::InputAddressed: return false;
- case DerivationType::CAFixed: return true;
- case DerivationType::CAFloating: return false;
- case DerivationType::DeferredInputAddressed: return false;
- };
- assert(false);
+bool DerivationType::isImpure() const {
+ return std::visit(overloaded {
+ [](const InputAddressed & ia) {
+ return false;
+ },
+ [](const ContentAddressed & ca) {
+ return !ca.pure;
+ },
+ }, raw());
}
@@ -178,34 +177,26 @@ static DerivationOutput parseDerivationOutput(const Store & store,
if (hashS != "") {
validatePath(pathS);
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
- return DerivationOutput {
- .output = DerivationOutputCAFixed {
- // FIXME non-trivial fixed refs set
- .ca = contentAddressFromMethodHashAndRefs(
- method, std::move(hash), {}),
- },
+ return DerivationOutput::CAFixed {
+ // FIXME non-trivial fixed refs set
+ .ca = contentAddressFromMethodHashAndRefs(
+ method, std::move(hash), {}),
};
} else {
settings.requireExperimentalFeature(Xp::CaDerivations);
assert(pathS == "");
- return DerivationOutput {
- .output = DerivationOutputCAFloating {
- .method = std::move(method),
- .hashType = std::move(hashType),
- },
+ return DerivationOutput::CAFloating {
+ .method = std::move(method),
+ .hashType = std::move(hashType),
};
}
} else {
if (pathS == "") {
- return DerivationOutput {
- .output = DerivationOutputDeferred { }
- };
+ return DerivationOutput::Deferred { };
}
validatePath(pathS);
- return DerivationOutput {
- .output = DerivationOutputInputAddressed {
- .path = store.parseStorePath(pathS),
- }
+ return DerivationOutput::InputAddressed {
+ .path = store.parseStorePath(pathS),
};
}
}
@@ -333,27 +324,27 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
std::visit(overloaded {
- [&](const DerivationOutputInputAddressed & doi) {
+ [&](const DerivationOutput::InputAddressed & doi) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
- [&](const DerivationOutputCAFixed & dof) {
+ [&](const DerivationOutput::CAFixed & dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, printMethodAlgo(dof.ca));
s += ','; printUnquotedString(s, getContentAddressHash(dof.ca).to_string(Base16, false));
},
- [&](const DerivationOutputCAFloating & dof) {
+ [&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
- [&](const DerivationOutputDeferred &) {
+ [&](const DerivationOutput::Deferred &) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
}
- }, i.second.output);
+ }, i.second.raw());
s += ')';
}
@@ -421,13 +412,13 @@ DerivationType BasicDerivation::type() const
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
- [&](const DerivationOutputInputAddressed &) {
+ [&](const DerivationOutput::InputAddressed &) {
inputAddressedOutputs.insert(i.first);
},
- [&](const DerivationOutputCAFixed &) {
+ [&](const DerivationOutput::CAFixed &) {
fixedCAOutputs.insert(i.first);
},
- [&](const DerivationOutputCAFloating & dof) {
+ [&](const DerivationOutput::CAFloating & dof) {
floatingCAOutputs.insert(i.first);
if (!floatingHashType) {
floatingHashType = dof.hashType;
@@ -436,27 +427,37 @@ DerivationType BasicDerivation::type() const
throw Error("All floating outputs must use the same hash type");
}
},
- [&](const DerivationOutputDeferred &) {
+ [&](const DerivationOutput::Deferred &) {
deferredIAOutputs.insert(i.first);
},
- }, i.second.output);
+ }, i.second.raw());
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
throw Error("Must have at least one output");
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
- return DerivationType::InputAddressed;
+ return DerivationType::InputAddressed {
+ .deferred = false,
+ };
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
- return DerivationType::CAFixed;
+ return DerivationType::ContentAddressed {
+ .pure = false,
+ .fixed = true,
+ };
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
- return DerivationType::CAFloating;
+ return DerivationType::ContentAddressed {
+ .pure = true,
+ .fixed = false,
+ };
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
- return DerivationType::DeferredInputAddressed;
+ return DerivationType::InputAddressed {
+ .deferred = true,
+ };
} else {
throw Error("Can't mix derivation output types");
}
@@ -508,13 +509,13 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
- bool isDeferred = false;
+ auto type = drv.type();
+
/* Return a fixed hash for fixed-output derivations. */
- switch (drv.type()) {
- case DerivationType::CAFixed: {
+ if (type.isFixed()) {
std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) {
- auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
+ auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
auto hash = hashString(htSHA256, "fixed:out:"
+ printMethodAlgo(dof.ca) + ":"
+ getContentAddressHash(dof.ca).to_string(Base16, false) + ":"
@@ -523,33 +524,37 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
}
return outputHashes;
}
- case DerivationType::CAFloating:
- isDeferred = true;
- break;
- case DerivationType::InputAddressed:
- break;
- case DerivationType::DeferredInputAddressed:
- break;
- }
+
+ auto kind = std::visit(overloaded {
+ [](const DerivationType::InputAddressed & ia) {
+ /* This might be a "pesimistically" deferred output, so we don't
+ "taint" the kind yet. */
+ return DrvHash::Kind::Regular;
+ },
+ [](const DerivationType::ContentAddressed & ca) {
+ return ca.fixed
+ ? DrvHash::Kind::Regular
+ : DrvHash::Kind::Deferred;
+ },
+ }, drv.type().raw());
/* For other derivations, replace the inputs paths with recursive
calls to this function. */
std::map<std::string, StringSet> inputs2;
- for (auto & i : drv.inputDrvs) {
- const auto & res = pathDerivationModulo(store, i.first);
+ for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
+ // Avoid lambda capture restriction with standard / Clang
+ auto & inputOutputs = inputOutputs0;
+ const auto & res = pathDerivationModulo(store, drvPath);
std::visit(overloaded {
// Regular non-CA derivation, replace derivation
- [&](const Hash & drvHash) {
- inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
- },
- [&](const DeferredHash & deferredHash) {
- isDeferred = true;
- inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
+ [&](const DrvHash & drvHash) {
+ kind |= drvHash.kind;
+ inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs);
},
// CA derivation's output hashes
[&](const CaOutputHashes & outputHashes) {
std::set<std::string> justOut = { "out" };
- for (auto & output : i.second) {
+ for (auto & output : inputOutputs) {
/* Put each one in with a single "out" output.. */
const auto h = outputHashes.at(output);
inputs2.insert_or_assign(
@@ -557,15 +562,24 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
justOut);
}
},
- }, res);
+ }, res.raw());
}
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
- if (isDeferred)
- return DeferredHash { hash };
- else
- return hash;
+ return DrvHash { .hash = hash, .kind = kind };
+}
+
+
+void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept
+{
+ switch (other) {
+ case DrvHash::Kind::Regular:
+ break;
+ case DrvHash::Kind::Deferred:
+ self = other;
+ break;
+ }
}
@@ -573,20 +587,15 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
{
std::map<std::string, Hash> res;
std::visit(overloaded {
- [&](const Hash & drvHash) {
- for (auto & outputName : drv.outputNames()) {
- res.insert({outputName, drvHash});
- }
- },
- [&](const DeferredHash & deferredHash) {
+ [&](const DrvHash & drvHash) {
for (auto & outputName : drv.outputNames()) {
- res.insert({outputName, deferredHash.hash});
+ res.insert({outputName, drvHash.hash});
}
},
[&](const CaOutputHashes & outputHashes) {
res = outputHashes;
},
- }, hashDerivationModulo(store, drv, true));
+ }, hashDerivationModulo(store, drv, true).raw());
return res;
}
@@ -667,27 +676,27 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
for (auto & i : drv.outputs) {
out << i.first;
std::visit(overloaded {
- [&](const DerivationOutputInputAddressed & doi) {
+ [&](const DerivationOutput::InputAddressed & doi) {
out << store.printStorePath(doi.path)
<< ""
<< "";
},
- [&](const DerivationOutputCAFixed & dof) {
+ [&](const DerivationOutput::CAFixed & dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first))
<< printMethodAlgo(dof.ca)
<< getContentAddressHash(dof.ca).to_string(Base16, false);
},
- [&](const DerivationOutputCAFloating & dof) {
+ [&](const DerivationOutput::CAFloating & dof) {
out << ""
<< (makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
- [&](const DerivationOutputDeferred &) {
+ [&](const DerivationOutput::Deferred &) {
out << ""
<< ""
<< "";
},
- }, i.second.output);
+ }, i.second.raw());
}
worker_proto::write(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
@@ -735,45 +744,63 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) {
- if (std::holds_alternative<DerivationOutputDeferred>(output.output)) {
- Hash h = std::get<Hash>(hashModulo);
+ if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
+ auto & h = hashModulo.requireNoFixedNonDeferred();
auto outPath = store.makeOutputPath(outputName, h, drv.name);
drv.env[outputName] = store.printStorePath(outPath);
- output = DerivationOutput {
- .output = DerivationOutputInputAddressed {
- .path = std::move(outPath),
- },
+ output = DerivationOutput::InputAddressed {
+ .path = std::move(outPath),
};
}
}
}
+const Hash & DrvHashModulo::requireNoFixedNonDeferred() const {
+ auto * drvHashOpt = std::get_if<DrvHash>(&raw());
+ assert(drvHashOpt);
+ assert(drvHashOpt->kind == DrvHash::Kind::Regular);
+ return drvHashOpt->hash;
+}
+
+static bool tryResolveInput(
+ Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
+ const StorePath & inputDrv, const StringSet & inputOutputs)
+{
+ auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
+
+ auto getOutput = [&](const std::string & outputName) {
+ auto & actualPathOpt = inputDrvOutputs.at(outputName);
+ if (!actualPathOpt)
+ warn("output %s of input %s missing, aborting the resolving",
+ outputName,
+ store.printStorePath(inputDrv)
+ );
+ return actualPathOpt;
+ };
+
+ for (auto & outputName : inputOutputs) {
+ auto actualPathOpt = getOutput(outputName);
+ if (!actualPathOpt) return false;
+ auto actualPath = *actualPathOpt;
+ inputRewrites.emplace(
+ downstreamPlaceholder(store, inputDrv, outputName),
+ store.printStorePath(actualPath));
+ inputSrcs.insert(std::move(actualPath));
+ }
+
+ return true;
+}
+
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation
StringMap inputRewrites;
- for (auto & input : inputDrvs) {
- auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first);
- StringSet newOutputNames;
- for (auto & outputName : input.second) {
- auto actualPathOpt = inputDrvOutputs.at(outputName);
- if (!actualPathOpt) {
- warn("output %s of input %s missing, aborting the resolving",
- outputName,
- store.printStorePath(input.first)
- );
- return std::nullopt;
- }
- auto actualPath = *actualPathOpt;
- inputRewrites.emplace(
- downstreamPlaceholder(store, input.first, outputName),
- store.printStorePath(actualPath));
- resolved.inputSrcs.insert(std::move(actualPath));
- }
- }
+ for (auto & [inputDrv, inputOutputs] : inputDrvs)
+ if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
+ return std::nullopt;
rewriteDerivation(store, resolved, inputRewrites);
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index f59887465..f41a9d93b 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -4,6 +4,7 @@
#include "types.hh"
#include "hash.hh"
#include "content-address.hh"
+#include "repair-flag.hh"
#include "sync.hh"
#include <map>
@@ -44,19 +45,31 @@ struct DerivationOutputCAFloating
*/
struct DerivationOutputDeferred {};
-struct DerivationOutput
+typedef std::variant<
+ DerivationOutputInputAddressed,
+ DerivationOutputCAFixed,
+ DerivationOutputCAFloating,
+ DerivationOutputDeferred
+> _DerivationOutputRaw;
+
+struct DerivationOutput : _DerivationOutputRaw
{
- std::variant<
- DerivationOutputInputAddressed,
- DerivationOutputCAFixed,
- DerivationOutputCAFloating,
- DerivationOutputDeferred
- > output;
+ using Raw = _DerivationOutputRaw;
+ using Raw::Raw;
+
+ using InputAddressed = DerivationOutputInputAddressed;
+ using CAFixed = DerivationOutputCAFixed;
+ using CAFloating = DerivationOutputCAFloating;
+ using Deferred = DerivationOutputDeferred;
/* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer
interface provided by BasicDerivation::outputsAndOptPaths */
std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+
+ inline const Raw & raw() const {
+ return static_cast<const Raw &>(*this);
+ }
};
typedef std::map<std::string, DerivationOutput> DerivationOutputs;
@@ -72,30 +85,50 @@ typedef std::map<std::string, std::pair<DerivationOutput, std::optional<StorePat
output IDs we are interested in. */
typedef std::map<StorePath, StringSet> DerivationInputs;
-enum struct DerivationType : uint8_t {
- InputAddressed,
- DeferredInputAddressed,
- CAFixed,
- CAFloating,
+struct DerivationType_InputAddressed {
+ bool deferred;
};
-/* Do the outputs of the derivation have paths calculated from their content,
- or from the derivation itself? */
-bool derivationIsCA(DerivationType);
-
-/* Is the content of the outputs fixed a-priori via a hash? Never true for
- non-CA derivations. */
-bool derivationIsFixed(DerivationType);
-
-/* Is the derivation impure and needs to access non-deterministic resources, or
- pure and can be sandboxed? Note that whether or not we actually sandbox the
- derivation is controlled separately. Never true for non-CA derivations. */
-bool derivationIsImpure(DerivationType);
+struct DerivationType_ContentAddressed {
+ bool pure;
+ bool fixed;
+};
-/* Does the derivation knows its own output paths?
- * Only true when there's no floating-ca derivation involved in the closure.
- */
-bool derivationHasKnownOutputPaths(DerivationType);
+typedef std::variant<
+ DerivationType_InputAddressed,
+ DerivationType_ContentAddressed
+> _DerivationTypeRaw;
+
+struct DerivationType : _DerivationTypeRaw {
+ using Raw = _DerivationTypeRaw;
+ using Raw::Raw;
+ using InputAddressed = DerivationType_InputAddressed;
+ using ContentAddressed = DerivationType_ContentAddressed;
+
+
+ /* Do the outputs of the derivation have paths calculated from their content,
+ or from the derivation itself? */
+ bool isCA() const;
+
+ /* Is the content of the outputs fixed a-priori via a hash? Never true for
+ non-CA derivations. */
+ bool isFixed() const;
+
+ /* Is the derivation impure and needs to access non-deterministic resources, or
+ pure and can be sandboxed? Note that whether or not we actually sandbox the
+ derivation is controlled separately. Never true for non-CA derivations. */
+ bool isImpure() const;
+
+ /* Does the derivation knows its own output paths?
+ Only true when there's no floating-ca derivation involved in the
+ closure, or if fixed output.
+ */
+ bool hasKnownOutputPaths() const;
+
+ inline const Raw & raw() const {
+ return static_cast<const Raw &>(*this);
+ }
+};
struct BasicDerivation
{
@@ -150,8 +183,6 @@ struct Derivation : BasicDerivation
class Store;
-enum RepairFlag : bool { NoRepair = false, Repair = true };
-
/* Write a derivation to the Nix store, and return its path. */
StorePath writeDerivation(Store & store,
const Derivation & drv,
@@ -175,13 +206,43 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
-struct DeferredHash { Hash hash; };
+struct DrvHash {
+ Hash hash;
+
+ enum struct Kind: bool {
+ // Statically determined derivations.
+ // This hash will be directly used to compute the output paths
+ Regular,
+ // Floating-output derivations (and their reverse dependencies).
+ Deferred,
+ };
+
+ Kind kind;
+};
+
+void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
typedef std::variant<
- Hash, // regular DRV normalized hash
- CaOutputHashes, // Fixed-output derivation hashes
- DeferredHash // Deferred hashes for floating outputs drvs and their dependencies
-> DrvHashModulo;
+ // Regular normalized derivation hash, and whether it was deferred (because
+ // an ancestor derivation is a floating content addressed derivation).
+ DrvHash,
+ // Fixed-output derivation hashes
+ CaOutputHashes
+> _DrvHashModuloRaw;
+
+struct DrvHashModulo : _DrvHashModuloRaw {
+ using Raw = _DrvHashModuloRaw;
+ using Raw::Raw;
+
+ /* Get hash, throwing if it is per-output CA hashes or a
+ deferred Drv hash.
+ */
+ const Hash & requireNoFixedNonDeferred() const;
+
+ inline const Raw & raw() const {
+ return static_cast<const Raw &>(*this);
+ }
+};
/* Returns hashes with the details of fixed-output subderivations
expunged.
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index 194489580..319b1c790 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -1,4 +1,5 @@
#include "derived-path.hh"
+#include "derivations.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
@@ -11,6 +12,21 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
return res;
}
+nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
+ nlohmann::json res;
+ res["drvPath"] = store->printStorePath(drvPath);
+ // Fallback for the input-addressed derivation case: We expect to always be
+ // able to print the output paths, so let’s do it
+ auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath);
+ for (const auto& output : outputs) {
+ if (knownOutputs.at(output))
+ res["outputs"][output] = store->printStorePath(knownOutputs.at(output).value());
+ else
+ res["outputs"][output] = nullptr;
+ }
+ return res;
+}
+
nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
@@ -35,16 +51,22 @@ StorePathSet BuiltPath::outPaths() const
);
}
-nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store) {
+template<typename T>
+nlohmann::json stuffToJSON(const std::vector<T> & ts, ref<Store> store) {
auto res = nlohmann::json::array();
- for (const BuiltPath & buildable : buildables) {
- std::visit([&res, store](const auto & buildable) {
- res.push_back(buildable.toJSON(store));
- }, buildable.raw());
+ for (const T & t : ts) {
+ std::visit([&res, store](const auto & t) {
+ res.push_back(t.toJSON(store));
+ }, t.raw());
}
return res;
}
+nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store)
+{ return stuffToJSON<BuiltPath>(buildables, store); }
+nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
+{ return stuffToJSON<DerivedPath>(paths, store); }
+
std::string DerivedPath::Opaque::to_string(const Store & store) const {
return store.printStorePath(path);
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index ef7c5326c..5a38203ae 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -46,6 +46,7 @@ struct DerivedPathBuilt {
std::string to_string(const Store & store) const;
static DerivedPathBuilt parse(const Store & store, std::string_view);
+ nlohmann::json toJSON(ref<Store> store) const;
};
using _DerivedPathRaw = std::variant<
@@ -120,5 +121,6 @@ typedef std::vector<DerivedPath> DerivedPaths;
typedef std::vector<BuiltPath> BuiltPaths;
nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store);
+nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref<Store> store);
}
diff --git a/src/libstore/gc-store.cc b/src/libstore/gc-store.cc
deleted file mode 100644
index 3dbdec53b..000000000
--- a/src/libstore/gc-store.cc
+++ /dev/null
@@ -1,13 +0,0 @@
-#include "gc-store.hh"
-
-namespace nix {
-
-GcStore & requireGcStore(Store & store)
-{
- auto * gcStore = dynamic_cast<GcStore *>(&store);
- if (!gcStore)
- throw UsageError("Garbage collection not supported by this store");
- return *gcStore;
-}
-
-}
diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh
index 829f70dc4..b3cbbad74 100644
--- a/src/libstore/gc-store.hh
+++ b/src/libstore/gc-store.hh
@@ -61,6 +61,8 @@ struct GCResults
struct GcStore : public virtual Store
{
+ inline static std::string operationName = "Garbage collection";
+
/* Add an indirect root, which is merely a symlink to `path' from
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
to be a symlink to a store path. The garbage collector will
@@ -79,6 +81,4 @@ struct GcStore : public virtual Store
virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
};
-GcStore & requireGcStore(Store & store);
-
}
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 024da66c1..f65fb1b2e 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -678,7 +678,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
alive.insert(start);
try {
StorePathSet closure;
- computeFSClosure(*path, closure);
+ computeFSClosure(*path, closure,
+ /* flipDirection */ false, gcKeepOutputs, gcKeepDerivations);
for (auto & p : closure)
alive.insert(p);
} catch (InvalidPath &) { }
@@ -841,7 +842,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
if (unlink(path.c_str()) == -1)
throw SysError("deleting '%1%'", path);
- results.bytesFreed += st.st_size;
+ /* Do not accound for deleted file here. Rely on deletePath()
+ accounting. */
}
struct stat st;
diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh
index fbd49dc2c..e6fb3201a 100644
--- a/src/libstore/local-fs-store.hh
+++ b/src/libstore/local-fs-store.hh
@@ -2,6 +2,7 @@
#include "store-api.hh"
#include "gc-store.hh"
+#include "log-store.hh"
namespace nix {
@@ -24,7 +25,10 @@ struct LocalFSStoreConfig : virtual StoreConfig
"physical path to the Nix store"};
};
-class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store, virtual GcStore
+class LocalFSStore : public virtual LocalFSStoreConfig,
+ public virtual Store,
+ public virtual GcStore,
+ public virtual LogStore
{
public:
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 52ca5a34a..9c0d482f7 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -698,11 +698,11 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
std::optional<Hash> h;
for (auto & i : drv.outputs) {
std::visit(overloaded {
- [&](const DerivationOutputInputAddressed & doia) {
+ [&](const DerivationOutput::InputAddressed & doia) {
if (!h) {
// somewhat expensive so we do lazily
- auto temp = hashDerivationModulo(*this, drv, true);
- h = std::get<Hash>(temp);
+ auto h0 = hashDerivationModulo(*this, drv, true);
+ h = h0.requireNoFixedNonDeferred();
}
StorePath recomputed = makeOutputPath(i.first, *h, drvName);
if (doia.path != recomputed)
@@ -710,16 +710,17 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
envHasRightPath(doia.path, i.first);
},
- [&](const DerivationOutputCAFixed & dof) {
+ [&](const DerivationOutput::CAFixed & dof) {
auto path = dof.path(*this, drvName, i.first);
envHasRightPath(path, i.first);
},
- [&](const DerivationOutputCAFloating &) {
+ [&](const DerivationOutput::CAFloating &) {
/* Nothing to check */
},
- [&](const DerivationOutputDeferred &) {
+ [&](const DerivationOutput::Deferred &) {
+ /* Nothing to check */
},
- }, i.second.output);
+ }, i.second.raw());
}
}
diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh
new file mode 100644
index 000000000..ff1b92e17
--- /dev/null
+++ b/src/libstore/log-store.hh
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "store-api.hh"
+
+
+namespace nix {
+
+struct LogStore : public virtual Store
+{
+ inline static std::string operationName = "Build log storage and retrieval";
+
+ /* Return the build log of the specified store path, if available,
+ or null otherwise. */
+ virtual std::optional<std::string> getBuildLog(const StorePath & path) = 0;
+
+ virtual void addBuildLog(const StorePath & path, std::string_view log) = 0;
+
+ static LogStore & require(Store & store);
+};
+
+}
diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc
new file mode 100644
index 000000000..9655a0555
--- /dev/null
+++ b/src/libstore/make-content-addressed.cc
@@ -0,0 +1,83 @@
+#include "make-content-addressed.hh"
+#include "references.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+ Store & srcStore,
+ Store & dstStore,
+ const StorePathSet & storePaths)
+{
+ StorePathSet closure;
+ srcStore.computeFSClosure(storePaths, closure);
+
+ auto paths = srcStore.topoSortPaths(closure);
+
+ std::reverse(paths.begin(), paths.end());
+
+ std::map<StorePath, StorePath> remappings;
+
+ for (auto & path : paths) {
+ auto pathS = srcStore.printStorePath(path);
+ auto oldInfo = srcStore.queryPathInfo(path);
+ std::string oldHashPart(path.hashPart());
+
+ StringSink sink;
+ srcStore.narFromPath(path, sink);
+
+ StringMap rewrites;
+
+ PathReferences<StorePath> refs;
+ refs.hasSelfReference = oldInfo->hasSelfReference;
+ for (auto & ref : oldInfo->references) {
+ auto i = remappings.find(ref);
+ auto replacement = i != remappings.end() ? i->second : ref;
+ // FIXME: warn about unremapped paths?
+ if (replacement != ref) {
+ rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
+ refs.references.insert(std::move(replacement));
+ }
+ }
+
+ sink.s = rewriteStrings(sink.s, rewrites);
+
+ HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+ hashModuloSink(sink.s);
+
+ auto narModuloHash = hashModuloSink.finish().first;
+
+ ValidPathInfo info {
+ dstStore,
+ StorePathDescriptor {
+ .name = std::string { path.name() },
+ .info = FixedOutputInfo {
+ {
+ .method = FileIngestionMethod::Recursive,
+ .hash = narModuloHash,
+ },
+ std::move(refs),
+ },
+ },
+ Hash::dummy,
+ };
+
+ printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path));
+
+ StringSink sink2;
+ RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2);
+ rsink2(sink.s);
+ rsink2.flush();
+
+ info.narHash = hashString(htSHA256, sink2.s);
+ info.narSize = sink.s.size();
+
+ StringSource source(sink2.s);
+ dstStore.addToStore(info, source);
+
+ remappings.insert_or_assign(std::move(path), std::move(info.path));
+ }
+
+ return remappings;
+}
+
+}
diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh
new file mode 100644
index 000000000..c4a66ed41
--- /dev/null
+++ b/src/libstore/make-content-addressed.hh
@@ -0,0 +1,12 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+ Store & srcStore,
+ Store & dstStore,
+ const StorePathSet & storePaths);
+
+}
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index e411cbe04..42965d66e 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -87,7 +87,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
auto out = drv.outputs.find("out");
if (out == drv.outputs.end())
return std::nullopt;
- if (auto dof = std::get_if<DerivationOutputCAFixed>(&out->second.output)) {
+ if (auto dof = std::get_if<DerivationOutput::CAFixed>(&out->second)) {
return std::visit(overloaded {
[&](const TextInfo & ti) -> std::optional<ContentAddress> {
if (!ti.references.empty())
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 8c65053e4..f2288a04e 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -93,7 +93,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
StringSet res;
for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
res.insert(i);
- if (!derivationHasKnownOutputPaths(drv.type()))
+ if (!drv.type().hasKnownOutputPaths())
res.insert("ca-derivations");
return res;
}
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index effcf099d..95bec21e8 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -1,5 +1,6 @@
#pragma once
+#include "derivations.hh"
#include "store-api.hh"
#include <nlohmann/json_fwd.hpp>
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index eb74d88c9..961d1bc08 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -5,6 +5,7 @@
#include "store-api.hh"
#include "gc-store.hh"
+#include "log-store.hh"
namespace nix {
@@ -30,7 +31,10 @@ struct RemoteStoreConfig : virtual StoreConfig
/* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */
-class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore
+class RemoteStore : public virtual RemoteStoreConfig,
+ public virtual Store,
+ public virtual GcStore,
+ public virtual LogStore
{
public:
diff --git a/src/libstore/repair-flag.hh b/src/libstore/repair-flag.hh
new file mode 100644
index 000000000..a13cda312
--- /dev/null
+++ b/src/libstore/repair-flag.hh
@@ -0,0 +1,7 @@
+#pragma once
+
+namespace nix {
+
+enum RepairFlag : bool { NoRepair = false, Repair = true };
+
+}
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index bb03daef4..62daa838c 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -52,6 +52,10 @@ public:
bool sameMachine() override
{ return false; }
+ // FIXME extend daemon protocol, move implementation to RemoteStore
+ std::optional<std::string> getBuildLog(const StorePath & path) override
+ { unsupported("getBuildLog"); }
+
private:
struct Connection : RemoteStore::Connection
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 6f4b6b98c..4362177f9 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1,6 +1,7 @@
#include "crypto.hh"
#include "fs-accessor.hh"
#include "globals.hh"
+#include "derivations.hh"
#include "store-api.hh"
#include "util.hh"
#include "nar-info-disk-cache.hh"
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 1a320bcb1..5ea606886 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -10,8 +10,8 @@
#include "sync.hh"
#include "globals.hh"
#include "config.hh"
-#include "derivations.hh"
#include "path-info.hh"
+#include "repair-flag.hh"
#include <atomic>
#include <limits>
@@ -62,6 +62,8 @@ MakeError(BadStorePath, Error);
MakeError(InvalidStoreURI, Error);
+struct BasicDerivation;
+struct Derivation;
class FSAccessor;
class NarInfoDiskCache;
class Store;
@@ -601,14 +603,6 @@ public:
*/
StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths);
- /* Return the build log of the specified store path, if available,
- or null otherwise. */
- virtual std::optional<std::string> getBuildLog(const StorePath & path)
- { return std::nullopt; }
-
- virtual void addBuildLog(const StorePath & path, std::string_view log)
- { unsupported("addBuildLog"); }
-
/* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */
void clearPathInfoCache()
diff --git a/src/libstore/store-cast.hh b/src/libstore/store-cast.hh
new file mode 100644
index 000000000..ff62fc359
--- /dev/null
+++ b/src/libstore/store-cast.hh
@@ -0,0 +1,16 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+template<typename T>
+T & require(Store & store)
+{
+ auto * castedStore = dynamic_cast<T *>(&store);
+ if (!castedStore)
+ throw UsageError("%s not supported by store '%s'", T::operationName, store.getUri());
+ return *castedStore;
+}
+
+}