aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libstore/build/derivation-goal.cc31
-rw-r--r--src/libstore/build/drv-output-substitution-goal.cc95
-rw-r--r--src/libstore/build/drv-output-substitution-goal.hh50
-rw-r--r--src/libstore/build/entry-points.cc8
-rw-r--r--src/libstore/build/substitution-goal.cc32
-rw-r--r--src/libstore/build/substitution-goal.hh9
-rw-r--r--src/libstore/build/worker.cc34
-rw-r--r--src/libstore/build/worker.hh17
-rw-r--r--src/libstore/local-store.hh2
9 files changed, 227 insertions, 51 deletions
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index c29237f5c..7dcd2a6eb 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -170,7 +170,7 @@ void DerivationGoal::getDerivation()
return;
}
- addWaitee(upcast_goal(worker.makeSubstitutionGoal(drvPath)));
+ addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
state = &DerivationGoal::loadDerivation;
}
@@ -246,17 +246,22 @@ void DerivationGoal::haveDerivation()
through substitutes. If that doesn't work, we'll build
them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
- for (auto & [_, status] : initialOutputs) {
+ for (auto & [outputName, status] : initialOutputs) {
if (!status.wanted) continue;
- if (!status.known) {
- warn("do not know how to query for unknown floating content-addressed derivation output yet");
- /* Nothing to wait for; tail call */
- return DerivationGoal::gaveUpOnSubstitution();
- }
- addWaitee(upcast_goal(worker.makeSubstitutionGoal(
- status.known->path,
- buildMode == bmRepair ? Repair : NoRepair,
- getDerivationCA(*drv))));
+ if (!status.known)
+ addWaitee(
+ upcast_goal(
+ worker.makeDrvOutputSubstitutionGoal(
+ DrvOutput{status.outputHash, outputName},
+ buildMode == bmRepair ? Repair : NoRepair
+ )
+ )
+ );
+ else
+ addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
+ status.known->path,
+ buildMode == bmRepair ? Repair : NoRepair,
+ getDerivationCA(*drv))));
}
if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -337,7 +342,7 @@ void DerivationGoal::gaveUpOnSubstitution()
if (!settings.useSubstitutes)
throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
- addWaitee(upcast_goal(worker.makeSubstitutionGoal(i)));
+ addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i)));
}
if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -388,7 +393,7 @@ void DerivationGoal::repairClosure()
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
auto drvPath2 = outputsToDrv.find(i);
if (drvPath2 == outputsToDrv.end())
- addWaitee(upcast_goal(worker.makeSubstitutionGoal(i, Repair)));
+ addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else
addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
}
diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc
new file mode 100644
index 000000000..a5ac4c49d
--- /dev/null
+++ b/src/libstore/build/drv-output-substitution-goal.cc
@@ -0,0 +1,95 @@
+#include "drv-output-substitution-goal.hh"
+#include "worker.hh"
+#include "substitution-goal.hh"
+
+namespace nix {
+
+DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
+ : Goal(worker)
+ , id(id)
+{
+ state = &DrvOutputSubstitutionGoal::init;
+ name = fmt("substitution of '%s'", id.to_string());
+ trace("created");
+}
+
+
+void DrvOutputSubstitutionGoal::init()
+{
+ trace("init");
+ subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+ tryNext();
+}
+
+void DrvOutputSubstitutionGoal::tryNext()
+{
+ trace("Trying next substituter");
+
+ if (subs.size() == 0) {
+ /* None left. Terminate this goal and let someone else deal
+ with it. */
+ debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
+
+ /* Hack: don't indicate failure if there were no substituters.
+ In that case the calling derivation should just do a
+ build. */
+ amDone(substituterFailed ? ecFailed : ecNoSubstituters);
+
+ if (substituterFailed) {
+ worker.failedSubstitutions++;
+ worker.updateProgress();
+ }
+
+ return;
+ }
+
+ auto sub = subs.front();
+ subs.pop_front();
+
+ // FIXME: Make async
+ outputInfo = sub->queryRealisation(id);
+ if (!outputInfo) {
+ tryNext();
+ return;
+ }
+
+ addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
+
+ if (waitees.empty()) outPathValid();
+ else state = &DrvOutputSubstitutionGoal::outPathValid;
+}
+
+void DrvOutputSubstitutionGoal::outPathValid()
+{
+ assert(outputInfo);
+ trace("Output path substituted");
+
+ if (nrFailed > 0) {
+ debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
+ amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
+ return;
+ }
+
+ worker.store.registerDrvOutput(*outputInfo);
+ finished();
+}
+
+void DrvOutputSubstitutionGoal::finished()
+{
+ trace("finished");
+ amDone(ecSuccess);
+}
+
+string DrvOutputSubstitutionGoal::key()
+{
+ /* "a$" ensures substitution goals happen before derivation
+ goals. */
+ return "a$" + std::string(id.to_string());
+}
+
+void DrvOutputSubstitutionGoal::work()
+{
+ (this->*state)();
+}
+
+}
diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh
new file mode 100644
index 000000000..63ab53d89
--- /dev/null
+++ b/src/libstore/build/drv-output-substitution-goal.hh
@@ -0,0 +1,50 @@
+#pragma once
+
+#include "store-api.hh"
+#include "goal.hh"
+#include "realisation.hh"
+
+namespace nix {
+
+class Worker;
+
+// Substitution of a derivation output.
+// This is done in three steps:
+// 1. Fetch the output info from a substituter
+// 2. Substitute the corresponding output path
+// 3. Register the output info
+class DrvOutputSubstitutionGoal : public Goal {
+private:
+ // The drv output we're trying to substitue
+ DrvOutput id;
+
+ // The realisation corresponding to the given output id.
+ // Will be filled once we can get it.
+ std::optional<Realisation> outputInfo;
+
+ /* The remaining substituters. */
+ std::list<ref<Store>> subs;
+
+ /* Whether a substituter failed. */
+ bool substituterFailed = false;
+
+public:
+ DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+
+ typedef void (DrvOutputSubstitutionGoal::*GoalState)();
+ GoalState state;
+
+ void init();
+ void tryNext();
+ void outPathValid();
+ void finished();
+
+ void timedOut(Error && ex) override { abort(); };
+
+ string key() override;
+
+ void work() override;
+
+};
+
+}
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index 01a564aba..686364440 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, Build
if (path.path.isDerivation())
goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
else
- goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
+ goals.insert(worker.makePathSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
}
worker.run(goals);
@@ -31,7 +31,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, Build
}
if (i->exitCode != Goal::ecSuccess) {
if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
- else if (auto i2 = dynamic_cast<SubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
+ else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
}
}
@@ -90,7 +90,7 @@ void Store::ensurePath(const StorePath & path)
if (isValidPath(path)) return;
Worker worker(*this);
- GoalPtr goal = worker.makeSubstitutionGoal(path);
+ GoalPtr goal = worker.makePathSubstitutionGoal(path);
Goals goals = {goal};
worker.run(goals);
@@ -108,7 +108,7 @@ void Store::ensurePath(const StorePath & path)
void LocalStore::repairPath(const StorePath & path)
{
Worker worker(*this);
- GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
+ GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
Goals goals = {goal};
worker.run(goals);
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index c4b0de78d..5d88b8758 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -5,20 +5,20 @@
namespace nix {
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
+PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
, ca(ca)
{
- state = &SubstitutionGoal::init;
+ state = &PathSubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
trace("created");
maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
}
-SubstitutionGoal::~SubstitutionGoal()
+PathSubstitutionGoal::~PathSubstitutionGoal()
{
try {
if (thr.joinable()) {
@@ -32,13 +32,13 @@ SubstitutionGoal::~SubstitutionGoal()
}
-void SubstitutionGoal::work()
+void PathSubstitutionGoal::work()
{
(this->*state)();
}
-void SubstitutionGoal::init()
+void PathSubstitutionGoal::init()
{
trace("init");
@@ -59,7 +59,7 @@ void SubstitutionGoal::init()
}
-void SubstitutionGoal::tryNext()
+void PathSubstitutionGoal::tryNext()
{
trace("trying next substituter");
@@ -154,16 +154,16 @@ void SubstitutionGoal::tryNext()
paths referenced by this one. */
for (auto & i : info->references)
if (i != storePath) /* ignore self-references */
- addWaitee(worker.makeSubstitutionGoal(i));
+ addWaitee(worker.makePathSubstitutionGoal(i));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
referencesValid();
else
- state = &SubstitutionGoal::referencesValid;
+ state = &PathSubstitutionGoal::referencesValid;
}
-void SubstitutionGoal::referencesValid()
+void PathSubstitutionGoal::referencesValid()
{
trace("all references realised");
@@ -177,12 +177,12 @@ void SubstitutionGoal::referencesValid()
if (i != storePath) /* ignore self-references */
assert(worker.store.isValidPath(i));
- state = &SubstitutionGoal::tryToRun;
+ state = &PathSubstitutionGoal::tryToRun;
worker.wakeUp(shared_from_this());
}
-void SubstitutionGoal::tryToRun()
+void PathSubstitutionGoal::tryToRun()
{
trace("trying to run");
@@ -221,11 +221,11 @@ void SubstitutionGoal::tryToRun()
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
- state = &SubstitutionGoal::finished;
+ state = &PathSubstitutionGoal::finished;
}
-void SubstitutionGoal::finished()
+void PathSubstitutionGoal::finished()
{
trace("substitute finished");
@@ -249,7 +249,7 @@ void SubstitutionGoal::finished()
}
/* Try the next substitute. */
- state = &SubstitutionGoal::tryNext;
+ state = &PathSubstitutionGoal::tryNext;
worker.wakeUp(shared_from_this());
return;
}
@@ -278,12 +278,12 @@ void SubstitutionGoal::finished()
}
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
+void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
{
}
-void SubstitutionGoal::handleEOF(int fd)
+void PathSubstitutionGoal::handleEOF(int fd)
{
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}
diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh
index dee2cecbf..3b3cb7e32 100644
--- a/src/libstore/build/substitution-goal.hh
+++ b/src/libstore/build/substitution-goal.hh
@@ -8,7 +8,7 @@ namespace nix {
class Worker;
-struct SubstitutionGoal : public Goal
+struct PathSubstitutionGoal : public Goal
{
/* The store path that should be realised through a substitute. */
StorePath storePath;
@@ -47,14 +47,15 @@ struct SubstitutionGoal : public Goal
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
- typedef void (SubstitutionGoal::*GoalState)();
+ typedef void (PathSubstitutionGoal::*GoalState)();
GoalState state;
/* Content address for recomputing store path */
std::optional<ContentAddress> ca;
- SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
- ~SubstitutionGoal();
+public:
+ PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+ ~PathSubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index b2223c3b6..616b17e61 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -1,6 +1,7 @@
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
+#include "drv-output-substitution-goal.hh"
#include "local-derivation-goal.hh"
#include "hook-instance.hh"
@@ -78,20 +79,32 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
}
-std::shared_ptr<SubstitutionGoal> Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
+std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
- std::weak_ptr<SubstitutionGoal> & goal_weak = substitutionGoals[path];
+ std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
auto goal = goal_weak.lock(); // FIXME
if (!goal) {
- goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
+ goal = std::make_shared<PathSubstitutionGoal>(path, *this, repair, ca);
goal_weak = goal;
wakeUp(goal);
}
return goal;
}
-template<typename G>
-static void removeGoal(std::shared_ptr<G> goal, std::map<StorePath, std::weak_ptr<G>> & goalMap)
+std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
+{
+ std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
+ auto goal = goal_weak.lock(); // FIXME
+ if (!goal) {
+ goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, repair, ca);
+ goal_weak = goal;
+ wakeUp(goal);
+ }
+ return goal;
+}
+
+template<typename K, typename G>
+static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{
/* !!! inefficient */
for (auto i = goalMap.begin();
@@ -109,8 +122,10 @@ void Worker::removeGoal(GoalPtr goal)
{
if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
nix::removeGoal(drvGoal, derivationGoals);
- else if (auto subGoal = std::dynamic_pointer_cast<SubstitutionGoal>(goal))
+ else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
nix::removeGoal(subGoal, substitutionGoals);
+ else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
+ nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
else
assert(false);
if (topGoals.find(goal) != topGoals.end()) {
@@ -217,7 +232,7 @@ void Worker::run(const Goals & _topGoals)
topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
topPaths.push_back({goal->drvPath, goal->wantedOutputs});
- } else if (auto goal = dynamic_cast<SubstitutionGoal *>(i.get())) {
+ } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
topPaths.push_back({goal->storePath});
}
}
@@ -471,7 +486,10 @@ void Worker::markContentsGood(const StorePath & path)
}
-GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal) {
+GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
+ return subGoal;
+}
+GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
return subGoal;
}
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 82e711191..918de35f6 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -4,6 +4,7 @@
#include "lock.hh"
#include "store-api.hh"
#include "goal.hh"
+#include "realisation.hh"
#include <future>
#include <thread>
@@ -12,18 +13,20 @@ namespace nix {
/* Forward definition. */
struct DerivationGoal;
-struct SubstitutionGoal;
+struct PathSubstitutionGoal;
+class DrvOutputSubstitutionGoal;
/* Workaround for not being able to declare a something like
- class SubstitutionGoal : public Goal;
+ class PathSubstitutionGoal : public Goal;
even when Goal is a complete type.
This is still a static cast. The purpose of exporting it is to define it in
- a place where `SubstitutionGoal` is concrete, and use it in a place where it
+ a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
is opaque. */
-GoalPtr upcast_goal(std::shared_ptr<SubstitutionGoal> subGoal);
+GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
+GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
@@ -72,7 +75,8 @@ private:
/* Maps used to prevent multiple instantiations of a goal for the
same derivation / path. */
std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
- std::map<StorePath, std::weak_ptr<SubstitutionGoal>> substitutionGoals;
+ std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
+ std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
/* Goals waiting for busy paths to be unlocked. */
WeakGoals waitingForAnyGoal;
@@ -146,7 +150,8 @@ public:
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */
- std::shared_ptr<SubstitutionGoal> makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+ std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+ std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 03bb0218d..fc67f215a 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -281,7 +281,9 @@ private:
void createUser(const std::string & userName, uid_t userId) override;
friend struct LocalDerivationGoal;
+ friend struct PathSubstitutionGoal;
friend struct SubstitutionGoal;
+ friend struct DerivationGoal;
};
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 21:01:57 +0000