aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/libstore/build/derivation-goal.cc40
-rw-r--r--src/libstore/build/drv-output-substitution-goal.cc13
-rw-r--r--src/libstore/build/entry-points.cc30
-rw-r--r--src/libstore/build/goal.cc29
-rw-r--r--src/libstore/build/goal.hh28
-rw-r--r--src/libstore/build/substitution-goal.cc10
-rw-r--r--src/libstore/build/worker.cc110
-rw-r--r--src/libstore/build/worker.hh60
8 files changed, 192 insertions, 128 deletions
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 3c4257f08..b8c4d278d 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -13,8 +13,11 @@
#include <boost/outcome/try.hpp>
#include <fstream>
+#include <kj/array.h>
#include <kj/async-unix.h>
+#include <kj/async.h>
#include <kj/debug.h>
+#include <kj/vector.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>
@@ -173,7 +176,7 @@ try {
state = &DerivationGoal::loadDerivation;
- return {WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}};
+ return waitForGoals(worker.goalFactory().makePathSubstitutionGoal(drvPath));
} catch (...) {
return {std::current_exception()};
}
@@ -272,13 +275,13 @@ try {
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
- WaitForGoals result;
+ kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
if (settings.useSubstitutes) {
if (parsedDrv->substitutesAllowed()) {
for (auto & [outputName, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known)
- result.goals.insert(
+ dependencies.add(
worker.goalFactory().makeDrvOutputSubstitutionGoal(
DrvOutput{status.outputHash, outputName},
buildMode == bmRepair ? Repair : NoRepair
@@ -286,7 +289,7 @@ try {
);
else {
auto * cap = getDerivationCA(*drv);
- result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(
+ dependencies.add(worker.goalFactory().makePathSubstitutionGoal(
status.known->path,
buildMode == bmRepair ? Repair : NoRepair,
cap ? std::optional { *cap } : std::nullopt));
@@ -297,11 +300,11 @@ try {
}
}
- if (result.goals.empty()) { /* to prevent hang (no wake-up event) */
+ if (dependencies.empty()) { /* to prevent hang (no wake-up event) */
return outputsSubstitutionTried(inBuildSlot);
} else {
state = &DerivationGoal::outputsSubstitutionTried;
- return {std::move(result)};
+ return waitForGoals(dependencies.releaseAsArray());
}
} catch (...) {
return {std::current_exception()};
@@ -383,7 +386,7 @@ try {
produced using a substitute. So we have to build instead. */
kj::Promise<Result<Goal::WorkResult>> DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) noexcept
try {
- WaitForGoals result;
+ kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
/* At this point we are building all outputs, so if more are wanted there
is no need to restart. */
@@ -396,7 +399,7 @@ try {
addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
if (!inputNode.value.empty())
- result.goals.insert(worker.goalFactory().makeGoal(
+ dependencies.add(worker.goalFactory().makeGoal(
DerivedPath::Built {
.drvPath = inputDrv,
.outputs = inputNode.value,
@@ -441,14 +444,14 @@ try {
if (!settings.useSubstitutes)
throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
- result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i));
+ dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i));
}
- if (result.goals.empty()) {/* to prevent hang (no wake-up event) */
+ if (dependencies.empty()) {/* to prevent hang (no wake-up event) */
return inputsRealised(inBuildSlot);
} else {
state = &DerivationGoal::inputsRealised;
- return {result};
+ return waitForGoals(dependencies.releaseAsArray());
}
} catch (...) {
return {std::current_exception()};
@@ -491,7 +494,7 @@ try {
}
/* Check each path (slow!). */
- WaitForGoals result;
+ kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
for (auto & i : outputClosure) {
if (worker.pathContentsGood(i)) continue;
printError(
@@ -499,9 +502,9 @@ try {
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
auto drvPath2 = outputsToDrv.find(i);
if (drvPath2 == outputsToDrv.end())
- result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i, Repair));
+ dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i, Repair));
else
- result.goals.insert(worker.goalFactory().makeGoal(
+ dependencies.add(worker.goalFactory().makeGoal(
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(drvPath2->second),
.outputs = OutputsSpec::All { },
@@ -509,12 +512,12 @@ try {
bmRepair));
}
- if (result.goals.empty()) {
+ if (dependencies.empty()) {
return {done(BuildResult::AlreadyValid, assertPathValidity())};
}
state = &DerivationGoal::closureRepaired;
- return {result};
+ return waitForGoals(dependencies.releaseAsArray());
} catch (...) {
return {std::current_exception()};
}
@@ -614,11 +617,12 @@ try {
worker.store.printStorePath(pathResolved),
});
- resolvedDrvGoal = worker.goalFactory().makeDerivationGoal(
+ auto dependency = worker.goalFactory().makeDerivationGoal(
pathResolved, wantedOutputs, buildMode);
+ resolvedDrvGoal = dependency.first;
state = &DerivationGoal::resolvedFinished;
- return {WaitForGoals{{resolvedDrvGoal}}};
+ return waitForGoals(std::move(dependency));
}
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc
index 6ef00d1ff..80b2c4cfb 100644
--- a/src/libstore/build/drv-output-substitution-goal.cc
+++ b/src/libstore/build/drv-output-substitution-goal.cc
@@ -4,6 +4,9 @@
#include "worker.hh"
#include "substitution-goal.hh"
#include "signals.hh"
+#include <kj/array.h>
+#include <kj/async.h>
+#include <kj/vector.h>
namespace nix {
@@ -106,7 +109,7 @@ try {
return tryNext(inBuildSlot);
}
- WaitForGoals result;
+ kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
if (depId != id) {
if (auto localOutputInfo = worker.store.queryRealisation(depId);
@@ -122,17 +125,17 @@ try {
);
return tryNext(inBuildSlot);
}
- result.goals.insert(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId));
+ dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId));
}
}
- result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath));
+ dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath));
- if (result.goals.empty()) {
+ if (dependencies.empty()) {
return outPathValid(inBuildSlot);
} else {
state = &DrvOutputSubstitutionGoal::outPathValid;
- return {std::move(result)};
+ return waitForGoals(dependencies.releaseAsArray());
}
} catch (...) {
return {std::current_exception()};
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index a0f18a02c..27c341295 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -17,9 +17,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
Worker worker(*this, evalStore ? *evalStore : *this, aio);
auto goals = runWorker(worker, [&](GoalFactory & gf) {
- Goals goals;
+ Worker::Targets goals;
for (auto & br : reqs)
- goals.insert(gf.makeGoal(br, buildMode));
+ goals.emplace(gf.makeGoal(br, buildMode));
return goals;
});
@@ -60,11 +60,11 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
std::vector<std::pair<const DerivedPath &, GoalPtr>> state;
auto goals = runWorker(worker, [&](GoalFactory & gf) {
- Goals goals;
+ Worker::Targets goals;
for (const auto & req : reqs) {
auto goal = gf.makeGoal(req, buildMode);
- goals.insert(goal);
- state.push_back({req, goal});
+ state.push_back({req, goal.first});
+ goals.emplace(std::move(goal));
}
return goals;
});
@@ -84,8 +84,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
Worker worker(*this, *this, aio);
try {
- auto goals = runWorker(worker, [&](GoalFactory & gf) -> Goals {
- return Goals{gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)};
+ auto goals = runWorker(worker, [&](GoalFactory & gf) {
+ Worker::Targets goals;
+ goals.emplace(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode));
+ return goals;
});
auto goal = *goals.begin();
return goal->buildResult.restrictTo(DerivedPath::Built {
@@ -110,7 +112,9 @@ void Store::ensurePath(const StorePath & path)
Worker worker(*this, *this, aio);
auto goals = runWorker(worker, [&](GoalFactory & gf) {
- return Goals{gf.makePathSubstitutionGoal(path)};
+ Worker::Targets goals;
+ goals.emplace(gf.makePathSubstitutionGoal(path));
+ return goals;
});
auto goal = *goals.begin();
@@ -130,7 +134,9 @@ void Store::repairPath(const StorePath & path)
Worker worker(*this, *this, aio);
auto goals = runWorker(worker, [&](GoalFactory & gf) {
- return Goals{gf.makePathSubstitutionGoal(path, Repair)};
+ Worker::Targets goals;
+ goals.emplace(gf.makePathSubstitutionGoal(path, Repair));
+ return goals;
});
auto goal = *goals.begin();
@@ -140,14 +146,16 @@ void Store::repairPath(const StorePath & path)
auto info = queryPathInfo(path);
if (info->deriver && isValidPath(*info->deriver)) {
worker.run([&](GoalFactory & gf) {
- return Goals{gf.makeGoal(
+ Worker::Targets goals;
+ goals.emplace(gf.makeGoal(
DerivedPath::Built{
.drvPath = makeConstantStorePathRef(*info->deriver),
// FIXME: Should just build the specific output we need.
.outputs = OutputsSpec::All{},
},
bmRepair
- )};
+ ));
+ return goals;
});
} else
throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));
diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
index 649093dbd..8a2f4ab35 100644
--- a/src/libstore/build/goal.cc
+++ b/src/libstore/build/goal.cc
@@ -1,4 +1,5 @@
#include "goal.hh"
+#include "async-collect.hh"
#include "worker.hh"
#include <kj/time.h>
@@ -28,4 +29,32 @@ try {
co_return std::current_exception();
}
+kj::Promise<Result<Goal::WorkResult>>
+Goal::waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<void>>> dependencies) noexcept
+try {
+ auto left = dependencies.size();
+ auto collectDeps = asyncCollect(std::move(dependencies));
+
+ while (auto item = co_await collectDeps.next()) {
+ left--;
+ auto & dep = *item;
+
+ trace(fmt("waitee '%s' done; %d left", dep->name, left));
+
+ if (dep->exitCode != Goal::ecSuccess) ++nrFailed;
+ if (dep->exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters;
+ if (dep->exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure;
+
+ waiteeDone(dep);
+
+ if (dep->exitCode == ecFailed && !settings.keepGoing) {
+ co_return result::success(ContinueImmediately{});
+ }
+ }
+
+ co_return result::success(ContinueImmediately{});
+} catch (...) {
+ co_return result::failure(std::current_exception());
+}
+
}
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index 1ccf9716b..e7a500a00 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -6,6 +6,7 @@
#include "types.hh"
#include "store-api.hh"
#include "build-result.hh"
+#include <concepts> // IWYU pragma: keep
#include <kj/async.h>
namespace nix {
@@ -71,17 +72,6 @@ struct Goal
const bool isDependency;
/**
- * Goals that this goal is waiting for.
- */
- Goals waitees;
-
- /**
- * Goals waiting for this one to finish. Must use weak pointers
- * here to prevent cycles.
- */
- WeakGoals waiters;
-
- /**
* Number of goals we are/were waiting for that have failed.
*/
size_t nrFailed = 0;
@@ -113,6 +103,9 @@ struct Goal
*/
BuildResult buildResult;
+ // for use by Worker only. will go away once work() is a promise.
+ kj::Own<kj::PromiseFulfiller<void>> notify;
+
protected:
AsyncSemaphore::Token slotToken;
@@ -122,9 +115,6 @@ public:
struct [[nodiscard]] StillAlive {};
struct [[nodiscard]] ContinueImmediately {};
- struct [[nodiscard]] WaitForGoals {
- Goals goals;
- };
struct [[nodiscard]] WaitForWorld {
kj::Promise<Outcome<void, Finished>> promise;
};
@@ -141,7 +131,6 @@ public:
struct [[nodiscard]] WorkResult : std::variant<
StillAlive,
ContinueImmediately,
- WaitForGoals,
WaitForWorld,
Finished>
{
@@ -151,6 +140,15 @@ public:
protected:
kj::Promise<Result<WorkResult>> waitForAWhile();
+ kj::Promise<Result<WorkResult>>
+ waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<void>>> dependencies) noexcept;
+
+ template<std::derived_from<Goal>... G>
+ kj::Promise<Result<Goal::WorkResult>>
+ waitForGoals(std::pair<std::shared_ptr<G>, kj::Promise<void>>... goals) noexcept
+ {
+ return waitForGoals(kj::arrOf<std::pair<GoalPtr, kj::Promise<void>>>(std::move(goals)...));
+ }
public:
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index 6d90196fa..74a63ca21 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -3,6 +3,8 @@
#include "nar-info.hh"
#include "signals.hh"
#include "finally.hh"
+#include <kj/array.h>
+#include <kj/vector.h>
namespace nix {
@@ -160,16 +162,16 @@ try {
/* To maintain the closure invariant, we first have to realise the
paths referenced by this one. */
- WaitForGoals result;
+ kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
for (auto & i : info->references)
if (i != storePath) /* ignore self-references */
- result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i));
+ dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i));
- if (result.goals.empty()) {/* to prevent hang (no wake-up event) */
+ if (dependencies.empty()) {/* to prevent hang (no wake-up event) */
return referencesValid(inBuildSlot);
} else {
state = &PathSubstitutionGoal::referencesValid;
- return {std::move(result)};
+ return waitForGoals(dependencies.releaseAsArray());
}
} catch (...) {
return {std::current_exception()};
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index e19917d91..68071a94c 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -52,26 +52,28 @@ Worker::~Worker()
}
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeDerivationGoalCommon(
const StorePath & drvPath,
const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
{
- std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
- std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
+ auto & goal_weak = derivationGoals[drvPath];
+ std::shared_ptr<DerivationGoal> goal = goal_weak.goal.lock();
if (!goal) {
goal = mkDrvGoal();
- goal_weak = goal;
+ goal->notify = std::move(goal_weak.fulfiller);
+ goal_weak.goal = goal;
wakeUp(goal);
} else {
goal->addWantedOutputs(wantedOutputs);
}
- return goal;
+ return {goal, goal_weak.promise->addBranch()};
}
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
- const OutputsSpec & wantedOutputs, BuildMode buildMode)
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeDerivationGoal(
+ const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode
+)
{
return makeDerivationGoalCommon(
drvPath,
@@ -89,8 +91,12 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
}
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
- const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeBasicDerivationGoal(
+ const StorePath & drvPath,
+ const BasicDerivation & drv,
+ const OutputsSpec & wantedOutputs,
+ BuildMode buildMode
+)
{
return makeDerivationGoalCommon(
drvPath,
@@ -108,55 +114,63 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
}
-std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
+std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
+Worker::makePathSubstitutionGoal(
+ const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca
+)
{
- std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
- auto goal = goal_weak.lock(); // FIXME
+ auto & goal_weak = substitutionGoals[path];
+ auto goal = goal_weak.goal.lock(); // FIXME
if (!goal) {
goal = std::make_shared<PathSubstitutionGoal>(path, *this, running, repair, ca);
- goal_weak = goal;
+ goal->notify = std::move(goal_weak.fulfiller);
+ goal_weak.goal = goal;
wakeUp(goal);
}
- return goal;
+ return {goal, goal_weak.promise->addBranch()};
}
-std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
+std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+Worker::makeDrvOutputSubstitutionGoal(
+ const DrvOutput & id, RepairFlag repair, std::optional<ContentAddress> ca
+)
{
- std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
- auto goal = goal_weak.lock(); // FIXME
+ auto & goal_weak = drvOutputSubstitutionGoals[id];
+ auto goal = goal_weak.goal.lock(); // FIXME
if (!goal) {
goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, running, repair, ca);
- goal_weak = goal;
+ goal->notify = std::move(goal_weak.fulfiller);
+ goal_weak.goal = goal;
wakeUp(goal);
}
- return goal;
+ return {goal, goal_weak.promise->addBranch()};
}
-GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
+std::pair<GoalPtr, kj::Promise<void>> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{
return std::visit(overloaded {
- [&](const DerivedPath::Built & bfd) -> GoalPtr {
+ [&](const DerivedPath::Built & bfd) -> std::pair<GoalPtr, kj::Promise<void>> {
if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
else
throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
},
- [&](const DerivedPath::Opaque & bo) -> GoalPtr {
+ [&](const DerivedPath::Opaque & bo) -> std::pair<GoalPtr, kj::Promise<void>> {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
},
}, req.raw());
}
-template<typename K, typename G>
-static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
+template<typename G>
+static void removeGoal(std::shared_ptr<G> goal, auto & goalMap)
{
/* !!! inefficient */
for (auto i = goalMap.begin();
i != goalMap.end(); )
- if (i->second.lock() == goal) {
+ if (i->second.goal.lock() == goal) {
auto j = i; ++j;
goalMap.erase(i);
i = j;
@@ -177,33 +191,8 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f)
hashMismatch |= f.hashMismatch;
checkMismatch |= f.checkMismatch;
- for (auto & i : goal->waiters) {
- if (GoalPtr waiting = i.lock()) {
- assert(waiting->waitees.count(goal));
- waiting->waitees.erase(goal);
-
- waiting->trace(fmt("waitee '%s' done; %d left", goal->name, waiting->waitees.size()));
-
- if (f.exitCode != Goal::ecSuccess) ++waiting->nrFailed;
- if (f.exitCode == Goal::ecNoSubstituters) ++waiting->nrNoSubstituters;
- if (f.exitCode == Goal::ecIncompleteClosure) ++waiting->nrIncompleteClosure;
-
- if (waiting->waitees.empty() || (f.exitCode == Goal::ecFailed && !settings.keepGoing)) {
- /* If we failed and keepGoing is not set, we remove all
- remaining waitees. */
- for (auto & i : waiting->waitees) {
- i->waiters.extract(waiting);
- }
- waiting->waitees.clear();
-
- wakeUp(waiting);
- }
-
- waiting->waiteeDone(goal);
- }
- }
- goal->waiters.clear();
removeGoal(goal);
+ goal->notify->fulfill();
goal->cleanup();
}
@@ -213,12 +202,6 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how)
overloaded{
[&](Goal::StillAlive) {},
[&](Goal::ContinueImmediately) { wakeUp(goal); },
- [&](Goal::WaitForGoals & w) {
- for (auto & dep : w.goals) {
- goal->waitees.insert(dep);
- dep->waiters.insert(goal);
- }
- },
[&](Goal::WaitForWorld & w) {
childStarted(goal, w.promise.then([](auto r) -> Result<Goal::WorkResult> {
if (r.has_value()) {
@@ -310,7 +293,7 @@ void Worker::updateStatistics()
}
}
-Goals Worker::run(std::function<Goals (GoalFactory &)> req)
+std::vector<GoalPtr> Worker::run(std::function<Targets (GoalFactory &)> req)
{
auto _topGoals = req(goalFactory());
@@ -320,7 +303,10 @@ Goals Worker::run(std::function<Goals (GoalFactory &)> req)
updateStatistics();
- topGoals = _topGoals;
+ topGoals.clear();
+ for (auto & [goal, _promise] : _topGoals) {
+ topGoals.insert(goal);
+ }
debug("entered goal loop");
@@ -374,7 +360,11 @@ Goals Worker::run(std::function<Goals (GoalFactory &)> req)
assert(!settings.keepGoing || awake.empty());
assert(!settings.keepGoing || children.isEmpty());
- return _topGoals;
+ std::vector<GoalPtr> results;
+ for (auto & [i, _p] : _topGoals) {
+ results.push_back(i);
+ }
+ return results;
}
void Worker::waitForInput()
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 834ecfda3..925d289bf 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -28,10 +28,10 @@ struct HookInstance;
class GoalFactory
{
public:
- virtual std::shared_ptr<DerivationGoal> makeDerivationGoal(
+ virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoal(
const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal
) = 0;
- virtual std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+ virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeBasicDerivationGoal(
const StorePath & drvPath,
const BasicDerivation & drv,
const OutputsSpec & wantedOutputs,
@@ -41,12 +41,14 @@ public:
/**
* @ref SubstitutionGoal "substitution goal"
*/
- virtual std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(
+ virtual std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
+ makePathSubstitutionGoal(
const StorePath & storePath,
RepairFlag repair = NoRepair,
std::optional<ContentAddress> ca = std::nullopt
) = 0;
- virtual std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(
+ virtual std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+ makeDrvOutputSubstitutionGoal(
const DrvOutput & id,
RepairFlag repair = NoRepair,
std::optional<ContentAddress> ca = std::nullopt
@@ -58,7 +60,8 @@ public:
* It will be a `DerivationGoal` for a `DerivedPath::Built` or
* a `SubstitutionGoal` for a `DerivedPath::Opaque`.
*/
- virtual GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0;
+ virtual std::pair<GoalPtr, kj::Promise<void>>
+ makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0;
};
// elaborate hoax to let goals access factory methods while hiding them from the public
@@ -94,13 +97,27 @@ private:
*/
WeakGoals awake;
+ template<typename G>
+ struct CachedGoal
+ {
+ std::weak_ptr<G> goal;
+ kj::Own<kj::ForkedPromise<void>> promise;
+ kj::Own<kj::PromiseFulfiller<void>> fulfiller;
+
+ CachedGoal()
+ {
+ auto pf = kj::newPromiseAndFulfiller<void>();
+ promise = kj::heap(pf.promise.fork());
+ fulfiller = std::move(pf.fulfiller);
+ }
+ };
/**
* Maps used to prevent multiple instantiations of a goal for the
* same derivation / path.
*/
- std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
- std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
- std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
+ std::map<StorePath, CachedGoal<DerivationGoal>> derivationGoals;
+ std::map<StorePath, CachedGoal<PathSubstitutionGoal>> substitutionGoals;
+ std::map<DrvOutput, CachedGoal<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
/**
* Cache for pathContentsGood().
@@ -226,21 +243,31 @@ public:
* @ref DerivationGoal "derivation goal"
*/
private:
- std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
+ std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoalCommon(
const StorePath & drvPath, const OutputsSpec & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
- std::shared_ptr<DerivationGoal> makeDerivationGoal(
+ std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoal(
const StorePath & drvPath,
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override;
- std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+ std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeBasicDerivationGoal(
const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override;
/**
* @ref SubstitutionGoal "substitution goal"
*/
- std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override;
- std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override;
+ std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
+ makePathSubstitutionGoal(
+ const StorePath & storePath,
+ RepairFlag repair = NoRepair,
+ std::optional<ContentAddress> ca = std::nullopt
+ ) override;
+ std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+ makeDrvOutputSubstitutionGoal(
+ const DrvOutput & id,
+ RepairFlag repair = NoRepair,
+ std::optional<ContentAddress> ca = std::nullopt
+ ) override;
/**
* Make a goal corresponding to the `DerivedPath`.
@@ -248,13 +275,16 @@ private:
* It will be a `DerivationGoal` for a `DerivedPath::Built` or
* a `SubstitutionGoal` for a `DerivedPath::Opaque`.
*/
- GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override;
+ std::pair<GoalPtr, kj::Promise<void>>
+ makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override;
public:
+ using Targets = std::map<GoalPtr, kj::Promise<void>>;
+
/**
* Loop until the specified top-level goals have finished.
*/
- Goals run(std::function<Goals (GoalFactory &)> req);
+ std::vector<GoalPtr> run(std::function<Targets (GoalFactory &)> req);
/***
* The exit status in case of failure.
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 00:58:27 +0000