libstore: turn Goal::WaitForGoals into a promise

also gets rid of explicit strong references to dependencies of any goal,
and weak references to dependers as well. those are now only held within
promises representing goal completion and thus independent of the goal's
relation to each other. the weak references to dependers was only needed
for notifications, and that's much better handled entirely by kj itself.

Change-Id: I00d06df9090f8d6336ee4bb0c1313a7052fb016b

8 files changed, 192 insertions, 128 deletions

diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 3c4257f08..b8c4d278d 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -13,8 +13,11 @@
 
 #include <boost/outcome/try.hpp>
 #include <fstream>
+#include <kj/array.h>
 #include <kj/async-unix.h>
+#include <kj/async.h>
 #include <kj/debug.h>
+#include <kj/vector.h>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <sys/un.h>
@@ -173,7 +176,7 @@ try {
 
 
     state = &DerivationGoal::loadDerivation;
-    return {WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}};
+    return waitForGoals(worker.goalFactory().makePathSubstitutionGoal(drvPath));
 } catch (...) {
     return {std::current_exception()};
 }
@@ -272,13 +275,13 @@ try {
     /* We are first going to try to create the invalid output paths
        through substitutes.  If that doesn't work, we'll build
        them. */
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     if (settings.useSubstitutes) {
         if (parsedDrv->substitutesAllowed()) {
             for (auto & [outputName, status] : initialOutputs) {
                 if (!status.wanted) continue;
                 if (!status.known)
-                    result.goals.insert(
+                    dependencies.add(
                         worker.goalFactory().makeDrvOutputSubstitutionGoal(
                             DrvOutput{status.outputHash, outputName},
                             buildMode == bmRepair ? Repair : NoRepair
@@ -286,7 +289,7 @@ try {
                     );
                 else {
                     auto * cap = getDerivationCA(*drv);
-                    result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(
+                    dependencies.add(worker.goalFactory().makePathSubstitutionGoal(
                         status.known->path,
                         buildMode == bmRepair ? Repair : NoRepair,
                         cap ? std::optional { *cap } : std::nullopt));
@@ -297,11 +300,11 @@ try {
         }
     }
 
-    if (result.goals.empty()) { /* to prevent hang (no wake-up event) */
+    if (dependencies.empty()) { /* to prevent hang (no wake-up event) */
         return outputsSubstitutionTried(inBuildSlot);
     } else {
         state = &DerivationGoal::outputsSubstitutionTried;
-        return {std::move(result)};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};
@@ -383,7 +386,7 @@ try {
    produced using a substitute.  So we have to build instead. */
 kj::Promise<Result<Goal::WorkResult>> DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) noexcept
 try {
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
 
     /* At this point we are building all outputs, so if more are wanted there
        is no need to restart. */
@@ -396,7 +399,7 @@ try {
 
         addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
             if (!inputNode.value.empty())
-                result.goals.insert(worker.goalFactory().makeGoal(
+                dependencies.add(worker.goalFactory().makeGoal(
                     DerivedPath::Built {
                         .drvPath = inputDrv,
                         .outputs = inputNode.value,
@@ -441,14 +444,14 @@ try {
         if (!settings.useSubstitutes)
             throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
                 worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
-        result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i));
+        dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i));
     }
 
-    if (result.goals.empty()) {/* to prevent hang (no wake-up event) */
+    if (dependencies.empty()) {/* to prevent hang (no wake-up event) */
         return inputsRealised(inBuildSlot);
     } else {
         state = &DerivationGoal::inputsRealised;
-        return {result};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};
@@ -491,7 +494,7 @@ try {
         }
 
     /* Check each path (slow!). */
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     for (auto & i : outputClosure) {
         if (worker.pathContentsGood(i)) continue;
         printError(
@@ -499,9 +502,9 @@ try {
             worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
         auto drvPath2 = outputsToDrv.find(i);
         if (drvPath2 == outputsToDrv.end())
-            result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i, Repair));
+            dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i, Repair));
         else
-            result.goals.insert(worker.goalFactory().makeGoal(
+            dependencies.add(worker.goalFactory().makeGoal(
                 DerivedPath::Built {
                     .drvPath = makeConstantStorePathRef(drvPath2->second),
                     .outputs = OutputsSpec::All { },
@@ -509,12 +512,12 @@ try {
                 bmRepair));
     }
 
-    if (result.goals.empty()) {
+    if (dependencies.empty()) {
         return {done(BuildResult::AlreadyValid, assertPathValidity())};
     }
 
     state = &DerivationGoal::closureRepaired;
-    return {result};
+    return waitForGoals(dependencies.releaseAsArray());
 } catch (...) {
     return {std::current_exception()};
 }
@@ -614,11 +617,12 @@ try {
                        worker.store.printStorePath(pathResolved),
                    });
 
-            resolvedDrvGoal = worker.goalFactory().makeDerivationGoal(
+            auto dependency = worker.goalFactory().makeDerivationGoal(
                 pathResolved, wantedOutputs, buildMode);
+            resolvedDrvGoal = dependency.first;
 
             state = &DerivationGoal::resolvedFinished;
-            return {WaitForGoals{{resolvedDrvGoal}}};
+            return waitForGoals(std::move(dependency));
         }
 
         std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc
index 6ef00d1ff..80b2c4cfb 100644
--- a/src/libstore/build/drv-output-substitution-goal.cc
+++ b/src/libstore/build/drv-output-substitution-goal.cc
@@ -4,6 +4,9 @@
 #include "worker.hh"
 #include "substitution-goal.hh"
 #include "signals.hh"
+#include <kj/array.h>
+#include <kj/async.h>
+#include <kj/vector.h>
 
 namespace nix {
 
@@ -106,7 +109,7 @@ try {
         return tryNext(inBuildSlot);
     }
 
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
         if (depId != id) {
             if (auto localOutputInfo = worker.store.queryRealisation(depId);
@@ -122,17 +125,17 @@ try {
                 );
                 return tryNext(inBuildSlot);
             }
-            result.goals.insert(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId));
+            dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId));
         }
     }
 
-    result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath));
+    dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath));
 
-    if (result.goals.empty()) {
+    if (dependencies.empty()) {
         return outPathValid(inBuildSlot);
     } else {
         state = &DrvOutputSubstitutionGoal::outPathValid;
-        return {std::move(result)};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index a0f18a02c..27c341295 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -17,9 +17,9 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod
     Worker worker(*this, evalStore ? *evalStore : *this, aio);
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        Goals goals;
+        Worker::Targets goals;
         for (auto & br : reqs)
-            goals.insert(gf.makeGoal(br, buildMode));
+            goals.emplace(gf.makeGoal(br, buildMode));
         return goals;
     });
 
@@ -60,11 +60,11 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults(
     std::vector<std::pair<const DerivedPath &, GoalPtr>> state;
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        Goals goals;
+        Worker::Targets goals;
         for (const auto & req : reqs) {
             auto goal = gf.makeGoal(req, buildMode);
-            goals.insert(goal);
-            state.push_back({req, goal});
+            state.push_back({req, goal.first});
+            goals.emplace(std::move(goal));
         }
         return goals;
     });
@@ -84,8 +84,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
     Worker worker(*this, *this, aio);
 
     try {
-        auto goals = runWorker(worker, [&](GoalFactory & gf) -> Goals {
-            return Goals{gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)};
+        auto goals = runWorker(worker, [&](GoalFactory & gf) {
+            Worker::Targets goals;
+            goals.emplace(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode));
+            return goals;
         });
         auto goal = *goals.begin();
         return goal->buildResult.restrictTo(DerivedPath::Built {
@@ -110,7 +112,9 @@ void Store::ensurePath(const StorePath & path)
     Worker worker(*this, *this, aio);
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        return Goals{gf.makePathSubstitutionGoal(path)};
+        Worker::Targets goals;
+        goals.emplace(gf.makePathSubstitutionGoal(path));
+        return goals;
     });
     auto goal = *goals.begin();
 
@@ -130,7 +134,9 @@ void Store::repairPath(const StorePath & path)
     Worker worker(*this, *this, aio);
 
     auto goals = runWorker(worker, [&](GoalFactory & gf) {
-        return Goals{gf.makePathSubstitutionGoal(path, Repair)};
+        Worker::Targets goals;
+        goals.emplace(gf.makePathSubstitutionGoal(path, Repair));
+        return goals;
     });
     auto goal = *goals.begin();
 
@@ -140,14 +146,16 @@ void Store::repairPath(const StorePath & path)
         auto info = queryPathInfo(path);
         if (info->deriver && isValidPath(*info->deriver)) {
             worker.run([&](GoalFactory & gf) {
-                return Goals{gf.makeGoal(
+                Worker::Targets goals;
+                goals.emplace(gf.makeGoal(
                     DerivedPath::Built{
                         .drvPath = makeConstantStorePathRef(*info->deriver),
                         // FIXME: Should just build the specific output we need.
                         .outputs = OutputsSpec::All{},
                     },
                     bmRepair
-                )};
+                ));
+                return goals;
             });
         } else
             throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path));
diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
index 649093dbd..8a2f4ab35 100644
--- a/src/libstore/build/goal.cc
+++ b/src/libstore/build/goal.cc
@@ -1,4 +1,5 @@
 #include "goal.hh"
+#include "async-collect.hh"
 #include "worker.hh"
 #include <kj/time.h>
 
@@ -28,4 +29,32 @@ try {
     co_return std::current_exception();
 }
 
+kj::Promise<Result<Goal::WorkResult>>
+Goal::waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<void>>> dependencies) noexcept
+try {
+    auto left = dependencies.size();
+    auto collectDeps = asyncCollect(std::move(dependencies));
+
+    while (auto item = co_await collectDeps.next()) {
+        left--;
+        auto & dep = *item;
+
+        trace(fmt("waitee '%s' done; %d left", dep->name, left));
+
+        if (dep->exitCode != Goal::ecSuccess) ++nrFailed;
+        if (dep->exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters;
+        if (dep->exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure;
+
+        waiteeDone(dep);
+
+        if (dep->exitCode == ecFailed && !settings.keepGoing) {
+            co_return result::success(ContinueImmediately{});
+        }
+    }
+
+    co_return result::success(ContinueImmediately{});
+} catch (...) {
+    co_return result::failure(std::current_exception());
+}
+
 }
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index 1ccf9716b..e7a500a00 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -6,6 +6,7 @@
 #include "types.hh"
 #include "store-api.hh"
 #include "build-result.hh"
+#include <concepts> // IWYU pragma: keep
 #include <kj/async.h>
 
 namespace nix {
@@ -71,17 +72,6 @@ struct Goal
     const bool isDependency;
 
     /**
-     * Goals that this goal is waiting for.
-     */
-    Goals waitees;
-
-    /**
-     * Goals waiting for this one to finish.  Must use weak pointers
-     * here to prevent cycles.
-     */
-    WeakGoals waiters;
-
-    /**
      * Number of goals we are/were waiting for that have failed.
      */
     size_t nrFailed = 0;
@@ -113,6 +103,9 @@ struct Goal
      */
     BuildResult buildResult;
 
+    // for use by Worker only. will go away once work() is a promise.
+    kj::Own<kj::PromiseFulfiller<void>> notify;
+
 protected:
     AsyncSemaphore::Token slotToken;
 
@@ -122,9 +115,6 @@ public:
 
     struct [[nodiscard]] StillAlive {};
     struct [[nodiscard]] ContinueImmediately {};
-    struct [[nodiscard]] WaitForGoals {
-        Goals goals;
-    };
     struct [[nodiscard]] WaitForWorld {
         kj::Promise<Outcome<void, Finished>> promise;
     };
@@ -141,7 +131,6 @@ public:
     struct [[nodiscard]] WorkResult : std::variant<
                                           StillAlive,
                                           ContinueImmediately,
-                                          WaitForGoals,
                                           WaitForWorld,
                                           Finished>
     {
@@ -151,6 +140,15 @@ public:
 
 protected:
     kj::Promise<Result<WorkResult>> waitForAWhile();
+    kj::Promise<Result<WorkResult>>
+    waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<void>>> dependencies) noexcept;
+
+    template<std::derived_from<Goal>... G>
+    kj::Promise<Result<Goal::WorkResult>>
+    waitForGoals(std::pair<std::shared_ptr<G>, kj::Promise<void>>... goals) noexcept
+    {
+        return waitForGoals(kj::arrOf<std::pair<GoalPtr, kj::Promise<void>>>(std::move(goals)...));
+    }
 
 public:
 
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index 6d90196fa..74a63ca21 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -3,6 +3,8 @@
 #include "nar-info.hh"
 #include "signals.hh"
 #include "finally.hh"
+#include <kj/array.h>
+#include <kj/vector.h>
 
 namespace nix {
 
@@ -160,16 +162,16 @@ try {
 
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
-    WaitForGoals result;
+    kj::Vector<std::pair<GoalPtr, kj::Promise<void>>> dependencies;
     for (auto & i : info->references)
         if (i != storePath) /* ignore self-references */
-            result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i));
+            dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i));
 
-    if (result.goals.empty()) {/* to prevent hang (no wake-up event) */
+    if (dependencies.empty()) {/* to prevent hang (no wake-up event) */
         return referencesValid(inBuildSlot);
     } else {
         state = &PathSubstitutionGoal::referencesValid;
-        return {std::move(result)};
+        return waitForGoals(dependencies.releaseAsArray());
     }
 } catch (...) {
     return {std::current_exception()};
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index e19917d91..68071a94c 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -52,26 +52,28 @@ Worker::~Worker()
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeDerivationGoalCommon(
     const StorePath & drvPath,
     const OutputsSpec & wantedOutputs,
     std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
 {
-    std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
-    std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
+    auto & goal_weak = derivationGoals[drvPath];
+    std::shared_ptr<DerivationGoal> goal = goal_weak.goal.lock();
     if (!goal) {
         goal = mkDrvGoal();
-        goal_weak = goal;
+        goal->notify = std::move(goal_weak.fulfiller);
+        goal_weak.goal = goal;
         wakeUp(goal);
     } else {
         goal->addWantedOutputs(wantedOutputs);
     }
-    return goal;
+    return {goal, goal_weak.promise->addBranch()};
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
-    const OutputsSpec & wantedOutputs, BuildMode buildMode)
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeDerivationGoal(
+    const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode
+)
 {
     return makeDerivationGoalCommon(
         drvPath,
@@ -89,8 +91,12 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
 }
 
 
-std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
-    const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode)
+std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> Worker::makeBasicDerivationGoal(
+    const StorePath & drvPath,
+    const BasicDerivation & drv,
+    const OutputsSpec & wantedOutputs,
+    BuildMode buildMode
+)
 {
     return makeDerivationGoalCommon(
         drvPath,
@@ -108,55 +114,63 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath
 }
 
 
-std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
+std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
+Worker::makePathSubstitutionGoal(
+    const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca
+)
 {
-    std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
-    auto goal = goal_weak.lock(); // FIXME
+    auto & goal_weak = substitutionGoals[path];
+    auto goal = goal_weak.goal.lock(); // FIXME
     if (!goal) {
         goal = std::make_shared<PathSubstitutionGoal>(path, *this, running, repair, ca);
-        goal_weak = goal;
+        goal->notify = std::move(goal_weak.fulfiller);
+        goal_weak.goal = goal;
         wakeUp(goal);
     }
-    return goal;
+    return {goal, goal_weak.promise->addBranch()};
 }
 
 
-std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
+std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+Worker::makeDrvOutputSubstitutionGoal(
+    const DrvOutput & id, RepairFlag repair, std::optional<ContentAddress> ca
+)
 {
-    std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
-    auto goal = goal_weak.lock(); // FIXME
+    auto & goal_weak = drvOutputSubstitutionGoals[id];
+    auto goal = goal_weak.goal.lock(); // FIXME
     if (!goal) {
         goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, running, repair, ca);
-        goal_weak = goal;
+        goal->notify = std::move(goal_weak.fulfiller);
+        goal_weak.goal = goal;
         wakeUp(goal);
     }
-    return goal;
+    return {goal, goal_weak.promise->addBranch()};
 }
 
 
-GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
+std::pair<GoalPtr, kj::Promise<void>> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
 {
     return std::visit(overloaded {
-        [&](const DerivedPath::Built & bfd) -> GoalPtr {
+        [&](const DerivedPath::Built & bfd) -> std::pair<GoalPtr, kj::Promise<void>> {
             if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
                 return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
             else
                 throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
         },
-        [&](const DerivedPath::Opaque & bo) -> GoalPtr {
+        [&](const DerivedPath::Opaque & bo) -> std::pair<GoalPtr, kj::Promise<void>> {
             return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
         },
     }, req.raw());
 }
 
 
-template<typename K, typename G>
-static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
+template<typename G>
+static void removeGoal(std::shared_ptr<G> goal, auto & goalMap)
 {
     /* !!! inefficient */
     for (auto i = goalMap.begin();
          i != goalMap.end(); )
-        if (i->second.lock() == goal) {
+        if (i->second.goal.lock() == goal) {
             auto j = i; ++j;
             goalMap.erase(i);
             i = j;
@@ -177,33 +191,8 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f)
     hashMismatch |= f.hashMismatch;
     checkMismatch |= f.checkMismatch;
 
-    for (auto & i : goal->waiters) {
-        if (GoalPtr waiting = i.lock()) {
-            assert(waiting->waitees.count(goal));
-            waiting->waitees.erase(goal);
-
-            waiting->trace(fmt("waitee '%s' done; %d left", goal->name, waiting->waitees.size()));
-
-            if (f.exitCode != Goal::ecSuccess) ++waiting->nrFailed;
-            if (f.exitCode == Goal::ecNoSubstituters) ++waiting->nrNoSubstituters;
-            if (f.exitCode == Goal::ecIncompleteClosure) ++waiting->nrIncompleteClosure;
-
-            if (waiting->waitees.empty() || (f.exitCode == Goal::ecFailed && !settings.keepGoing)) {
-                /* If we failed and keepGoing is not set, we remove all
-                   remaining waitees. */
-                for (auto & i : waiting->waitees) {
-                    i->waiters.extract(waiting);
-                }
-                waiting->waitees.clear();
-
-                wakeUp(waiting);
-            }
-
-            waiting->waiteeDone(goal);
-        }
-    }
-    goal->waiters.clear();
     removeGoal(goal);
+    goal->notify->fulfill();
     goal->cleanup();
 }
 
@@ -213,12 +202,6 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how)
         overloaded{
             [&](Goal::StillAlive) {},
             [&](Goal::ContinueImmediately) { wakeUp(goal); },
-            [&](Goal::WaitForGoals & w) {
-                for (auto & dep : w.goals) {
-                    goal->waitees.insert(dep);
-                    dep->waiters.insert(goal);
-                }
-            },
             [&](Goal::WaitForWorld & w) {
                 childStarted(goal, w.promise.then([](auto r) -> Result<Goal::WorkResult> {
                     if (r.has_value()) {
@@ -310,7 +293,7 @@ void Worker::updateStatistics()
     }
 }
 
-Goals Worker::run(std::function<Goals (GoalFactory &)> req)
+std::vector<GoalPtr> Worker::run(std::function<Targets (GoalFactory &)> req)
 {
     auto _topGoals = req(goalFactory());
 
@@ -320,7 +303,10 @@ Goals Worker::run(std::function<Goals (GoalFactory &)> req)
 
     updateStatistics();
 
-    topGoals = _topGoals;
+    topGoals.clear();
+    for (auto & [goal, _promise] : _topGoals) {
+        topGoals.insert(goal);
+    }
 
     debug("entered goal loop");
 
@@ -374,7 +360,11 @@ Goals Worker::run(std::function<Goals (GoalFactory &)> req)
     assert(!settings.keepGoing || awake.empty());
     assert(!settings.keepGoing || children.isEmpty());
 
-    return _topGoals;
+    std::vector<GoalPtr> results;
+    for (auto & [i, _p] : _topGoals) {
+        results.push_back(i);
+    }
+    return results;
 }
 
 void Worker::waitForInput()
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 834ecfda3..925d289bf 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -28,10 +28,10 @@ struct HookInstance;
 class GoalFactory
 {
 public:
-    virtual std::shared_ptr<DerivationGoal> makeDerivationGoal(
+    virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoal(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal
     ) = 0;
-    virtual std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+    virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeBasicDerivationGoal(
         const StorePath & drvPath,
         const BasicDerivation & drv,
         const OutputsSpec & wantedOutputs,
@@ -41,12 +41,14 @@ public:
     /**
      * @ref SubstitutionGoal "substitution goal"
      */
-    virtual std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(
+    virtual std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
+    makePathSubstitutionGoal(
         const StorePath & storePath,
         RepairFlag repair = NoRepair,
         std::optional<ContentAddress> ca = std::nullopt
     ) = 0;
-    virtual std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(
+    virtual std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+    makeDrvOutputSubstitutionGoal(
         const DrvOutput & id,
         RepairFlag repair = NoRepair,
         std::optional<ContentAddress> ca = std::nullopt
@@ -58,7 +60,8 @@ public:
      * It will be a `DerivationGoal` for a `DerivedPath::Built` or
      * a `SubstitutionGoal` for a `DerivedPath::Opaque`.
      */
-    virtual GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0;
+    virtual std::pair<GoalPtr, kj::Promise<void>>
+    makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0;
 };
 
 // elaborate hoax to let goals access factory methods while hiding them from the public
@@ -94,13 +97,27 @@ private:
      */
     WeakGoals awake;
 
+    template<typename G>
+    struct CachedGoal
+    {
+        std::weak_ptr<G> goal;
+        kj::Own<kj::ForkedPromise<void>> promise;
+        kj::Own<kj::PromiseFulfiller<void>> fulfiller;
+
+        CachedGoal()
+        {
+            auto pf = kj::newPromiseAndFulfiller<void>();
+            promise = kj::heap(pf.promise.fork());
+            fulfiller = std::move(pf.fulfiller);
+        }
+    };
     /**
      * Maps used to prevent multiple instantiations of a goal for the
      * same derivation / path.
      */
-    std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
-    std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
-    std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
+    std::map<StorePath, CachedGoal<DerivationGoal>> derivationGoals;
+    std::map<StorePath, CachedGoal<PathSubstitutionGoal>> substitutionGoals;
+    std::map<DrvOutput, CachedGoal<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
 
     /**
      * Cache for pathContentsGood().
@@ -226,21 +243,31 @@ public:
      * @ref DerivationGoal "derivation goal"
      */
 private:
-    std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
+    std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoalCommon(
         const StorePath & drvPath, const OutputsSpec & wantedOutputs,
         std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
-    std::shared_ptr<DerivationGoal> makeDerivationGoal(
+    std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeDerivationGoal(
         const StorePath & drvPath,
         const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override;
-    std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+    std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<void>> makeBasicDerivationGoal(
         const StorePath & drvPath, const BasicDerivation & drv,
         const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override;
 
     /**
      * @ref SubstitutionGoal "substitution goal"
      */
-    std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override;
-    std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override;
+    std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<void>>
+    makePathSubstitutionGoal(
+        const StorePath & storePath,
+        RepairFlag repair = NoRepair,
+        std::optional<ContentAddress> ca = std::nullopt
+    ) override;
+    std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<void>>
+    makeDrvOutputSubstitutionGoal(
+        const DrvOutput & id,
+        RepairFlag repair = NoRepair,
+        std::optional<ContentAddress> ca = std::nullopt
+    ) override;
 
     /**
      * Make a goal corresponding to the `DerivedPath`.
@@ -248,13 +275,16 @@ private:
      * It will be a `DerivationGoal` for a `DerivedPath::Built` or
      * a `SubstitutionGoal` for a `DerivedPath::Opaque`.
      */
-    GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override;
+    std::pair<GoalPtr, kj::Promise<void>>
+    makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override;
 
 public:
+    using Targets = std::map<GoalPtr, kj::Promise<void>>;
+
     /**
      * Loop until the specified top-level goals have finished.
      */
-    Goals run(std::function<Goals (GoalFactory &)> req);
+    std::vector<GoalPtr> run(std::function<Targets (GoalFactory &)> req);
 
     /***
      * The exit status in case of failure.