aboutsummaryrefslogtreecommitdiff
path: root/src/libstore/build
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore/build')
-rw-r--r--src/libstore/build/derivation-goal.cc54
-rw-r--r--src/libstore/build/derivation-goal.hh3
-rw-r--r--src/libstore/build/drv-output-substitution-goal.cc12
-rw-r--r--src/libstore/build/goal.cc7
-rw-r--r--src/libstore/build/goal.hh20
-rw-r--r--src/libstore/build/local-derivation-goal.cc15
-rw-r--r--src/libstore/build/substitution-goal.cc16
-rw-r--r--src/libstore/build/worker.cc20
-rw-r--r--src/libstore/build/worker.hh74
9 files changed, 118 insertions, 103 deletions
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 17a2b04f1..aa89f9e7d 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -174,10 +174,9 @@ Goal::WorkResult DerivationGoal::getDerivation()
return loadDerivation();
}
- addWaitee(worker.makePathSubstitutionGoal(drvPath));
state = &DerivationGoal::loadDerivation;
- return StillAlive{};
+ return WaitForGoals{{worker.makePathSubstitutionGoal(drvPath)}};
}
@@ -268,11 +267,12 @@ Goal::WorkResult DerivationGoal::haveDerivation()
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
+ WaitForGoals result;
if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
for (auto & [outputName, status] : initialOutputs) {
if (!status.wanted) continue;
if (!status.known)
- addWaitee(
+ result.goals.insert(
worker.makeDrvOutputSubstitutionGoal(
DrvOutput{status.outputHash, outputName},
buildMode == bmRepair ? Repair : NoRepair
@@ -280,18 +280,18 @@ Goal::WorkResult DerivationGoal::haveDerivation()
);
else {
auto * cap = getDerivationCA(*drv);
- addWaitee(worker.makePathSubstitutionGoal(
+ result.goals.insert(worker.makePathSubstitutionGoal(
status.known->path,
buildMode == bmRepair ? Repair : NoRepair,
cap ? std::optional { *cap } : std::nullopt));
}
}
- if (waitees.empty()) { /* to prevent hang (no wake-up event) */
+ if (result.goals.empty()) { /* to prevent hang (no wake-up event) */
return outputsSubstitutionTried();
} else {
state = &DerivationGoal::outputsSubstitutionTried;
- return StillAlive{};
+ return result;
}
}
@@ -362,6 +362,8 @@ Goal::WorkResult DerivationGoal::outputsSubstitutionTried()
produced using a substitute. So we have to build instead. */
Goal::WorkResult DerivationGoal::gaveUpOnSubstitution()
{
+ WaitForGoals result;
+
/* At this point we are building all outputs, so if more are wanted there
is no need to restart. */
needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed;
@@ -373,7 +375,7 @@ Goal::WorkResult DerivationGoal::gaveUpOnSubstitution()
addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) {
if (!inputNode.value.empty())
- addWaitee(worker.makeGoal(
+ result.goals.insert(worker.makeGoal(
DerivedPath::Built {
.drvPath = inputDrv,
.outputs = inputNode.value,
@@ -418,14 +420,14 @@ Goal::WorkResult DerivationGoal::gaveUpOnSubstitution()
if (!settings.useSubstitutes)
throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
- addWaitee(worker.makePathSubstitutionGoal(i));
+ result.goals.insert(worker.makePathSubstitutionGoal(i));
}
- if (waitees.empty()) {/* to prevent hang (no wake-up event) */
+ if (result.goals.empty()) {/* to prevent hang (no wake-up event) */
return inputsRealised();
} else {
state = &DerivationGoal::inputsRealised;
- return StillAlive{};
+ return result;
}
}
@@ -466,6 +468,7 @@ Goal::WorkResult DerivationGoal::repairClosure()
}
/* Check each path (slow!). */
+ WaitForGoals result;
for (auto & i : outputClosure) {
if (worker.pathContentsGood(i)) continue;
printError(
@@ -473,9 +476,9 @@ Goal::WorkResult DerivationGoal::repairClosure()
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
auto drvPath2 = outputsToDrv.find(i);
if (drvPath2 == outputsToDrv.end())
- addWaitee(worker.makePathSubstitutionGoal(i, Repair));
+ result.goals.insert(worker.makePathSubstitutionGoal(i, Repair));
else
- addWaitee(worker.makeGoal(
+ result.goals.insert(worker.makeGoal(
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(drvPath2->second),
.outputs = OutputsSpec::All { },
@@ -483,12 +486,12 @@ Goal::WorkResult DerivationGoal::repairClosure()
bmRepair));
}
- if (waitees.empty()) {
+ if (result.goals.empty()) {
return done(BuildResult::AlreadyValid, assertPathValidity());
}
state = &DerivationGoal::closureRepaired;
- return StillAlive{};
+ return result;
}
@@ -580,10 +583,9 @@ Goal::WorkResult DerivationGoal::inputsRealised()
resolvedDrvGoal = worker.makeDerivationGoal(
pathResolved, wantedOutputs, buildMode);
- addWaitee(resolvedDrvGoal);
state = &DerivationGoal::resolvedFinished;
- return StillAlive{};
+ return WaitForGoals{{resolvedDrvGoal}};
}
std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
@@ -648,8 +650,7 @@ Goal::WorkResult DerivationGoal::inputsRealised()
slot to become available, since we don't need one if there is a
build hook. */
state = &DerivationGoal::tryToBuild;
- worker.wakeUp(shared_from_this());
- return StillAlive{};
+ return ContinueImmediately{};
}
Goal::WorkResult DerivationGoal::started()
@@ -701,8 +702,7 @@ Goal::WorkResult DerivationGoal::tryToBuild()
if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for lock on %s", Magenta(showPaths(lockFiles))));
- worker.waitForAWhile(shared_from_this());
- return StillAlive{};
+ return WaitForAWhile{};
}
actLock.reset();
@@ -753,9 +753,8 @@ Goal::WorkResult DerivationGoal::tryToBuild()
if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlTalkative, actBuildWaiting,
fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath))));
- worker.waitForAWhile(shared_from_this());
outputLocks.unlock();
- return StillAlive{};
+ return WaitForAWhile{};
case rpDecline:
/* We should do it ourselves. */
break;
@@ -765,8 +764,7 @@ Goal::WorkResult DerivationGoal::tryToBuild()
actLock.reset();
state = &DerivationGoal::tryLocalBuild;
- worker.wakeUp(shared_from_this());
- return StillAlive{};
+ return ContinueImmediately{};
}
Goal::WorkResult DerivationGoal::tryLocalBuild() {
@@ -1508,10 +1506,6 @@ Goal::Finished DerivationGoal::done(
buildResult.status = status;
if (ex)
buildResult.errorMsg = fmt("%s", Uncolored(ex->info().msg));
- if (buildResult.status == BuildResult::TimedOut)
- worker.timedOut = true;
- if (buildResult.status == BuildResult::PermanentFailure)
- worker.permanentFailure = true;
mcExpectedBuilds.reset();
mcRunningBuilds.reset();
@@ -1537,6 +1531,10 @@ Goal::Finished DerivationGoal::done(
return Finished{
.result = buildResult.success() ? ecSuccess : ecFailed,
.ex = ex ? std::make_unique<Error>(std::move(*ex)) : nullptr,
+ .permanentFailure = buildResult.status == BuildResult::PermanentFailure,
+ .timedOut = buildResult.status == BuildResult::TimedOut,
+ .hashMismatch = anyHashMismatchSeen,
+ .checkMismatch = anyCheckMismatchSeen,
};
}
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index c43e2aed5..257282308 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -107,6 +107,9 @@ struct DerivationGoal : public Goal
*/
NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed;
+ bool anyHashMismatchSeen = false;
+ bool anyCheckMismatchSeen = false;
+
/**
* See `retrySubstitution`; just for that field.
*/
diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc
index 62e86e1cc..b41dae5d6 100644
--- a/src/libstore/build/drv-output-substitution-goal.cc
+++ b/src/libstore/build/drv-output-substitution-goal.cc
@@ -41,8 +41,7 @@ Goal::WorkResult DrvOutputSubstitutionGoal::tryNext()
if maxSubstitutionJobs == 0, we still allow a substituter to run. This
prevents infinite waiting. */
if (worker.runningSubstitutions >= std::max(1U, settings.maxSubstitutionJobs.get())) {
- worker.waitForBuildSlot(shared_from_this());
- return StillAlive{};
+ return WaitForSlot{};
}
maintainRunningSubstitutions =
@@ -101,6 +100,7 @@ Goal::WorkResult DrvOutputSubstitutionGoal::realisationFetched()
return tryNext();
}
+ WaitForGoals result;
for (const auto & [depId, depPath] : outputInfo->dependentRealisations) {
if (depId != id) {
if (auto localOutputInfo = worker.store.queryRealisation(depId);
@@ -116,17 +116,17 @@ Goal::WorkResult DrvOutputSubstitutionGoal::realisationFetched()
);
return tryNext();
}
- addWaitee(worker.makeDrvOutputSubstitutionGoal(depId));
+ result.goals.insert(worker.makeDrvOutputSubstitutionGoal(depId));
}
}
- addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
+ result.goals.insert(worker.makePathSubstitutionGoal(outputInfo->outPath));
- if (waitees.empty()) {
+ if (result.goals.empty()) {
return outPathValid();
} else {
state = &DrvOutputSubstitutionGoal::outPathValid;
- return StillAlive{};
+ return result;
}
}
diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
index f26c2c671..a4e66d989 100644
--- a/src/libstore/build/goal.cc
+++ b/src/libstore/build/goal.cc
@@ -11,13 +11,6 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const {
}
-void Goal::addWaitee(GoalPtr waitee)
-{
- waitees.insert(waitee);
- waitee->waiters.insert(shared_from_this());
-}
-
-
void Goal::trace(std::string_view s)
{
debug("%1%: %2%", name, s);
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index dd29b9fc4..5d7bb72b8 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -106,12 +106,28 @@ struct Goal : public std::enable_shared_from_this<Goal>
public:
struct [[nodiscard]] StillAlive {};
+ struct [[nodiscard]] WaitForSlot {};
+ struct [[nodiscard]] WaitForAWhile {};
+ struct [[nodiscard]] ContinueImmediately {};
+ struct [[nodiscard]] WaitForGoals {
+ Goals goals;
+ };
struct [[nodiscard]] Finished {
ExitCode result;
std::unique_ptr<Error> ex;
+ bool permanentFailure = false;
+ bool timedOut = false;
+ bool hashMismatch = false;
+ bool checkMismatch = false;
};
- struct [[nodiscard]] WorkResult : std::variant<StillAlive, Finished>
+ struct [[nodiscard]] WorkResult : std::variant<
+ StillAlive,
+ WaitForSlot,
+ WaitForAWhile,
+ ContinueImmediately,
+ WaitForGoals,
+ Finished>
{
WorkResult() = delete;
using variant::variant;
@@ -133,8 +149,6 @@ public:
virtual WorkResult work() = 0;
- void addWaitee(GoalPtr waitee);
-
virtual void waiteeDone(GoalPtr waitee) { }
virtual WorkResult handleChildOutput(int fd, std::string_view data)
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index c435a3c00..8b640e4ad 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -158,9 +158,8 @@ Goal::WorkResult LocalDerivationGoal::tryLocalBuild()
unsigned int curBuilds = worker.getNrLocalBuilds();
if (curBuilds >= settings.maxBuildJobs) {
state = &DerivationGoal::tryToBuild;
- worker.waitForBuildSlot(shared_from_this());
outputLocks.unlock();
- return StillAlive{};
+ return WaitForSlot{};
}
assert(derivationType);
@@ -202,8 +201,7 @@ Goal::WorkResult LocalDerivationGoal::tryLocalBuild()
if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath))));
- worker.waitForAWhile(shared_from_this());
- return StillAlive{};
+ return WaitForAWhile{};
}
}
@@ -237,8 +235,9 @@ Goal::WorkResult LocalDerivationGoal::tryLocalBuild()
} catch (BuildError & e) {
outputLocks.unlock();
buildUser.reset();
- worker.permanentFailure = true;
- return done(BuildResult::InputRejected, {}, std::move(e));
+ auto report = done(BuildResult::InputRejected, {}, std::move(e));
+ report.permanentFailure = true;
+ return report;
}
/* This state will be reached when we get EOF on the child's
@@ -2197,7 +2196,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
if (wanted != got) {
/* Throw an error after registering the path as
valid. */
- worker.hashMismatch = true;
+ anyHashMismatchSeen = true;
// XXX: shameless layering violation hack that makes the hash mismatch error at least not utterly worthless
auto guessedUrl = getOr(drv->env, "urls", getOr(drv->env, "url", "(unknown)"));
delayedException = std::make_exception_ptr(
@@ -2284,7 +2283,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
if (!worker.store.isValidPath(newInfo.path)) continue;
ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
if (newInfo.narHash != oldInfo.narHash) {
- worker.checkMismatch = true;
+ anyCheckMismatchSeen = true;
if (settings.runDiffHook || settings.keepFailed) {
auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
deletePath(dst);
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index fb2949fd0..67a5f20bb 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -152,15 +152,16 @@ Goal::WorkResult PathSubstitutionGoal::tryNext()
/* To maintain the closure invariant, we first have to realise the
paths referenced by this one. */
+ WaitForGoals result;
for (auto & i : info->references)
if (i != storePath) /* ignore self-references */
- addWaitee(worker.makePathSubstitutionGoal(i));
+ result.goals.insert(worker.makePathSubstitutionGoal(i));
- if (waitees.empty()) {/* to prevent hang (no wake-up event) */
+ if (result.goals.empty()) {/* to prevent hang (no wake-up event) */
return referencesValid();
} else {
state = &PathSubstitutionGoal::referencesValid;
- return StillAlive{};
+ return result;
}
}
@@ -181,8 +182,7 @@ Goal::WorkResult PathSubstitutionGoal::referencesValid()
assert(worker.store.isValidPath(i));
state = &PathSubstitutionGoal::tryToRun;
- worker.wakeUp(shared_from_this());
- return StillAlive{};
+ return ContinueImmediately{};
}
@@ -194,8 +194,7 @@ Goal::WorkResult PathSubstitutionGoal::tryToRun()
if maxSubstitutionJobs == 0, we still allow a substituter to run. This
prevents infinite waiting. */
if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) {
- worker.waitForBuildSlot(shared_from_this());
- return StillAlive{};
+ return WaitForSlot{};
}
maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
@@ -256,8 +255,7 @@ Goal::WorkResult PathSubstitutionGoal::finished()
/* Try the next substitute. */
state = &PathSubstitutionGoal::tryNext;
- worker.wakeUp(shared_from_this());
- return StillAlive{};
+ return ContinueImmediately{};
}
worker.markContentsGood(storePath);
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 135cfecf5..aaa4e8907 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -21,10 +21,6 @@ Worker::Worker(Store & store, Store & evalStore)
nrLocalBuilds = 0;
nrSubstitutions = 0;
lastWokenUp = steady_time_point::min();
- permanentFailure = false;
- timedOut = false;
- hashMismatch = false;
- checkMismatch = false;
}
@@ -145,6 +141,11 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f)
assert(!goal->exitCode.has_value());
goal->exitCode = f.result;
+ permanentFailure |= f.permanentFailure;
+ timedOut |= f.timedOut;
+ hashMismatch |= f.hashMismatch;
+ checkMismatch |= f.checkMismatch;
+
if (f.ex) {
if (!goal->waiters.empty())
logError(f.ex->info());
@@ -187,6 +188,15 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how)
std::visit(
overloaded{
[&](Goal::StillAlive) {},
+ [&](Goal::WaitForSlot) { waitForBuildSlot(goal); },
+ [&](Goal::WaitForAWhile) { waitForAWhile(goal); },
+ [&](Goal::ContinueImmediately) { wakeUp(goal); },
+ [&](Goal::WaitForGoals & w) {
+ for (auto & dep : w.goals) {
+ goal->waitees.insert(dep);
+ dep->waiters.insert(goal);
+ }
+ },
[&](Goal::Finished & f) { goalFinished(goal, f); },
},
how
@@ -521,7 +531,7 @@ void Worker::waitForInput()
if (rd == 0 || (rd == -1 && errno == EIO)) {
debug("%1%: got EOF", goal->getName());
goal->handleEOF(k);
- wakeUp(goal);
+ handleWorkResult(goal, Goal::ContinueImmediately{});
j->fds.erase(k);
} else if (rd == -1) {
if (errno != EINTR)
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index 5af93b49e..a741b2672 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -105,35 +105,59 @@ private:
*/
std::map<StorePath, bool> pathContentsGoodCache;
- void goalFinished(GoalPtr goal, Goal::Finished & f);
- void handleWorkResult(GoalPtr goal, Goal::WorkResult how);
-
-public:
-
- const Activity act;
- const Activity actDerivations;
- const Activity actSubstitutions;
-
/**
* Set if at least one derivation had a BuildError (i.e. permanent
* failure).
*/
- bool permanentFailure;
+ bool permanentFailure = false;
/**
* Set if at least one derivation had a timeout.
*/
- bool timedOut;
+ bool timedOut = false;
/**
* Set if at least one derivation fails with a hash mismatch.
*/
- bool hashMismatch;
+ bool hashMismatch = false;
/**
* Set if at least one derivation is not deterministic in check mode.
*/
- bool checkMismatch;
+ bool checkMismatch = false;
+
+ void goalFinished(GoalPtr goal, Goal::Finished & f);
+ void handleWorkResult(GoalPtr goal, Goal::WorkResult how);
+
+ /**
+ * Put `goal` to sleep until a build slot becomes available (which
+ * might be right away).
+ */
+ void waitForBuildSlot(GoalPtr goal);
+
+ /**
+ * Wait for a few seconds and then retry this goal. Used when
+ * waiting for a lock held by another process. This kind of
+ * polling is inefficient, but POSIX doesn't really provide a way
+ * to wait for multiple locks in the main select() loop.
+ */
+ void waitForAWhile(GoalPtr goal);
+
+ /**
+ * Wake up a goal (i.e., there is something for it to do).
+ */
+ void wakeUp(GoalPtr goal);
+
+ /**
+ * Wait for input to become available.
+ */
+ void waitForInput();
+
+public:
+
+ const Activity act;
+ const Activity actDerivations;
+ const Activity actSubstitutions;
Store & store;
Store & evalStore;
@@ -206,11 +230,6 @@ public:
void removeGoal(GoalPtr goal);
/**
- * Wake up a goal (i.e., there is something for it to do).
- */
- void wakeUp(GoalPtr goal);
-
- /**
* Return the number of local build processes currently running (but not
* remote builds via the build hook).
*/
@@ -234,29 +253,10 @@ public:
void childTerminated(Goal * goal);
/**
- * Put `goal` to sleep until a build slot becomes available (which
- * might be right away).
- */
- void waitForBuildSlot(GoalPtr goal);
-
- /**
- * Wait for a few seconds and then retry this goal. Used when
- * waiting for a lock held by another process. This kind of
- * polling is inefficient, but POSIX doesn't really provide a way
- * to wait for multiple locks in the main select() loop.
- */
- void waitForAWhile(GoalPtr goal);
-
- /**
* Loop until the specified top-level goals have finished.
*/
void run(const Goals & topGoals);
- /**
- * Wait for input to become available.
- */
- void waitForInput();
-
/***
* The exit status in case of failure.
*
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 06:17:41 +0000