aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libstore/build/goal.hh107
-rw-r--r--src/libstore/build/hook-instance.hh31
-rw-r--r--src/libstore/build/substitution-goal.hh89
-rw-r--r--src/libstore/build/worker.hh195
4 files changed, 422 insertions, 0 deletions
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
new file mode 100644
index 000000000..360c160ce
--- /dev/null
+++ b/src/libstore/build/goal.hh
@@ -0,0 +1,107 @@
+#pragma once
+
+#include "types.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+/* Forward definition. */
+struct Goal;
+struct Worker;
+
+/* A pointer to a goal. */
+typedef std::shared_ptr<Goal> GoalPtr;
+typedef std::weak_ptr<Goal> WeakGoalPtr;
+
+struct CompareGoalPtrs {
+ bool operator() (const GoalPtr & a, const GoalPtr & b) const;
+};
+
+/* Set of goals. */
+typedef set<GoalPtr, CompareGoalPtrs> Goals;
+typedef list<WeakGoalPtr> WeakGoals;
+
+/* A map of paths to goals (and the other way around). */
+typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap;
+
+struct Goal : public std::enable_shared_from_this<Goal>
+{
+ typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode;
+
+ /* Backlink to the worker. */
+ Worker & worker;
+
+ /* Goals that this goal is waiting for. */
+ Goals waitees;
+
+ /* Goals waiting for this one to finish. Must use weak pointers
+ here to prevent cycles. */
+ WeakGoals waiters;
+
+ /* Number of goals we are/were waiting for that have failed. */
+ unsigned int nrFailed;
+
+ /* Number of substitution goals we are/were waiting for that
+ failed because there are no substituters. */
+ unsigned int nrNoSubstituters;
+
+ /* Number of substitution goals we are/were waiting for that
+ failed because othey had unsubstitutable references. */
+ unsigned int nrIncompleteClosure;
+
+ /* Name of this goal for debugging purposes. */
+ string name;
+
+ /* Whether the goal is finished. */
+ ExitCode exitCode;
+
+ /* Exception containing an error message, if any. */
+ std::optional<Error> ex;
+
+ Goal(Worker & worker) : worker(worker)
+ {
+ nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
+ exitCode = ecBusy;
+ }
+
+ virtual ~Goal()
+ {
+ trace("goal destroyed");
+ }
+
+ virtual void work() = 0;
+
+ void addWaitee(GoalPtr waitee);
+
+ virtual void waiteeDone(GoalPtr waitee, ExitCode result);
+
+ virtual void handleChildOutput(int fd, const string & data)
+ {
+ abort();
+ }
+
+ virtual void handleEOF(int fd)
+ {
+ abort();
+ }
+
+ void trace(const FormatOrString & fs);
+
+ string getName()
+ {
+ return name;
+ }
+
+ /* Callback in case of a timeout. It should wake up its waiters,
+ get rid of any running child processes that are being monitored
+ by the worker (important!), etc. */
+ virtual void timedOut(Error && ex) = 0;
+
+ virtual string key() = 0;
+
+ void amDone(ExitCode result, std::optional<Error> ex = {});
+};
+
+void addToWeakGoals(WeakGoals & goals, GoalPtr p);
+
+}
diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh
new file mode 100644
index 000000000..9e8cff128
--- /dev/null
+++ b/src/libstore/build/hook-instance.hh
@@ -0,0 +1,31 @@
+#pragma once
+
+#include "logging.hh"
+#include "serialise.hh"
+
+namespace nix {
+
+struct HookInstance
+{
+ /* Pipes for talking to the build hook. */
+ Pipe toHook;
+
+ /* Pipe for the hook's standard output/error. */
+ Pipe fromHook;
+
+ /* Pipe for the builder's standard output/error. */
+ Pipe builderOut;
+
+ /* The process ID of the hook. */
+ Pid pid;
+
+ FdSink sink;
+
+ std::map<ActivityId, Activity> activities;
+
+ HookInstance();
+
+ ~HookInstance();
+};
+
+}
diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh
new file mode 100644
index 000000000..3ae9a9e6b
--- /dev/null
+++ b/src/libstore/build/substitution-goal.hh
@@ -0,0 +1,89 @@
+#pragma once
+
+#include "lock.hh"
+#include "store-api.hh"
+#include "goal.hh"
+
+namespace nix {
+
+class Worker;
+
+class SubstitutionGoal : public Goal
+{
+ friend class Worker;
+
+private:
+ /* The store path that should be realised through a substitute. */
+ StorePath storePath;
+
+ /* The path the substituter refers to the path as. This will be
+ * different when the stores have different names. */
+ std::optional<StorePath> subPath;
+
+ /* The remaining substituters. */
+ std::list<ref<Store>> subs;
+
+ /* The current substituter. */
+ std::shared_ptr<Store> sub;
+
+ /* Whether a substituter failed. */
+ bool substituterFailed = false;
+
+ /* Path info returned by the substituter's query info operation. */
+ std::shared_ptr<const ValidPathInfo> info;
+
+ /* Pipe for the substituter's standard output. */
+ Pipe outPipe;
+
+ /* The substituter thread. */
+ std::thread thr;
+
+ std::promise<void> promise;
+
+ /* Whether to try to repair a valid path. */
+ RepairFlag repair;
+
+ /* Location where we're downloading the substitute. Differs from
+ storePath when doing a repair. */
+ Path destPath;
+
+ std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
+ maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
+
+ typedef void (SubstitutionGoal::*GoalState)();
+ GoalState state;
+
+ /* Content address for recomputing store path */
+ std::optional<ContentAddress> ca;
+
+public:
+ SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+ ~SubstitutionGoal();
+
+ void timedOut(Error && ex) override { abort(); };
+
+ string key() override
+ {
+ /* "a$" ensures substitution goals happen before derivation
+ goals. */
+ return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath);
+ }
+
+ void work() override;
+
+ /* The states. */
+ void init();
+ void tryNext();
+ void gotInfo();
+ void referencesValid();
+ void tryToRun();
+ void finished();
+
+ /* Callback used by the worker to write to the log. */
+ void handleChildOutput(int fd, const string & data) override;
+ void handleEOF(int fd) override;
+
+ StorePath getStorePath() { return storePath; }
+};
+
+}
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
new file mode 100644
index 000000000..a54316343
--- /dev/null
+++ b/src/libstore/build/worker.hh
@@ -0,0 +1,195 @@
+#pragma once
+
+#include "types.hh"
+#include "lock.hh"
+#include "local-store.hh"
+#include "goal.hh"
+
+namespace nix {
+
+/* Forward definition. */
+class DerivationGoal;
+
+typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
+
+
+/* A mapping used to remember for each child process to what goal it
+ belongs, and file descriptors for receiving log data and output
+ path creation commands. */
+struct Child
+{
+ WeakGoalPtr goal;
+ Goal * goal2; // ugly hackery
+ set<int> fds;
+ bool respectTimeouts;
+ bool inBuildSlot;
+ steady_time_point lastOutput; /* time we last got output on stdout/stderr */
+ steady_time_point timeStarted;
+};
+
+/* Forward definition. */
+struct HookInstance;
+
+/* The worker class. */
+class Worker
+{
+private:
+
+ /* Note: the worker should only have strong pointers to the
+ top-level goals. */
+
+ /* The top-level goals of the worker. */
+ Goals topGoals;
+
+ /* Goals that are ready to do some work. */
+ WeakGoals awake;
+
+ /* Goals waiting for a build slot. */
+ WeakGoals wantingToBuild;
+
+ /* Child processes currently running. */
+ std::list<Child> children;
+
+ /* Number of build slots occupied. This includes local builds and
+ substitutions but not remote builds via the build hook. */
+ unsigned int nrLocalBuilds;
+
+ /* Maps used to prevent multiple instantiations of a goal for the
+ same derivation / path. */
+ WeakGoalMap derivationGoals;
+ WeakGoalMap substitutionGoals;
+
+ /* Goals waiting for busy paths to be unlocked. */
+ WeakGoals waitingForAnyGoal;
+
+ /* Goals sleeping for a few seconds (polling a lock). */
+ WeakGoals waitingForAWhile;
+
+ /* Last time the goals in `waitingForAWhile' where woken up. */
+ steady_time_point lastWokenUp;
+
+ /* Cache for pathContentsGood(). */
+ std::map<StorePath, bool> pathContentsGoodCache;
+
+public:
+
+ const Activity act;
+ const Activity actDerivations;
+ const Activity actSubstitutions;
+
+ /* Set if at least one derivation had a BuildError (i.e. permanent
+ failure). */
+ bool permanentFailure;
+
+ /* Set if at least one derivation had a timeout. */
+ bool timedOut;
+
+ /* Set if at least one derivation fails with a hash mismatch. */
+ bool hashMismatch;
+
+ /* Set if at least one derivation is not deterministic in check mode. */
+ bool checkMismatch;
+
+ LocalStore & store;
+
+ std::unique_ptr<HookInstance> hook;
+
+ uint64_t expectedBuilds = 0;
+ uint64_t doneBuilds = 0;
+ uint64_t failedBuilds = 0;
+ uint64_t runningBuilds = 0;
+
+ uint64_t expectedSubstitutions = 0;
+ uint64_t doneSubstitutions = 0;
+ uint64_t failedSubstitutions = 0;
+ uint64_t runningSubstitutions = 0;
+ uint64_t expectedDownloadSize = 0;
+ uint64_t doneDownloadSize = 0;
+ uint64_t expectedNarSize = 0;
+ uint64_t doneNarSize = 0;
+
+ /* Whether to ask the build hook if it can build a derivation. If
+ it answers with "decline-permanently", we don't try again. */
+ bool tryBuildHook = true;
+
+ Worker(LocalStore & store);
+ ~Worker();
+
+ /* Make a goal (with caching). */
+
+ /* derivation goal */
+private:
+ std::shared_ptr<DerivationGoal> makeDerivationGoalCommon(
+ const StorePath & drvPath, const StringSet & wantedOutputs,
+ std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal);
+public:
+ std::shared_ptr<DerivationGoal> makeDerivationGoal(
+ const StorePath & drvPath,
+ const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
+ std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(
+ const StorePath & drvPath, const BasicDerivation & drv,
+ const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
+
+ /* substitution goal */
+ GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+
+ /* Remove a dead goal. */
+ void removeGoal(GoalPtr goal);
+
+ /* Wake up a goal (i.e., there is something for it to do). */
+ void wakeUp(GoalPtr goal);
+
+ /* Return the number of local build and substitution processes
+ currently running (but not remote builds via the build
+ hook). */
+ unsigned int getNrLocalBuilds();
+
+ /* Registers a running child process. `inBuildSlot' means that
+ the process counts towards the jobs limit. */
+ void childStarted(GoalPtr goal, const set<int> & fds,
+ bool inBuildSlot, bool respectTimeouts);
+
+ /* Unregisters a running child process. `wakeSleepers' should be
+ false if there is no sense in waking up goals that are sleeping
+ because they can't run yet (e.g., there is no free build slot,
+ or the hook would still say `postpone'). */
+ void childTerminated(Goal * goal, bool wakeSleepers = true);
+
+ /* Put `goal' to sleep until a build slot becomes available (which
+ might be right away). */
+ void waitForBuildSlot(GoalPtr goal);
+
+ /* Wait for any goal to finish. Pretty indiscriminate way to
+ wait for some resource that some other goal is holding. */
+ void waitForAnyGoal(GoalPtr goal);
+
+ /* Wait for a few seconds and then retry this goal. Used when
+ waiting for a lock held by another process. This kind of
+ polling is inefficient, but POSIX doesn't really provide a way
+ to wait for multiple locks in the main select() loop. */
+ void waitForAWhile(GoalPtr goal);
+
+ /* Loop until the specified top-level goals have finished. */
+ void run(const Goals & topGoals);
+
+ /* Wait for input to become available. */
+ void waitForInput();
+
+ unsigned int exitStatus();
+
+ /* Check whether the given valid path exists and has the right
+ contents. */
+ bool pathContentsGood(const StorePath & path);
+
+ void markContentsGood(const StorePath & path);
+
+ void updateProgress()
+ {
+ actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
+ actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
+ act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
+ act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
+ }
+};
+
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 22:32:29 +0000