diff options
author | John Ericson <John.Ericson@Obsidian.Systems> | 2023-04-17 10:16:57 -0400 |
---|---|---|
committer | John Ericson <John.Ericson@Obsidian.Systems> | 2023-04-17 10:16:57 -0400 |
commit | e12efa365462bf7c65e6b531a7ace4fc1660e2cc (patch) | |
tree | e11959347637a16cd85a9e104f87c2fe97ce3e26 /src/libstore | |
parent | 1fcd49dbbdf13d673ab7a94b5dd9f9c8b55f5321 (diff) | |
parent | e641de085b625e56b723f45e8355deaa01ea3a1a (diff) |
Merge remote-tracking branch 'upstream/master' into ca-drv-exotic
Diffstat (limited to 'src/libstore')
79 files changed, 1748 insertions, 684 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 628e9b9db..fcd763a9d 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -315,6 +315,7 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }, @@ -433,6 +434,7 @@ StorePath BinaryCacheStore::addToStore( }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }, diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c1d08926d..49f271d24 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "crypto.hh" #include "store-api.hh" @@ -45,6 +46,11 @@ struct BinaryCacheStoreConfig : virtual StoreConfig )"}; }; + +/** + * @note subclasses must implement at least one of the two + * virtual getFile() methods. + */ class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store, public virtual LogStore @@ -74,14 +80,15 @@ public: std::string && data, const std::string & mimeType); - /* Note: subclasses must implement at least one of the two - following getFile() methods. */ - - /* Dump the contents of the specified file to a sink. */ + /** + * Dump the contents of the specified file to a sink. + */ virtual void getFile(const std::string & path, Sink & sink); - /* Fetch the specified file and call the specified callback with - the result. A subclass may implement this asynchronously. */ + /** + * Fetch the specified file and call the specified callback with + * the result. A subclass may implement this asynchronously. + */ virtual void getFile( const std::string & path, Callback<std::optional<std::string>> callback) noexcept; diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index a5749cf33..27d1a1b6c 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "realisation.hh" #include "derived-path.hh" @@ -11,9 +12,12 @@ namespace nix { struct BuildResult { - /* Note: don't remove status codes, and only add new status codes - at the end of the list, to prevent client/server - incompatibilities in the nix-store --serve protocol. */ + /** + * @note This is directly used in the nix-store --serve protocol. + * That means we need to worry about compatability across versions. + * Therefore, don't remove status codes, and only add new status + * codes at the end of the list. + */ enum Status { Built = 0, Substituted, @@ -21,8 +25,10 @@ struct BuildResult PermanentFailure, InputRejected, OutputRejected, - TransientFailure, // possibly transient - CachedFailure, // no longer used + /// possibly transient + TransientFailure, + /// no longer used + CachedFailure, TimedOut, MiscFailure, DependencyFailed, @@ -32,7 +38,12 @@ struct BuildResult NoSubstituters, } status = MiscFailure; - // FIXME: include entire ErrorInfo object. + /** + * Information about the error if the build failed. + * + * @todo This should be an entire ErrorInfo object, not just a + * string, for richer information. + */ std::string errorMsg; std::string toString() const { @@ -52,33 +63,46 @@ struct BuildResult case LogLimitExceeded: return "LogLimitExceeded"; case NotDeterministic: return "NotDeterministic"; case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + case NoSubstituters: return "NoSubstituters"; default: return "Unknown"; }; }(); return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); } - /* How many times this build was performed. */ + /** + * How many times this build was performed. + */ unsigned int timesBuilt = 0; - /* If timesBuilt > 1, whether some builds did not produce the same - result. (Note that 'isNonDeterministic = false' does not mean - the build is deterministic, just that we don't have evidence of - non-determinism.) */ + /** + * If timesBuilt > 1, whether some builds did not produce the same + * result. (Note that 'isNonDeterministic = false' does not mean + * the build is deterministic, just that we don't have evidence of + * non-determinism.) + */ bool isNonDeterministic = false; - /* The derivation we built or the store path we substituted. */ + /** + * The derivation we built or the store path we substituted. + */ DerivedPath path; - /* For derivations, a mapping from the names of the wanted outputs - to actual paths. */ + /** + * For derivations, a mapping from the names of the wanted outputs + * to actual paths. + */ DrvOutputs builtOutputs; - /* The start/stop times of the build (or one of the rounds, if it - was repeated). */ + /** + * The start/stop times of the build (or one of the rounds, if it + * was repeated). + */ time_t startTime = 0, stopTime = 0; - /* User and system CPU time the build took. */ + /** + * User and system CPU time the build took. + */ std::optional<std::chrono::microseconds> cpuUser, cpuSystem; bool success() diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 596034c0f..26faf8c8e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -911,7 +911,11 @@ void DerivationGoal::buildDone() msg += line; msg += "\n"; } - msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.", + auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) + ? "nix log" + : "nix-store -l"; + msg += fmt("For full logs, run '" ANSI_BOLD "%s %s" ANSI_NORMAL "'.", + nixLogCommand, worker.store.printStorePath(drvPath)); } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 707e38b4b..3a6f0c2d9 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "parsed-derivations.hh" #include "lock.hh" @@ -15,8 +16,10 @@ struct HookInstance; typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; -/* Unless we are repairing, we don't both to test validity and just assume it, - so the choices are `Absent` or `Valid`. */ +/** + * Unless we are repairing, we don't both to test validity and just assume it, + * so the choices are `Absent` or `Valid`. + */ enum struct PathStatus { Corrupt, Absent, @@ -26,11 +29,15 @@ enum struct PathStatus { struct InitialOutputStatus { StorePath path; PathStatus status; - /* Valid in the store, and additionally non-corrupt if we are repairing */ + /** + * Valid in the store, and additionally non-corrupt if we are repairing + */ bool isValid() const { return status == PathStatus::Valid; } - /* Merely present, allowed to be corrupt */ + /** + * Merely present, allowed to be corrupt + */ bool isPresent() const { return status == PathStatus::Corrupt || status == PathStatus::Valid; @@ -45,59 +52,87 @@ struct InitialOutput { struct DerivationGoal : public Goal { - /* Whether to use an on-disk .drv file. */ + /** + * Whether to use an on-disk .drv file. + */ bool useDerivation; - /* The path of the derivation. */ + /** The path of the derivation. */ StorePath drvPath; - /* The goal for the corresponding resolved derivation */ + /** + * The goal for the corresponding resolved derivation + */ std::shared_ptr<DerivationGoal> resolvedDrvGoal; - /* The specific outputs that we need to build. Empty means all of - them. */ + /** + * The specific outputs that we need to build. Empty means all of + * them. + */ OutputsSpec wantedOutputs; - /* Mapping from input derivations + output names to actual store - paths. This is filled in by waiteeDone() as each dependency - finishes, before inputsRealised() is reached, */ + /** + * Mapping from input derivations + output names to actual store + * paths. This is filled in by waiteeDone() as each dependency + * finishes, before inputsRealised() is reached. + */ std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs; - /* Whether additional wanted outputs have been added. */ + /** + * Whether additional wanted outputs have been added. + */ bool needRestart = false; - /* Whether to retry substituting the outputs after building the - inputs. This is done in case of an incomplete closure. */ + /** + * Whether to retry substituting the outputs after building the + * inputs. This is done in case of an incomplete closure. + */ bool retrySubstitution = false; - /* Whether we've retried substitution, in which case we won't try - again. */ + /** + * Whether we've retried substitution, in which case we won't try + * again. + */ bool retriedSubstitution = false; - /* The derivation stored at drvPath. */ + /** + * The derivation stored at drvPath. + */ std::unique_ptr<Derivation> drv; std::unique_ptr<ParsedDerivation> parsedDrv; - /* The remainder is state held during the build. */ + /** + * The remainder is state held during the build. + */ - /* Locks on (fixed) output paths. */ + /** + * Locks on (fixed) output paths. + */ PathLocks outputLocks; - /* All input paths (that is, the union of FS closures of the - immediate input paths). */ + /** + * All input paths (that is, the union of FS closures of the + * immediate input paths). + */ StorePathSet inputPaths; std::map<std::string, InitialOutput> initialOutputs; - /* File descriptor for the log file. */ + /** + * File descriptor for the log file. + */ AutoCloseFD fdLogFile; std::shared_ptr<BufferedSink> logFileSink, logSink; - /* Number of bytes received from the builder's stdout/stderr. */ + /** + * Number of bytes received from the builder's stdout/stderr. + */ unsigned long logSize; - /* The most recent log lines. */ + /** + * The most recent log lines. + */ std::list<std::string> logTail; std::string currentLogLine; @@ -105,10 +140,14 @@ struct DerivationGoal : public Goal std::string currentHookLine; - /* The build hook. */ + /** + * The build hook. + */ std::unique_ptr<HookInstance> hook; - /* The sort of derivation we are building. */ + /** + * The sort of derivation we are building. + */ DerivationType derivationType; typedef void (DerivationGoal::*GoalState)(); @@ -120,12 +159,16 @@ struct DerivationGoal : public Goal std::unique_ptr<Activity> act; - /* Activity that denotes waiting for a lock. */ + /** + * Activity that denotes waiting for a lock. + */ std::unique_ptr<Activity> actLock; std::map<ActivityId, Activity> builderActivities; - /* The remote machine on which we're building. */ + /** + * The remote machine on which we're building. + */ std::string machineName; DerivationGoal(const StorePath & drvPath, @@ -142,10 +185,14 @@ struct DerivationGoal : public Goal void work() override; - /* Add wanted outputs to an already existing derivation goal. */ + /** + * Add wanted outputs to an already existing derivation goal. + */ void addWantedOutputs(const OutputsSpec & outputs); - /* The states. */ + /** + * The states. + */ void getDerivation(); void loadDerivation(); void haveDerivation(); @@ -159,28 +206,42 @@ struct DerivationGoal : public Goal void resolvedFinished(); - /* Is the build hook willing to perform the build? */ + /** + * Is the build hook willing to perform the build? + */ HookReply tryBuildHook(); virtual int getChildStatus(); - /* Check that the derivation outputs all exist and register them - as valid. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ virtual DrvOutputs registerOutputs(); - /* Open a log file and a pipe to it. */ + /** + * Open a log file and a pipe to it. + */ Path openLogFile(); - /* Sign the newly built realisation if the store allows it */ + /** + * Sign the newly built realisation if the store allows it + */ virtual void signRealisation(Realisation&) {} - /* Close the log file. */ + /** + * Close the log file. + */ void closeLogFile(); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ virtual void closeReadPipes(); - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ virtual void cleanupHookFinally(); virtual void cleanupPreChildKill(); virtual void cleanupPostChildKill(); @@ -190,30 +251,40 @@ struct DerivationGoal : public Goal virtual bool isReadDesc(int fd); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; void flushLine(); - /* Wrappers around the corresponding Store methods that first consult the - derivation. This is currently needed because when there is no drv file - there also is no DB entry. */ + /** + * Wrappers around the corresponding Store methods that first consult the + * derivation. This is currently needed because when there is no drv file + * there also is no DB entry. + */ std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(); OutputPathMap queryDerivationOutputMap(); - /* Update 'initialOutputs' to determine the current status of the - outputs of the derivation. Also returns a Boolean denoting - whether all outputs are valid and non-corrupt, and a - 'DrvOutputs' structure containing the valid and wanted - outputs. */ + /** + * Update 'initialOutputs' to determine the current status of the + * outputs of the derivation. Also returns a Boolean denoting + * whether all outputs are valid and non-corrupt, and a + * 'DrvOutputs' structure containing the valid and wanted + * outputs. + */ std::pair<bool, DrvOutputs> checkPathValidity(); - /* Aborts if any output is not valid or corrupt, and otherwise - returns a 'DrvOutputs' structure containing the wanted - outputs. */ + /** + * Aborts if any output is not valid or corrupt, and otherwise + * returns a 'DrvOutputs' structure containing the wanted + * outputs. + */ DrvOutputs assertPathValidity(); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ virtual void killChild(); void repairClosure(); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index e4b044790..697ddb283 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "goal.hh" @@ -10,24 +11,34 @@ namespace nix { class Worker; -// Substitution of a derivation output. -// This is done in three steps: -// 1. Fetch the output info from a substituter -// 2. Substitute the corresponding output path -// 3. Register the output info +/** + * Substitution of a derivation output. + * This is done in three steps: + * 1. Fetch the output info from a substituter + * 2. Substitute the corresponding output path + * 3. Register the output info + */ class DrvOutputSubstitutionGoal : public Goal { - // The drv output we're trying to substitue + /** + * The drv output we're trying to substitue + */ DrvOutput id; - // The realisation corresponding to the given output id. - // Will be filled once we can get it. + /** + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ std::shared_ptr<const Realisation> outputInfo; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list<ref<Store>> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr<Store> sub; struct DownloadState @@ -38,7 +49,9 @@ class DrvOutputSubstitutionGoal : public Goal { std::shared_ptr<DownloadState> downloadState; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; public: diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 776eb86bc..f4bf6f38b 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "store-api.hh" @@ -6,11 +7,15 @@ namespace nix { -/* Forward definition. */ +/** + * Forward definition. + */ struct Goal; class Worker; -/* A pointer to a goal. */ +/** + * A pointer to a goal. + */ typedef std::shared_ptr<Goal> GoalPtr; typedef std::weak_ptr<Goal> WeakGoalPtr; @@ -18,48 +23,72 @@ struct CompareGoalPtrs { bool operator() (const GoalPtr & a, const GoalPtr & b) const; }; -/* Set of goals. */ +/** + * Set of goals. + */ typedef std::set<GoalPtr, CompareGoalPtrs> Goals; typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals; -/* A map of paths to goals (and the other way around). */ +/** + * A map of paths to goals (and the other way around). + */ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; struct Goal : public std::enable_shared_from_this<Goal> { typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - /* Backlink to the worker. */ + /** + * Backlink to the worker. + */ Worker & worker; - /* Goals that this goal is waiting for. */ + /** + * Goals that this goal is waiting for. + */ Goals waitees; - /* Goals waiting for this one to finish. Must use weak pointers - here to prevent cycles. */ + /** + * Goals waiting for this one to finish. Must use weak pointers + * here to prevent cycles. + */ WeakGoals waiters; - /* Number of goals we are/were waiting for that have failed. */ + /** + * Number of goals we are/were waiting for that have failed. + */ size_t nrFailed = 0; - /* Number of substitution goals we are/were waiting for that - failed because there are no substituters. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because there are no substituters. + */ size_t nrNoSubstituters = 0; - /* Number of substitution goals we are/were waiting for that - failed because they had unsubstitutable references. */ + /** + * Number of substitution goals we are/were waiting for that + * failed because they had unsubstitutable references. + */ size_t nrIncompleteClosure = 0; - /* Name of this goal for debugging purposes. */ + /** + * Name of this goal for debugging purposes. + */ std::string name; - /* Whether the goal is finished. */ + /** + * Whether the goal is finished. + */ ExitCode exitCode = ecBusy; - /* Build result. */ + /** + * Build result. + */ BuildResult buildResult; - /* Exception containing an error message, if any. */ + /** + * Exception containing an error message, if any. + */ std::optional<Error> ex; Goal(Worker & worker, DerivedPath path) @@ -95,9 +124,11 @@ struct Goal : public std::enable_shared_from_this<Goal> return name; } - /* Callback in case of a timeout. It should wake up its waiters, - get rid of any running child processes that are being monitored - by the worker (important!), etc. */ + /** + * Callback in case of a timeout. It should wake up its waiters, + * get rid of any running child processes that are being monitored + * by the worker (important!), etc. + */ virtual void timedOut(Error && ex) = 0; virtual std::string key() = 0; diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index 9e8cff128..d84f62877 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "logging.hh" #include "serialise.hh" @@ -7,16 +8,24 @@ namespace nix { struct HookInstance { - /* Pipes for talking to the build hook. */ + /** + * Pipes for talking to the build hook. + */ Pipe toHook; - /* Pipe for the hook's standard output/error. */ + /** + * Pipe for the hook's standard output/error. + */ Pipe fromHook; - /* Pipe for the builder's standard output/error. */ + /** + * Pipe for the builder's standard output/error. + */ Pipe builderOut; - /* The process ID of the hook. */ + /** + * The process ID of the hook. + */ Pid pid; FdSink sink; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 7cb80977d..bbec4aea0 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1415,6 +1415,9 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo virtual void addBuildLog(const StorePath & path, std::string_view log) override { unsupported("addBuildLog"); } + + std::optional<TrustedFlag> isTrustedClient() override + { return NotTrusted; } }; @@ -1467,7 +1470,7 @@ void LocalDerivationGoal::startDaemon() FdSink to(remote.get()); try { daemon::processConnection(store, from, to, - daemon::NotTrusted, daemon::Recursive); + NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { ignoreException(); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index c9ecc8828..42d32a31a 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivation-goal.hh" #include "local-store.hh" @@ -9,49 +10,75 @@ struct LocalDerivationGoal : public DerivationGoal { LocalStore & getLocalStore(); - /* User selected for running the builder. */ + /** + * User selected for running the builder. + */ std::unique_ptr<UserLock> buildUser; - /* The process ID of the builder. */ + /** + * The process ID of the builder. + */ Pid pid; - /* The cgroup of the builder, if any. */ + /** + * The cgroup of the builder, if any. + */ std::optional<Path> cgroup; - /* The temporary directory. */ + /** + * The temporary directory. + */ Path tmpDir; - /* The path of the temporary directory in the sandbox. */ + /** + * The path of the temporary directory in the sandbox. + */ Path tmpDirInSandbox; - /* Master side of the pseudoterminal used for the builder's - standard output/error. */ + /** + * Master side of the pseudoterminal used for the builder's + * standard output/error. + */ AutoCloseFD builderOut; - /* Pipe for synchronising updates to the builder namespaces. */ + /** + * Pipe for synchronising updates to the builder namespaces. + */ Pipe userNamespaceSync; - /* The mount namespace and user namespace of the builder, used to add additional - paths to the sandbox as a result of recursive Nix calls. */ + /** + * The mount namespace and user namespace of the builder, used to add additional + * paths to the sandbox as a result of recursive Nix calls. + */ AutoCloseFD sandboxMountNamespace; AutoCloseFD sandboxUserNamespace; - /* On Linux, whether we're doing the build in its own user - namespace. */ + /** + * On Linux, whether we're doing the build in its own user + * namespace. + */ bool usingUserNamespace = true; - /* Whether we're currently doing a chroot build. */ + /** + * Whether we're currently doing a chroot build. + */ bool useChroot = false; Path chrootRootDir; - /* RAII object to delete the chroot directory. */ + /** + * RAII object to delete the chroot directory. + */ std::shared_ptr<AutoDelete> autoDelChroot; - /* Whether to run the build in a private network namespace. */ + /** + * Whether to run the build in a private network namespace. + */ bool privateNetwork = false; - /* Stuff we need to pass to initChild(). */ + /** + * Stuff we need to pass to initChild(). + */ struct ChrootPath { Path source; bool optional; @@ -70,30 +97,35 @@ struct LocalDerivationGoal : public DerivationGoal SandboxProfile additionalSandboxProfile; #endif - /* Hash rewriting. */ + /** + * Hash rewriting. + */ StringMap inputRewrites, outputRewrites; typedef map<StorePath, StorePath> RedirectedOutputs; RedirectedOutputs redirectedOutputs; - /* The outputs paths used during the build. - - - Input-addressed derivations or fixed content-addressed outputs are - sometimes built when some of their outputs already exist, and can not - be hidden via sandboxing. We use temporary locations instead and - rewrite after the build. Otherwise the regular predetermined paths are - put here. - - - Floating content-addressed derivations do not know their final build - output paths until the outputs are hashed, so random locations are - used, and then renamed. The randomness helps guard against hidden - self-references. + /** + * The outputs paths used during the build. + * + * - Input-addressed derivations or fixed content-addressed outputs are + * sometimes built when some of their outputs already exist, and can not + * be hidden via sandboxing. We use temporary locations instead and + * rewrite after the build. Otherwise the regular predetermined paths are + * put here. + * + * - Floating content-addressed derivations do not know their final build + * output paths until the outputs are hashed, so random locations are + * used, and then renamed. The randomness helps guard against hidden + * self-references. */ OutputPathMap scratchOutputs; - /* Path registration info from the previous round, if we're - building multiple times. Since this contains the hash, it - allows us to compare whether two rounds produced the same - result. */ + /** + * Path registration info from the previous round, if we're + * building multiple times. Since this contains the hash, it + * allows us to compare whether two rounds produced the same + * result. + */ std::map<Path, ValidPathInfo> prevInfos; uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } @@ -101,25 +133,37 @@ struct LocalDerivationGoal : public DerivationGoal const static Path homeDir; - /* The recursive Nix daemon socket. */ + /** + * The recursive Nix daemon socket. + */ AutoCloseFD daemonSocket; - /* The daemon main thread. */ + /** + * The daemon main thread. + */ std::thread daemonThread; - /* The daemon worker threads. */ + /** + * The daemon worker threads. + */ std::vector<std::thread> daemonWorkerThreads; - /* Paths that were added via recursive Nix calls. */ + /** + * Paths that were added via recursive Nix calls. + */ StorePathSet addedPaths; - /* Realisations that were added via recursive Nix calls. */ + /** + * Realisations that were added via recursive Nix calls. + */ std::set<DrvOutput> addedDrvOutputs; - /* Recursive Nix calls are only allowed to build or realize paths - in the original input closure or added via a recursive Nix call - (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where - /nix/store/<bla> is some arbitrary path in a binary cache). */ + /** + * Recursive Nix calls are only allowed to build or realize paths + * in the original input closure or added via a recursive Nix call + * (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where + * /nix/store/<bla> is some arbitrary path in a binary cache). + */ bool isAllowed(const StorePath & path) { return inputPaths.count(path) || addedPaths.count(path); @@ -137,55 +181,81 @@ struct LocalDerivationGoal : public DerivationGoal virtual ~LocalDerivationGoal() override; - /* Whether we need to perform hash rewriting if there are valid output paths. */ + /** + * Whether we need to perform hash rewriting if there are valid output paths. + */ bool needsHashRewrite(); - /* The additional states. */ + /** + * The additional states. + */ void tryLocalBuild() override; - /* Start building a derivation. */ + /** + * Start building a derivation. + */ void startBuilder(); - /* Fill in the environment for the builder. */ + /** + * Fill in the environment for the builder. + */ void initEnv(); - /* Setup tmp dir location. */ + /** + * Setup tmp dir location. + */ void initTmpDir(); - /* Write a JSON file containing the derivation attributes. */ + /** + * Write a JSON file containing the derivation attributes. + */ void writeStructuredAttrs(); void startDaemon(); void stopDaemon(); - /* Add 'path' to the set of paths that may be referenced by the - outputs, and make it appear in the sandbox. */ + /** + * Add 'path' to the set of paths that may be referenced by the + * outputs, and make it appear in the sandbox. + */ void addDependency(const StorePath & path); - /* Make a file owned by the builder. */ + /** + * Make a file owned by the builder. + */ void chownToBuilder(const Path & path); int getChildStatus() override; - /* Run the builder's process. */ + /** + * Run the builder's process. + */ void runChild(); - /* Check that the derivation outputs all exist and register them - as valid. */ + /** + * Check that the derivation outputs all exist and register them + * as valid. + */ DrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; - /* Check that an output meets the requirements specified by the - 'outputChecks' attribute (or the legacy - '{allowed,disallowed}{References,Requisites}' attributes). */ + /** + * Check that an output meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + */ void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs); - /* Close the read side of the logger pipe. */ + /** + * Close the read side of the logger pipe. + */ void closeReadPipes() override; - /* Cleanup hooks for buildDone() */ + /** + * Cleanup hooks for buildDone() + */ void cleanupHookFinally() override; void cleanupPreChildKill() override; void cleanupPostChildKill() override; @@ -195,24 +265,36 @@ struct LocalDerivationGoal : public DerivationGoal bool isReadDesc(int fd) override; - /* Delete the temporary directory, if we have one. */ + /** + * Delete the temporary directory, if we have one. + */ void deleteTmpDir(bool force); - /* Forcibly kill the child process, if any. */ + /** + * Forcibly kill the child process, if any. + */ void killChild() override; - /* Kill any processes running under the build user UID or in the - cgroup of the build. */ + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ void killSandbox(bool getStats); - /* Create alternative path calculated from but distinct from the - input, so we can avoid overwriting outputs (or other store paths) - that already exist. */ + /** + * Create alternative path calculated from but distinct from the + * input, so we can avoid overwriting outputs (or other store paths) + * that already exist. + */ StorePath makeFallbackPath(const StorePath & path); - /* Make a path to another based on the output name along with the - derivation hash. */ - /* FIXME add option to randomize, so we can audit whether our - rewrites caught everything */ + + /** + * Make a path to another based on the output name along with the + * derivation hash. + * + * @todo Add option to randomize, so we can audit whether our + * rewrites caught everything + */ StorePath makeFallbackPath(std::string_view outputName); }; diff --git a/src/libstore/build/personality.hh b/src/libstore/build/personality.hh index 30e4f4062..91b730fab 100644 --- a/src/libstore/build/personality.hh +++ b/src/libstore/build/personality.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <string> diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index a73f8e666..c2b7fc95a 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "lock.hh" #include "store-api.hh" @@ -10,38 +11,58 @@ class Worker; struct PathSubstitutionGoal : public Goal { - /* The store path that should be realised through a substitute. */ + /** + * The store path that should be realised through a substitute. + */ StorePath storePath; - /* The path the substituter refers to the path as. This will be - different when the stores have different names. */ + /** + * The path the substituter refers to the path as. This will be + * different when the stores have different names. + */ std::optional<StorePath> subPath; - /* The remaining substituters. */ + /** + * The remaining substituters. + */ std::list<ref<Store>> subs; - /* The current substituter. */ + /** + * The current substituter. + */ std::shared_ptr<Store> sub; - /* Whether a substituter failed. */ + /** + * Whether a substituter failed. + */ bool substituterFailed = false; - /* Path info returned by the substituter's query info operation. */ + /** + * Path info returned by the substituter's query info operation. + */ std::shared_ptr<const ValidPathInfo> info; - /* Pipe for the substituter's standard output. */ + /** + * Pipe for the substituter's standard output. + */ Pipe outPipe; - /* The substituter thread. */ + /** + * The substituter thread. + */ std::thread thr; std::promise<void> promise; - /* Whether to try to repair a valid path. */ + /** + * Whether to try to repair a valid path. + */ RepairFlag repair; - /* Location where we're downloading the substitute. Differs from - storePath when doing a repair. */ + /** + * Location where we're downloading the substitute. Differs from + * storePath when doing a repair. + */ Path destPath; std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions, @@ -50,7 +71,9 @@ struct PathSubstitutionGoal : public Goal typedef void (PathSubstitutionGoal::*GoalState)(); GoalState state; - /* Content address for recomputing store path */ + /** + * Content address for recomputing store path + */ std::optional<ContentAddress> ca; void done( @@ -64,16 +87,20 @@ public: void timedOut(Error && ex) override { abort(); }; + /** + * We prepend "a$" to the key name to ensure substitution goals + * happen before derivation goals. + */ std::string key() override { - /* "a$" ensures substitution goals happen before derivation - goals. */ return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } void work() override; - /* The states. */ + /** + * The states. + */ void init(); void tryNext(); void gotInfo(); @@ -81,7 +108,9 @@ public: void tryToRun(); void finished(); - /* Callback used by the worker to write to the log. */ + /** + * Callback used by the worker to write to the log. + */ void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6d68d3cf1..48a1a27fa 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "lock.hh" @@ -16,24 +17,29 @@ struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; -/* Workaround for not being able to declare a something like - - class PathSubstitutionGoal : public Goal; - - even when Goal is a complete type. - - This is still a static cast. The purpose of exporting it is to define it in - a place where `PathSubstitutionGoal` is concrete, and use it in a place where it - is opaque. */ +/** + * Workaround for not being able to declare a something like + * + * ```c++ + * class PathSubstitutionGoal : public Goal; + * ``` + * even when Goal is a complete type. + * + * This is still a static cast. The purpose of exporting it is to define it in + * a place where `PathSubstitutionGoal` is concrete, and use it in a place where it + * is opaque. + */ GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal); GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal); typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; -/* A mapping used to remember for each child process to what goal it - belongs, and file descriptors for receiving log data and output - path creation commands. */ +/** + * A mapping used to remember for each child process to what goal it + * belongs, and file descriptors for receiving log data and output + * path creation commands. + */ struct Child { WeakGoalPtr goal; @@ -41,14 +47,19 @@ struct Child std::set<int> fds; bool respectTimeouts; bool inBuildSlot; - steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + /** + * Time we last got output on stdout/stderr + */ + steady_time_point lastOutput; steady_time_point timeStarted; }; /* Forward definition. */ struct HookInstance; -/* The worker class. */ +/** + * The worker class. + */ class Worker { private: @@ -56,38 +67,58 @@ private: /* Note: the worker should only have strong pointers to the top-level goals. */ - /* The top-level goals of the worker. */ + /** + * The top-level goals of the worker. + */ Goals topGoals; - /* Goals that are ready to do some work. */ + /** + * Goals that are ready to do some work. + */ WeakGoals awake; - /* Goals waiting for a build slot. */ + /** + * Goals waiting for a build slot. + */ WeakGoals wantingToBuild; - /* Child processes currently running. */ + /** + * Child processes currently running. + */ std::list<Child> children; - /* Number of build slots occupied. This includes local builds and - substitutions but not remote builds via the build hook. */ + /** + * Number of build slots occupied. This includes local builds and + * substitutions but not remote builds via the build hook. + */ unsigned int nrLocalBuilds; - /* Maps used to prevent multiple instantiations of a goal for the - same derivation / path. */ + /** + * Maps used to prevent multiple instantiations of a goal for the + * same derivation / path. + */ std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals; std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals; std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals; - /* Goals waiting for busy paths to be unlocked. */ + /** + * Goals waiting for busy paths to be unlocked. + */ WeakGoals waitingForAnyGoal; - /* Goals sleeping for a few seconds (polling a lock). */ + /** + * Goals sleeping for a few seconds (polling a lock). + */ WeakGoals waitingForAWhile; - /* Last time the goals in `waitingForAWhile' where woken up. */ + /** + * Last time the goals in `waitingForAWhile` where woken up. + */ steady_time_point lastWokenUp; - /* Cache for pathContentsGood(). */ + /** + * Cache for pathContentsGood(). + */ std::map<StorePath, bool> pathContentsGoodCache; public: @@ -96,17 +127,25 @@ public: const Activity actDerivations; const Activity actSubstitutions; - /* Set if at least one derivation had a BuildError (i.e. permanent - failure). */ + /** + * Set if at least one derivation had a BuildError (i.e. permanent + * failure). + */ bool permanentFailure; - /* Set if at least one derivation had a timeout. */ + /** + * Set if at least one derivation had a timeout. + */ bool timedOut; - /* Set if at least one derivation fails with a hash mismatch. */ + /** + * Set if at least one derivation fails with a hash mismatch. + */ bool hashMismatch; - /* Set if at least one derivation is not deterministic in check mode. */ + /** + * Set if at least one derivation is not deterministic in check mode. + */ bool checkMismatch; Store & store; @@ -128,16 +167,22 @@ public: uint64_t expectedNarSize = 0; uint64_t doneNarSize = 0; - /* Whether to ask the build hook if it can build a derivation. If - it answers with "decline-permanently", we don't try again. */ + /** + * Whether to ask the build hook if it can build a derivation. If + * it answers with "decline-permanently", we don't try again. + */ bool tryBuildHook = true; Worker(Store & store, Store & evalStore); ~Worker(); - /* Make a goal (with caching). */ + /** + * Make a goal (with caching). + */ - /* derivation goal */ + /** + * derivation goal + */ private: std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -150,56 +195,80 @@ public: const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); - /* substitution goal */ + /** + * substitution goal + */ std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); - /* Remove a dead goal. */ + /** + * Remove a dead goal. + */ void removeGoal(GoalPtr goal); - /* Wake up a goal (i.e., there is something for it to do). */ + /** + * Wake up a goal (i.e., there is something for it to do). + */ void wakeUp(GoalPtr goal); - /* Return the number of local build and substitution processes - currently running (but not remote builds via the build - hook). */ + /** + * Return the number of local build and substitution processes + * currently running (but not remote builds via the build + * hook). + */ unsigned int getNrLocalBuilds(); - /* Registers a running child process. `inBuildSlot' means that - the process counts towards the jobs limit. */ + /** + * Registers a running child process. `inBuildSlot` means that + * the process counts towards the jobs limit. + */ void childStarted(GoalPtr goal, const std::set<int> & fds, bool inBuildSlot, bool respectTimeouts); - /* Unregisters a running child process. `wakeSleepers' should be - false if there is no sense in waking up goals that are sleeping - because they can't run yet (e.g., there is no free build slot, - or the hook would still say `postpone'). */ + /** + * Unregisters a running child process. `wakeSleepers` should be + * false if there is no sense in waking up goals that are sleeping + * because they can't run yet (e.g., there is no free build slot, + * or the hook would still say `postpone`). + */ void childTerminated(Goal * goal, bool wakeSleepers = true); - /* Put `goal' to sleep until a build slot becomes available (which - might be right away). */ + /** + * Put `goal` to sleep until a build slot becomes available (which + * might be right away). + */ void waitForBuildSlot(GoalPtr goal); - /* Wait for any goal to finish. Pretty indiscriminate way to - wait for some resource that some other goal is holding. */ + /** + * Wait for any goal to finish. Pretty indiscriminate way to + * wait for some resource that some other goal is holding. + */ void waitForAnyGoal(GoalPtr goal); - /* Wait for a few seconds and then retry this goal. Used when - waiting for a lock held by another process. This kind of - polling is inefficient, but POSIX doesn't really provide a way - to wait for multiple locks in the main select() loop. */ + /** + * Wait for a few seconds and then retry this goal. Used when + * waiting for a lock held by another process. This kind of + * polling is inefficient, but POSIX doesn't really provide a way + * to wait for multiple locks in the main select() loop. + */ void waitForAWhile(GoalPtr goal); - /* Loop until the specified top-level goals have finished. */ + /** + * Loop until the specified top-level goals have finished. + */ void run(const Goals & topGoals); - /* Wait for input to become available. */ + /** + * Wait for input to become available. + */ void waitForInput(); unsigned int exitStatus(); - /* Check whether the given valid path exists and has the right - contents. */ + /** + * Check whether the given valid path exists and has the right + * contents. + */ bool pathContentsGood(const StorePath & path); void markContentsGood(const StorePath & path); diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 66597e456..d201fb3ac 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index a018de3af..0923c2adb 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 7704d2f00..962b63e83 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <variant> #include "hash.hh" diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 03f85c103..35216d470 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" @@ -11,8 +12,10 @@ struct Key std::string name; std::string key; - /* Construct Key from a string in the format - ‘<name>:<key-in-base64>’. */ + /** + * Construct Key from a string in the format + * ‘<name>:<key-in-base64>’. + */ Key(std::string_view s); std::string to_string() const; @@ -28,7 +31,9 @@ struct SecretKey : Key { SecretKey(std::string_view s); - /* Return a detached signature of the given string. */ + /** + * Return a detached signature of the given string. + */ std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; @@ -52,8 +57,10 @@ private: typedef std::map<std::string, PublicKey> PublicKeys; -/* Return true iff ‘sig’ is a correct signature over ‘data’ using one - of the given public keys. */ +/** + * @return true iff ‘sig’ is a correct signature over ‘data’ using one + * of the given public keys. + */ bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 0e2169035..139a05587 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1036,6 +1036,15 @@ void processConnection( if (GET_PROTOCOL_MINOR(clientVersion) >= 33) to << nixVersion; + if (GET_PROTOCOL_MINOR(clientVersion) >= 35) { + // We and the underlying store both need to trust the client for + // it to be trusted. + auto temp = trusted + ? store->isTrustedClient() + : std::optional { NotTrusted }; + worker_proto::write(*store, to, temp); + } + /* Send startup error messages to the client. */ tunnelLogger->startWork(); diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 8c765615c..1964c0d99 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -1,11 +1,11 @@ #pragma once +///@file #include "serialise.hh" #include "store-api.hh" namespace nix::daemon { -enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index aaae864f4..56f30f2e4 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -312,6 +312,15 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi } +/** + * Print a derivation string literal to an `std::string`. + * + * This syntax does not generalize to the expression language, which needs to + * escape `$`. + * + * @param res Where to print to + * @param s Which logical string to print + */ static void printString(std::string & res, std::string_view s) { boost::container::small_vector<char, 64 * 1024> buffer; @@ -888,6 +897,67 @@ std::optional<BasicDerivation> Derivation::tryResolve( return resolved; } + +void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const +{ + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); + drvName = drvName.substr(0, drvName.size() - drvExtension.size()); + + if (drvName != name) { + throw Error("Derivation '%s' has name '%s' which does not match its path", store.printStorePath(drvPath), name); + } + + auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) + { + auto j = env.find(varName); + if (j == env.end() || store.parseStorePath(j->second) != actual) + throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", + store.printStorePath(drvPath), varName, store.printStorePath(actual)); + }; + + + // Don't need the answer, but do this anyways to assert is proper + // combination. The code below is more general and naturally allows + // combinations that are currently prohibited. + type(); + + std::optional<DrvHash> hashesModulo; + for (auto & i : outputs) { + std::visit(overloaded { + [&](const DerivationOutput::InputAddressed & doia) { + if (!hashesModulo) { + // somewhat expensive so we do lazily + hashesModulo = hashDerivationModulo(store, *this, true); + } + auto currentOutputHash = get(hashesModulo->hashes, i.first); + if (!currentOutputHash) + throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), i.first); + StorePath recomputed = store.makeOutputPath(i.first, *currentOutputHash, drvName); + if (doia.path != recomputed) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + store.printStorePath(drvPath), store.printStorePath(doia.path), store.printStorePath(recomputed)); + envHasRightPath(doia.path, i.first); + }, + [&](const DerivationOutput::CAFixed & dof) { + auto path = dof.path(store, drvName, i.first); + envHasRightPath(path, i.first); + }, + [&](const DerivationOutput::CAFloating &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ + }, + }, i.second.raw()); + } +} + + const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( @@ -916,10 +986,79 @@ nlohmann::json DerivationOutput::toJSON( return res; } + +DerivationOutput DerivationOutput::fromJSON( + const Store & store, std::string_view drvName, std::string_view outputName, + const nlohmann::json & _json) +{ + std::set<std::string_view> keys; + auto json = (std::map<std::string, nlohmann::json>) _json; + + for (const auto & [key, _] : json) + keys.insert(key); + + auto methodAlgo = [&]() -> std::pair<FileIngestionMethod, HashType> { + std::string hashAlgo = json["hashAlgo"]; + auto method = FileIngestionMethod::Flat; + if (hashAlgo.substr(0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = hashAlgo.substr(2); + } + auto hashType = parseHashType(hashAlgo); + return { method, hashType }; + }; + + if (keys == (std::set<std::string_view> { "path" })) { + return DerivationOutput::InputAddressed { + .path = store.parseStorePath((std::string) json["path"]), + }; + } + + else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) { + auto [method, hashType] = methodAlgo(); + auto dof = DerivationOutput::CAFixed { + .ca = ContentAddressWithReferences::fromParts( + method, + Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + {}), + }; + if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) + throw Error("Path doesn't match derivation output"); + return dof; + } + + else if (keys == (std::set<std::string_view> { "hashAlgo" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::CAFloating { + .method = method, + .hashType = hashType, + }; + } + + else if (keys == (std::set<std::string_view> { })) { + return DerivationOutput::Deferred {}; + } + + else if (keys == (std::set<std::string_view> { "hashAlgo", "impure" })) { + auto [method, hashType] = methodAlgo(); + return DerivationOutput::Impure { + .method = method, + .hashType = hashType, + }; + } + + else { + throw Error("invalid JSON for derivation output"); + } +} + + nlohmann::json Derivation::toJSON(const Store & store) const { nlohmann::json res = nlohmann::json::object(); + res["name"] = name; + { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); @@ -950,4 +1089,43 @@ nlohmann::json Derivation::toJSON(const Store & store) const return res; } + +Derivation Derivation::fromJSON( + const Store & store, + const nlohmann::json & json) +{ + Derivation res; + + res.name = json["name"]; + + { + auto & outputsObj = json["outputs"]; + for (auto & [outputName, output] : outputsObj.items()) { + res.outputs.insert_or_assign( + outputName, + DerivationOutput::fromJSON(store, res.name, outputName, output)); + } + } + + { + auto & inputsList = json["inputSrcs"]; + for (auto & input : inputsList) + res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input))); + } + + { + auto & inputDrvsObj = json["inputDrvs"]; + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + res.inputDrvs[store.parseStorePath(inputDrvPath)] = + static_cast<const StringSet &>(inputOutputs); + } + + res.platform = json["system"]; + res.builder = json["builder"]; + res.args = json["args"]; + res.env = json["env"]; + + return res; +} + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index dff6b472c..65901ec6d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "path.hh" #include "types.hh" @@ -6,6 +7,7 @@ #include "content-address.hh" #include "repair-flag.hh" #include "sync.hh" +#include "comparator.hh" #include <map> #include <variant> @@ -23,6 +25,8 @@ class Store; struct DerivationOutputInputAddressed { StorePath path; + + GENERATE_CMP(DerivationOutputInputAddressed, me->path); }; /** @@ -43,6 +47,8 @@ struct DerivationOutputCAFixed * @param outputName The name of this output. */ StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + + GENERATE_CMP(DerivationOutputCAFixed, me->ca); }; /** @@ -61,13 +67,17 @@ struct DerivationOutputCAFloating * How the serialization will be hashed */ HashType hashType; + + GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType); }; /** * Input-addressed output which depends on a (CA) derivation whose hash * isn't known yet. */ -struct DerivationOutputDeferred {}; +struct DerivationOutputDeferred { + GENERATE_CMP(DerivationOutputDeferred); +}; /** * Impure output which is moved to a content-addressed location (like @@ -84,6 +94,8 @@ struct DerivationOutputImpure * How the serialization will be hashed */ HashType hashType; + + GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType); }; typedef std::variant< @@ -124,6 +136,11 @@ struct DerivationOutput : _DerivationOutputRaw const Store & store, std::string_view drvName, std::string_view outputName) const; + static DerivationOutput fromJSON( + const Store & store, + std::string_view drvName, + std::string_view outputName, + const nlohmann::json & json); }; typedef std::map<std::string, DerivationOutput> DerivationOutputs; @@ -241,8 +258,14 @@ struct DerivationType : _DerivationTypeRaw { struct BasicDerivation { - DerivationOutputs outputs; /* keyed on symbolic IDs */ - StorePathSet inputSrcs; /* inputs that are sources */ + /** + * keyed on symbolic IDs + */ + DerivationOutputs outputs; + /** + * inputs that are sources + */ + StorePathSet inputSrcs; std::string platform; Path builder; Strings args; @@ -272,6 +295,15 @@ struct BasicDerivation DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; static std::string_view nameFromPath(const StorePath & storePath); + + GENERATE_CMP(BasicDerivation, + me->outputs, + me->inputSrcs, + me->platform, + me->builder, + me->args, + me->env, + me->name); }; struct Derivation : BasicDerivation @@ -307,11 +339,26 @@ struct Derivation : BasicDerivation Store & store, const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const; + /* Check that the derivation is valid and does not present any + illegal states. + + This is mainly a matter of checking the outputs, where our C++ + representation supports all sorts of combinations we do not yet + allow. */ + void checkInvariants(Store & store, const StorePath & drvPath) const; + Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } nlohmann::json toJSON(const Store & store) const; + static Derivation fromJSON( + const Store & store, + const nlohmann::json & json); + + GENERATE_CMP(Derivation, + static_cast<const BasicDerivation &>(*me), + me->inputDrvs); }; @@ -388,12 +435,12 @@ void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; * * A fixed-output derivation is a derivation whose outputs have a * specified content hash and hash algorithm. (Currently they must have - * exactly one output (`out'), which is specified using the `outputHash' - * and `outputHashAlgo' attributes, but the algorithm doesn't assume + * exactly one output (`out`), which is specified using the `outputHash` + * and `outputHashAlgo` attributes, but the algorithm doesn't assume * this.) We don't want changes to such derivations to propagate upwards * through the dependency graph, changing output paths everywhere. * - * For instance, if we change the url in a call to the `fetchurl' + * For instance, if we change the url in a call to the `fetchurl` * function, we do not want to rebuild everything depending on it---after * all, (the hash of) the file being downloaded is unchanged. So the * *output paths* should not change. On the other hand, the *derivation diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index e5f0f1b33..9a2ffda39 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -62,15 +62,31 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const std::string DerivedPath::Built::to_string(const Store & store) const { return store.printStorePath(drvPath) - + "!" + + '^' + + outputs.to_string(); +} + +std::string DerivedPath::Built::to_string_legacy(const Store & store) const +{ + return store.printStorePath(drvPath) + + '!' + outputs.to_string(); } std::string DerivedPath::to_string(const Store & store) const { - return std::visit( - [&](const auto & req) { return req.to_string(store); }, - this->raw()); + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); +} + +std::string DerivedPath::to_string_legacy(const Store & store) const +{ + return std::visit(overloaded { + [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + }, this->raw()); } @@ -87,14 +103,24 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi }; } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) { - size_t n = s.find("!"); + size_t n = s.find(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); } +DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +{ + return parseWith(store, s, "^"); +} + +DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +{ + return parseWith(store, s, "!"); +} + RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const { RealisedPath::Set res; diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 72dbcc128..5f7acbebc 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "path.hh" @@ -47,8 +48,18 @@ struct DerivedPathBuilt { StorePath drvPath; OutputsSpec outputs; + /** + * Uses `^` as the separator + */ std::string to_string(const Store & store) const; - static DerivedPathBuilt parse(const Store & store, std::string_view, std::string_view); + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + */ + static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); nlohmann::json toJSON(ref<Store> store) const; GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); @@ -80,8 +91,22 @@ struct DerivedPath : _DerivedPathRaw { return static_cast<const Raw &>(*this); } + /** + * Uses `^` as the separator + */ std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + */ static DerivedPath parse(const Store & store, std::string_view); + /** + * Uses `!` as the separator + */ + static DerivedPath parseLegacy(const Store & store, std::string_view); }; /** diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 16e5fafd7..74d6ed3b5 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -39,6 +39,14 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store callback(nullptr); } + /** + * The dummy store is incapable of *not* trusting! :) + */ + virtual std::optional<TrustedFlag> isTrustedClient() override + { + return Trusted; + } + static std::set<std::string> uriSchemes() { return {"dummy"}; } @@ -63,6 +71,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void queryRealisationUncached(const DrvOutput &, Callback<std::shared_ptr<const Realisation>> callback) noexcept override { callback(nullptr); } + + virtual ref<FSAccessor> getFSAccessor() override + { unsupported("getFSAccessor"); } }; static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 1ba399a29..2346accbe 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -407,6 +407,10 @@ struct curlFileTransfer : public FileTransfer err = Misc; } else { // Don't bother retrying on certain cURL errors either + + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" switch (code) { case CURLE_FAILED_INIT: case CURLE_URL_MALFORMAT: @@ -427,6 +431,7 @@ struct curlFileTransfer : public FileTransfer default: // Shut up warnings break; } + #pragma GCC diagnostic pop } attempt++; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 07d58f53a..378c6ff78 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" @@ -87,39 +88,56 @@ struct FileTransfer { virtual ~FileTransfer() { } - /* Enqueue a data transfer request, returning a future to the result of - the download. The future may throw a FileTransferError - exception. */ + /** + * Enqueue a data transfer request, returning a future to the result of + * the download. The future may throw a FileTransferError + * exception. + */ virtual void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) = 0; std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request); - /* Synchronously download a file. */ + /** + * Synchronously download a file. + */ FileTransferResult download(const FileTransferRequest & request); - /* Synchronously upload a file. */ + /** + * Synchronously upload a file. + */ FileTransferResult upload(const FileTransferRequest & request); - /* Download a file, writing its data to a sink. The sink will be - invoked on the thread of the caller. */ + /** + * Download a file, writing its data to a sink. The sink will be + * invoked on the thread of the caller. + */ void download(FileTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared FileTransfer object. Using this object is preferred - because it enables connection reuse and HTTP/2 multiplexing. */ +/** + * @return a shared FileTransfer object. + * + * Using this object is preferred because it enables connection reuse + * and HTTP/2 multiplexing. + */ ref<FileTransfer> getFileTransfer(); -/* Return a new FileTransfer object. */ +/** + * @return a new FileTransfer object + * + * Prefer getFileTransfer() to this; see its docs for why. + */ ref<FileTransfer> makeFileTransfer(); class FileTransferError : public Error { public: FileTransfer::Error error; - std::optional<std::string> response; // intentionally optional + /// intentionally optional + std::optional<std::string> response; template<typename... Args> FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args); diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index c825e84f2..1df19e647 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -1,11 +1,14 @@ #pragma once +///@file #include "types.hh" namespace nix { -/* An abstract class for accessing a filesystem-like structure, such - as a (possibly remote) Nix store or the contents of a NAR file. */ +/** + * An abstract class for accessing a filesystem-like structure, such + * as a (possibly remote) Nix store or the contents of a NAR file. + */ class FSAccessor { public: @@ -14,8 +17,17 @@ public: struct Stat { Type type = tMissing; - uint64_t fileSize = 0; // regular files only + /** + * regular files only + */ + uint64_t fileSize = 0; + /** + * regular files only + */ bool isExecutable = false; // regular files only + /** + * regular files only + */ uint64_t narOffset = 0; // regular files only }; diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index b3cbbad74..2c26c65c4 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" @@ -11,19 +12,20 @@ typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots; struct GCOptions { - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ + /** + * Garbage collector operation: + * + * - `gcReturnLive`: return the set of paths reachable from + * (i.e. in the closure of) the roots. + * + * - `gcReturnDead`: return the set of paths not reachable from + * the roots. + * + * - `gcDeleteDead`: actually delete the latter set. + * + * - `gcDeleteSpecific`: delete the paths listed in + * `pathsToDelete`, insofar as they are not reachable. + */ typedef enum { gcReturnLive, gcReturnDead, @@ -33,28 +35,38 @@ struct GCOptions GCAction action{gcDeleteDead}; - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ + /** + * If `ignoreLiveness` is set, then reachability from the roots is + * ignored (dangerous!). However, the paths must still be + * unreferenced *within* the store (i.e., there can be no other + * store paths that depend on them). + */ bool ignoreLiveness{false}; - /* For `gcDeleteSpecific', the paths to delete. */ + /** + * For `gcDeleteSpecific`, the paths to delete. + */ StorePathSet pathsToDelete; - /* Stop after at least `maxFreed' bytes have been freed. */ + /** + * Stop after at least `maxFreed` bytes have been freed. + */ uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; }; struct GCResults { - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ + /** + * Depending on the action, the GC roots, or the paths that would + * be or have been deleted. + */ PathSet paths; - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ + /** + * For `gcReturnDead`, `gcDeleteDead` and `gcDeleteSpecific`, the + * number of bytes that would be or was freed. + */ uint64_t bytesFreed = 0; }; @@ -63,21 +75,27 @@ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ + /** + * Add an indirect root, which is merely a symlink to `path` from + * `/nix/var/nix/gcroots/auto/<hash of path>`. `path` is supposed + * to be a symlink to a store path. The garbage collector will + * automatically remove the indirect root when it finds that + * `path` has disappeared. + */ virtual void addIndirectRoot(const Path & path) = 0; - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ + /** + * Find the roots of the garbage collector. Each root is a pair + * `(link, storepath)` where `link` is the path of the symlink + * outside of the Nix store that point to `storePath`. If + * `censor` is true, privacy-sensitive information about roots + * found in `/proc` is censored. + */ virtual Roots findRoots(bool censor) = 0; - /* Perform a garbage collection. */ + /** + * Perform a garbage collection. + */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; }; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 823b4af74..1b38e32fb 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -7,12 +7,20 @@ #include <algorithm> #include <map> +#include <mutex> #include <thread> #include <dlfcn.h> #include <sys/utsname.h> #include <nlohmann/json.hpp> +#include <sodium/core.h> + +#ifdef __GLIBC__ +#include <gnu/lib-names.h> +#include <nss.h> +#include <dlfcn.h> +#endif namespace nix { @@ -41,7 +49,6 @@ Settings::Settings() , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; - lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); @@ -281,6 +288,42 @@ void initPlugins() settings.pluginFiles.pluginsLoaded = true; } +static void preloadNSS() +{ + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + static std::once_flag dns_resolve_flag; + + std::call_once(dns_resolve_flag, []() { +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) + warn("unable to load nss_dns backend"); + // FIXME: get hosts entry from nsswitch.conf. + __nss_configure_lookup("hosts", "files dns"); +#endif + }); +} + static bool initLibStoreDone = false; void assertLibStoreInitialized() { @@ -291,6 +334,24 @@ void assertLibStoreInitialized() { } void initLibStore() { + + initLibUtil(); + + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); + + loadConfFile(); + + preloadNSS(); + + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ +#if __APPLE__ + if (hasPrefix(getEnv("TMPDIR").value_or("/tmp"), "/var/folders/")) + unsetenv("TMPDIR"); +#endif + initLibStoreDone = true; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 299584f99..d6c5d437a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "config.hh" @@ -71,30 +72,46 @@ public: Path nixPrefix; - /* The directory where we store sources and derived files. */ + /** + * The directory where we store sources and derived files. + */ Path nixStore; Path nixDataDir; /* !!! fix */ - /* The directory where we log various operations. */ + /** + * The directory where we log various operations. + */ Path nixLogDir; - /* The directory where state is stored. */ + /** + * The directory where state is stored. + */ Path nixStateDir; - /* The directory where system configuration files are stored. */ + /** + * The directory where system configuration files are stored. + */ Path nixConfDir; - /* A list of user configuration files to load. */ + /** + * A list of user configuration files to load. + */ std::vector<Path> nixUserConfFiles; - /* The directory where the main programs are stored. */ + /** + * The directory where the main programs are stored. + */ Path nixBinDir; - /* The directory where the man pages are stored. */ + /** + * The directory where the man pages are stored. + */ Path nixManDir; - /* File name of the socket the daemon listens to. */ + /** + * File name of the socket the daemon listens to. + */ Path nixDaemonSocketFile; Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", @@ -120,7 +137,9 @@ public: )", {"build-fallback"}}; - /* Whether to show build log output in real time. */ + /** + * Whether to show build log output in real time. + */ bool verboseBuild = true; Setting<size_t> logLines{this, 10, "log-lines", @@ -156,8 +175,10 @@ public: )", {"build-cores"}, false}; - /* Read-only mode. Don't copy stuff to the store, don't change - the database. */ + /** + * Read-only mode. Don't copy stuff to the store, don't change + * the database. + */ bool readOnlyMode = false; Setting<std::string> thisSystem{ @@ -307,16 +328,6 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = auto-allocate-uids - auto-allocate-uids = true - ``` )"}; Setting<uint32_t> startId{this, @@ -346,16 +357,6 @@ public: Cgroups are required and enabled automatically for derivations that require the `uid-range` system feature. - - > **Warning** - > This is an experimental feature. - - To enable it, add the following to [`nix.conf`](#): - - ``` - extra-experimental-features = cgroups - use-cgroups = true - ``` )"}; #endif @@ -457,9 +458,6 @@ public: )", {"env-keep-derivations"}}; - /* Whether to lock the Nix client and worker to the same CPU. */ - bool lockCPU; - Setting<SandboxMode> sandboxMode{ this, #if __linux__ @@ -996,8 +994,10 @@ public: // FIXME: don't use a global variable. extern Settings settings; -/* This should be called after settings are initialized, but before - anything else */ +/** + * This should be called after settings are initialized, but before + * anything else + */ void initPlugins(); void loadConfFile(); @@ -1007,12 +1007,16 @@ std::vector<Path> getUserConfigFiles(); extern const std::string nixVersion; -/* NB: This is not sufficient. You need to call initNix() */ +/** + * NB: This is not sufficient. You need to call initNix() + */ void initLibStore(); -/* It's important to initialize before doing _anything_, which is why we - call upon the programmer to handle this correctly. However, we only add - this in a key locations, so as not to litter the code. */ +/** + * It's important to initialize before doing _anything_, which is why we + * call upon the programmer to handle this correctly. However, we only add + * this in a key locations, so as not to litter the code. + */ void assertLibStoreInitialized(); } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 238fd1d98..85c5eed4c 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -194,6 +194,18 @@ protected: }}); } + /** + * This isn't actually necessary read only. We support "upsert" now, so we + * have a notion of authentication via HTTP POST/PUT. + * + * For now, we conservatively say we don't know. + * + * \todo try to expose our HTTP authentication status. + */ + std::optional<TrustedFlag> isTrustedClient() override + { + return std::nullopt; + } }; static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regHttpBinaryCacheStore; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index a1c38d180..c3cb3032a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -342,6 +342,9 @@ public: void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } + virtual ref<FSAccessor> getFSAccessor() override + { unsupported("getFSAccessor"); } + void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override @@ -389,6 +392,15 @@ public: return conn->remoteVersion; } + /** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ + std::optional<TrustedFlag> isTrustedClient() override + { + return std::nullopt; + } + void queryRealisationUncached(const DrvOutput &, Callback<std::shared_ptr<const Realisation>> callback) noexcept override // TODO: Implement diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index e5ee6fc15..5481dd762 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -95,6 +95,10 @@ protected: return paths; } + std::optional<TrustedFlag> isTrustedClient() override + { + return Trusted; + } }; void LocalBinaryCacheStore::init() diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 796e72045..a03bb88f5 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "gc-store.hh" @@ -47,7 +48,9 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; ref<FSAccessor> getFSAccessor() override; - /* Register a permanent GC root. */ + /** + * Register a permanent GC root. + */ Path addPermRoot(const StorePath & storePath, const Path & gcRoot); virtual Path getRealStoreDir() { return realStoreDir; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 63039e6ad..7fb312c37 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -711,61 +711,6 @@ void canonicalisePathMetaData(const Path & path, } -void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) -{ - assert(drvPath.isDerivation()); - std::string drvName(drvPath.name()); - drvName = drvName.substr(0, drvName.size() - drvExtension.size()); - - auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) - { - auto j = drv.env.find(varName); - if (j == drv.env.end() || parseStorePath(j->second) != actual) - throw Error("derivation '%s' has incorrect environment variable '%s', should be '%s'", - printStorePath(drvPath), varName, printStorePath(actual)); - }; - - - // Don't need the answer, but do this anyways to assert is proper - // combination. The code below is more general and naturally allows - // combinations that are currently prohibited. - drv.type(); - - std::optional<DrvHash> hashesModulo; - for (auto & i : drv.outputs) { - std::visit(overloaded { - [&](const DerivationOutput::InputAddressed & doia) { - if (!hashesModulo) { - // somewhat expensive so we do lazily - hashesModulo = hashDerivationModulo(*this, drv, true); - } - auto currentOutputHash = get(hashesModulo->hashes, i.first); - if (!currentOutputHash) - throw Error("derivation '%s' has unexpected output '%s' (local-store / hashesModulo) named '%s'", - printStorePath(drvPath), printStorePath(doia.path), i.first); - StorePath recomputed = makeOutputPath(i.first, *currentOutputHash, drvName); - if (doia.path != recomputed) - throw Error("derivation '%s' has incorrect output '%s', should be '%s'", - printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); - envHasRightPath(doia.path, i.first); - }, - [&](const DerivationOutput::CAFixed & dof) { - auto path = dof.path(*this, drvName, i.first); - envHasRightPath(path, i.first); - }, - [&](const DerivationOutput::CAFloating &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Deferred &) { - /* Nothing to check */ - }, - [&](const DerivationOutput::Impure &) { - /* Nothing to check */ - }, - }, i.second.raw()); - } -} - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -876,7 +821,7 @@ uint64_t LocalStore::addValidPath(State & state, derivations). Note that if this throws an error, then the DB transaction is rolled back, so the path validity registration above is undone. */ - if (checkOutputs) checkDerivationOutputs(info.path, drv); + if (checkOutputs) drv.checkInvariants(*this, info.path); for (auto & i : drv.outputsAndOptPaths(*this)) { /* Floating CA derivations have indeterminate output paths until @@ -1134,57 +1079,6 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) } -// FIXME: move this, it's not specific to LocalStore. -void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) -{ - if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - for (auto & path : paths) { - if (infos.count(path.first)) - // Choose first succeeding substituter. - continue; - - auto subPath(path.first); - - // Recompute store path so that we can use a different store root. - if (path.second) { - subPath = makeFixedOutputPathFromCA( - path.first.name(), - ContentAddressWithReferences::withoutRefs(*path.second)); - if (sub->storeDir == storeDir) - assert(subPath == path.first); - if (subPath != path.first) - debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); - } else if (sub->storeDir != storeDir) continue; - - debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); - try { - auto info = sub->queryPathInfo(subPath); - - if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) - continue; - - auto narInfo = std::dynamic_pointer_cast<const NarInfo>( - std::shared_ptr<const ValidPathInfo>(info)); - infos.insert_or_assign(path.first, SubstitutablePathInfo{ - .deriver = info->deriver, - .references = info->references, - .downloadSize = narInfo ? narInfo->fileSize : 0, - .narSize = info->narSize, - }); - } catch (InvalidPath &) { - } catch (SubstituterDisabled &) { - } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; - } - } - } -} - - void LocalStore::registerValidPath(const ValidPathInfo & info) { registerValidPaths({{info.path, info}}); @@ -1226,8 +1120,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readInvalidDerivation(i.path)); + readInvalidDerivation(i.path).checkInvariants(*this, i.path); } /* Do a topological sort of the paths. This will throw an @@ -1435,6 +1328,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name }, .references = { .others = references, + // caller is not capable of creating a self-reference, because this is content-addressed without modulus .self = false, }, }; @@ -1753,6 +1647,11 @@ unsigned int LocalStore::getProtocol() return PROTOCOL_VERSION; } +std::optional<TrustedFlag> LocalStore::isTrustedClient() +{ + return Trusted; +} + #if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 639772b36..55add18dd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "sqlite.hh" @@ -18,10 +19,14 @@ namespace nix { -/* Nix store and database schema version. Version 1 (or 0) was Nix <= - 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is - Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ +/** + * Nix store and database schema version. + * + * Version 1 (or 0) was Nix <= + * 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + * Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + * Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. + */ const int nixSchemaVersion = 10; @@ -50,30 +55,40 @@ class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, { private: - /* Lock file used for upgrading. */ + /** + * Lock file used for upgrading. + */ AutoCloseFD globalLock; struct State { - /* The SQLite database object. */ + /** + * The SQLite database object. + */ SQLite db; struct Stmts; std::unique_ptr<Stmts> stmts; - /* The last time we checked whether to do an auto-GC, or an - auto-GC finished. */ + /** + * The last time we checked whether to do an auto-GC, or an + * auto-GC finished. + */ std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; - /* Whether auto-GC is running. If so, get gcFuture to wait for - the GC to finish. */ + /** + * Whether auto-GC is running. If so, get gcFuture to wait for + * the GC to finish. + */ bool gcRunning = false; std::shared_future<void> gcFuture; - /* How much disk space was available after the previous - auto-GC. If the current available disk space is below - minFree but not much below availAfterGC, then there is no - point in starting a new GC. */ + /** + * How much disk space was available after the previous + * auto-GC. If the current available disk space is below + * minFree but not much below availAfterGC, then there is no + * point in starting a new GC. + */ uint64_t availAfterGC = std::numeric_limits<uint64_t>::max(); std::unique_ptr<PublicKeys> publicKeys; @@ -96,11 +111,15 @@ private: public: - // Hack for build-remote.cc. + /** + * Hack for build-remote.cc. + */ PathSet locksHeld; - /* Initialise the local store, upgrading the schema if - necessary. */ + /** + * Initialise the local store, upgrading the schema if + * necessary. + */ LocalStore(const Params & params); LocalStore(std::string scheme, std::string path, const Params & params); @@ -109,7 +128,9 @@ public: static std::set<std::string> uriSchemes() { return {}; } - /* Implementations of abstract store API methods. */ + /** + * Implementations of abstract store API methods. + */ std::string getUri() override; @@ -133,9 +154,6 @@ public: StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) override; - bool pathInfoIsUntrusted(const ValidPathInfo &) override; bool realisationIsUntrusted(const Realisation & ) override; @@ -157,13 +175,19 @@ private: void createTempRootsFile(); - /* The file to which we write our temporary roots. */ + /** + * The file to which we write our temporary roots. + */ Sync<AutoCloseFD> _fdTempRoots; - /* The global GC lock. */ + /** + * The global GC lock. + */ Sync<AutoCloseFD> _fdGCLock; - /* Connection to the garbage collector. */ + /** + * Connection to the garbage collector. + */ Sync<AutoCloseFD> _fdRootsSocket; public: @@ -182,42 +206,54 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; - /* Optimise the disk space usage of the Nix store by hard-linking - files with the same contents. */ + /** + * Optimise the disk space usage of the Nix store by hard-linking + * files with the same contents. + */ void optimiseStore(OptimiseStats & stats); void optimiseStore() override; - /* Optimise a single store path. Optionally, test the encountered - symlinks for corruption. */ + /** + * Optimise a single store path. Optionally, test the encountered + * symlinks for corruption. + */ void optimisePath(const Path & path, RepairFlag repair); bool verifyStore(bool checkContents, RepairFlag repair) override; - /* Register the validity of a path, i.e., that `path' exists, that - the paths referenced by it exists, and in the case of an output - path of a derivation, that it has been produced by a successful - execution of the derivation (or something equivalent). Also - register the hash of the file system contents of the path. The - hash must be a SHA-256 hash. */ + /** + * Register the validity of a path, i.e., that `path` exists, that + * the paths referenced by it exists, and in the case of an output + * path of a derivation, that it has been produced by a successful + * execution of the derivation (or something equivalent). Also + * register the hash of the file system contents of the path. The + * hash must be a SHA-256 hash. + */ void registerValidPath(const ValidPathInfo & info); void registerValidPaths(const ValidPathInfos & infos); unsigned int getProtocol() override; + std::optional<TrustedFlag> isTrustedClient() override; + void vacuumDB(); void repairPath(const StorePath & path) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - /* If free disk space in /nix/store if below minFree, delete - garbage until it exceeds maxFree. */ + /** + * If free disk space in /nix/store if below minFree, delete + * garbage until it exceeds maxFree. + */ void autoGC(bool sync = true); - /* Register the store path 'output' as the output named 'outputName' of - derivation 'deriver'. */ + /** + * Register the store path 'output' as the output named 'outputName' of + * derivation 'deriver'. + */ void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping( @@ -247,7 +283,9 @@ private: void invalidatePath(State & state, const StorePath & path); - /* Delete a path from the Nix store. */ + /** + * Delete a path from the Nix store. + */ void invalidatePathChecked(const StorePath & path); void verifyPath(const Path & path, const StringSet & store, @@ -270,8 +308,6 @@ private: std::pair<Path, AutoCloseFD> createTempDirInStore(); - void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); - typedef std::unordered_set<ino_t> InodeHash; InodeHash loadInodeHash(); @@ -282,8 +318,10 @@ private: bool isValidPath_(State & state, const StorePath & path); void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); - /* Add signatures to a ValidPathInfo or Realisation using the secret keys - specified by the ‘secret-key-files’ option. */ + /** + * Add signatures to a ValidPathInfo or Realisation using the secret keys + * specified by the ‘secret-key-files’ option. + */ void signPathInfo(ValidPathInfo & info); void signRealisation(Realisation &); @@ -313,18 +351,23 @@ typedef std::pair<dev_t, ino_t> Inode; typedef std::set<Inode> InodesSeen; -/* "Fix", or canonicalise, the meta-data of the files in a store path - after it has been built. In particular: - - the last modification date on each file is set to 1 (i.e., - 00:00:01 1/1/1970 UTC) - - the permissions are set of 444 or 555 (i.e., read-only with or - without execute permission; setuid bits etc. are cleared) - - the owner and group are set to the Nix user and group, if we're - running as root. - If uidRange is not empty, this function will throw an error if it - encounters files owned by a user outside of the closed interval - [uidRange->first, uidRange->second]. -*/ +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ void canonicalisePathMetaData( const Path & path, std::optional<std::pair<uid_t, uid_t>> uidRange, diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 7f1934510..1c268e1fb 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" @@ -12,14 +13,18 @@ struct UserLock { virtual ~UserLock() { } - /* Get the first and last UID. */ + /** + * Get the first and last UID. + */ std::pair<uid_t, uid_t> getUIDRange() { auto first = getUID(); return {first, first + getUIDCount() - 1}; } - /* Get the first UID. */ + /** + * Get the first UID. + */ virtual uid_t getUID() = 0; virtual uid_t getUIDCount() = 0; @@ -29,8 +34,10 @@ struct UserLock virtual std::vector<gid_t> getSupplementaryGIDs() = 0; }; -/* Acquire a user lock for a UID range of size `nrIds`. Note that this - may return nullptr if no user is available. */ +/** + * Acquire a user lock for a UID range of size `nrIds`. Note that this + * may return nullptr if no user is available. + */ std::unique_ptr<UserLock> acquireUserLock(uid_t nrIds, bool useUserNamespace); bool useBuildUsers(); diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh index e4d95bab6..a84f7dbeb 100644 --- a/src/libstore/log-store.hh +++ b/src/libstore/log-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" @@ -9,8 +10,10 @@ struct LogStore : public virtual Store { inline static std::string operationName = "Build log storage and retrieval"; - /* Return the build log of the specified store path, if available, - or null otherwise. */ + /** + * Return the build log of the specified store path, if available, + * or null otherwise. + */ std::optional<std::string> getBuildLog(const StorePath & path); virtual std::optional<std::string> getBuildLogExact(const StorePath & path) = 0; diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 834626de9..1adeaf1f0 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh index c4a66ed41..2ce6ec7bc 100644 --- a/src/libstore/make-content-addressed.hh +++ b/src/libstore/make-content-addressed.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" diff --git a/src/libstore/names.hh b/src/libstore/names.hh index 3977fc6cc..d82b99bb4 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <memory> diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 9a0003588..f0dfcb19b 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -275,6 +275,7 @@ json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse) obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; + case FSAccessor::Type::tMissing: default: throw Error("path '%s' does not exist in NAR", path); } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 7d998ae0b..5e19bd3c7 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <functional> @@ -9,24 +10,30 @@ namespace nix { struct Source; -/* Return an object that provides access to the contents of a NAR - file. */ +/** + * Return an object that provides access to the contents of a NAR + * file. + */ ref<FSAccessor> makeNarAccessor(std::string && nar); ref<FSAccessor> makeNarAccessor(Source & source); -/* Create a NAR accessor from a NAR listing (in the format produced by - listNar()). The callback getNarBytes(offset, length) is used by the - readFile() method of the accessor to get the contents of files - inside the NAR. */ +/** + * Create a NAR accessor from a NAR listing (in the format produced by + * listNar()). The callback getNarBytes(offset, length) is used by the + * readFile() method of the accessor to get the contents of files + * inside the NAR. + */ typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes; ref<FSAccessor> makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); -/* Write a JSON representation of the contents of a NAR (except file - contents). */ +/** + * Write a JSON representation of the contents of a NAR (except file + * contents). + */ nlohmann::json listNar(ref<FSAccessor> accessor, const Path & path, bool recurse); } diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 4877f56d8..bbd1d05d5 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "ref.hh" #include "nar-info.hh" @@ -42,8 +43,10 @@ public: const std::string & uri, const DrvOutput & id) = 0; }; -/* Return a singleton cache object that can be used concurrently by - multiple threads. */ +/** + * Return a singleton cache object that can be used concurrently by + * multiple threads. + */ ref<NarInfoDiskCache> getNarInfoDiskCache(); ref<NarInfoDiskCache> getTestNarInfoDiskCache(Path dbPath); diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index a4dccb397..5dbdafac3 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "hash.hh" diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 0b7c98ac9..5a726fe90 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <cassert> #include <optional> diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index bfb3857c0..71085a604 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "derivations.hh" #include "store-api.hh" diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 97eb6638b..221523622 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "crypto.hh" #include "path.hh" @@ -18,8 +19,14 @@ struct SubstitutablePathInfo { std::optional<StorePath> deriver; StorePathSet references; - uint64_t downloadSize; /* 0 = unknown or inapplicable */ - uint64_t narSize; /* 0 = unknown */ + /** + * 0 = unknown or inapplicable + */ + uint64_t downloadSize; + /** + * 0 = unknown + */ + uint64_t narSize; }; typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; @@ -29,35 +36,40 @@ struct ValidPathInfo { StorePath path; std::optional<StorePath> deriver; - // TODO document this + /** + * \todo document this + */ Hash narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ + /** + * Whether the path is ultimately trusted, that is, it's a + * derivation output that was built locally. + */ bool ultimate = false; StringSet sigs; // note: not necessarily verified - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a Boolean, - and the store path would be computed from the name component, ‘narHash’ - and ‘references’. However, we support many types of content addresses. - */ + /** + * If non-empty, an assertion that the path is content-addressed, + * i.e., that the store path is computed from a cryptographic hash + * of the contents of the path, plus some other bits of data like + * the "name" part of the path. Such a path doesn't need + * signatures, since we don't have to trust anybody's claim that + * the path is the output of a particular derivation. (In the + * extensional store model, we have to trust that the *contents* + * of an output path of a derivation were actually produced by + * that derivation. In the intensional model, we have to trust + * that a particular output path was produced by a derivation; the + * path then implies the contents.) + * + * Ideally, the content-addressability assertion would just be a Boolean, + * and the store path would be computed from the name component, ‘narHash’ + * and ‘references’. However, we support many types of content addresses. + */ std::optional<ContentAddress> ca; bool operator == (const ValidPathInfo & i) const @@ -68,29 +80,42 @@ struct ValidPathInfo && references == i.references; } - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ + /** + * Return a fingerprint of the store path to be used in binary + * cache signatures. It contains the store path, the base-32 + * SHA-256 hash of the NAR serialisation of the path, the size of + * the NAR, and the sorted references. The size field is strictly + * speaking superfluous, but might prevent endless/excessive data + * attacks. + */ std::string fingerprint(const Store & store) const; void sign(const Store & store, const SecretKey & secretKey); + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ std::optional<ContentAddressWithReferences> contentAddressWithReferences() const; - /* Return true iff the path is verifiably content-addressed. */ + /** + * @return true iff the path is verifiably content-addressed. + */ bool isContentAddressed(const Store & store) const; static const size_t maxSigs = std::numeric_limits<size_t>::max(); - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ + /** + * Return the number of signatures on this .narinfo that were + * produced by one of the specified keys, or maxSigs if the path + * is content-addressed. + */ size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - /* Verify a single signature. */ + /** + * Verify a single signature. + */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; Strings shortRefs() const; diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 6893c3876..4f8dc4c1f 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 5d25656a5..d75850868 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -1,17 +1,19 @@ #pragma once +///@file #include "path.hh" #include "derived-path.hh" namespace nix { -/* This is a deprecated old type just for use by the old CLI, and older - versions of the RPC protocols. In new code don't use it; you want - `DerivedPath` instead. - - `DerivedPath` is better because it handles more cases, and does so more - explicitly without devious punning tricks. -*/ +/** + * This is a deprecated old type just for use by the old CLI, and older + * versions of the RPC protocols. In new code don't use it; you want + * `DerivedPath` instead. + * + * `DerivedPath` is better because it handles more cases, and does so more + * explicitly without devious punning tricks. + */ struct StorePathWithOutputs { StorePath path; @@ -30,9 +32,11 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s); class Store; -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path - and the outputs. */ +/** + * Split a string specifying a derivation and a set of outputs + * (/nix/store/hash-foo!out1,out2,...) into the derivation path + * and the outputs. + */ StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 2730541c6..4ca6747b3 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <string_view> diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 5e3a734b4..4921df352 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,15 +1,20 @@ #pragma once +///@file #include "util.hh" namespace nix { -/* Open (possibly create) a lock file and return the file descriptor. - -1 is returned if create is false and the lock could not be opened - because it doesn't exist. Any other error throws an exception. */ +/** + * Open (possibly create) a lock file and return the file descriptor. + * -1 is returned if create is false and the lock could not be opened + * because it doesn't exist. Any other error throws an exception. + */ AutoCloseFD openLockFile(const Path & path, bool create); -/* Delete an open lock file. */ +/** + * Delete an open lock file. + */ void deleteLockFile(const Path & path, int fd); enum LockType { ltRead, ltWrite, ltNone }; diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 3cadd5c2a..4e1f42e83 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -1,6 +1,7 @@ #pragma once +///@file -#include "types.hh" + #include "types.hh" #include "pathlocks.hh" #include <time.h> @@ -23,9 +24,11 @@ struct Generation typedef std::list<Generation> Generations; -/* Returns the list of currently present generations for the specified - profile, sorted by generation number. Also returns the number of - the current generation. */ +/** + * Returns the list of currently present generations for the specified + * profile, sorted by generation number. Also returns the number of + * the current generation. + */ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile); class LocalFSStore; @@ -46,26 +49,32 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, void switchLink(Path link, Path target); -/* Roll back a profile to the specified generation, or to the most - recent one older than the current. */ +/** + * Roll back a profile to the specified generation, or to the most + * recent one older than the current. + */ void switchGeneration( const Path & profile, std::optional<GenerationNumber> dstGen, bool dryRun); -/* Ensure exclusive access to a profile. Any command that modifies - the profile first acquires this lock. */ +/** + * Ensure exclusive access to a profile. Any command that modifies + * the profile first acquires this lock. + */ void lockProfile(PathLocks & lock, const Path & profile); -/* Optimistic locking is used by long-running operations like `nix-env - -i'. Instead of acquiring the exclusive lock for the entire - duration of the operation, we just perform the operation - optimistically (without an exclusive lock), and check at the end - whether the profile changed while we were busy (i.e., the symlink - target changed). If so, the operation is restarted. Restarting is - generally cheap, since the build results are still in the Nix - store. Most of the time, only the user environment has to be - rebuilt. */ +/** + * Optimistic locking is used by long-running operations like `nix-env + * -i'. Instead of acquiring the exclusive lock for the entire + * duration of the operation, we just perform the operation + * optimistically (without an exclusive lock), and check at the end + * whether the profile changed while we were busy (i.e., the symlink + * target changed). If so, the operation is restarted. Restarting is + * generally cheap, since the build results are still in the Nix + * store. Most of the time, only the user environment has to be + * rebuilt. + */ std::string optimisticLockProfile(const Path & profile); /** diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 48d0283de..a18cf2aa8 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <variant> diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 6f381f96c..52d71b333 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "hash.hh" #include "path.hh" diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 99f5544ef..e2673b6f6 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "fs-accessor.hh" #include "ref.hh" diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1c6b8530d..2abd3aa51 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -42,6 +42,40 @@ void write(const Store & store, Sink & out, const StorePath & storePath) } +std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _) +{ + auto temp = readNum<uint8_t>(from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return { Trusted }; + case 2: + return { NotTrusted }; + default: + throw Error("Invalid trusted status from remote"); + } +} + +void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted) +{ + if (!optTrusted) + out << (uint8_t)0; + else { + switch (*optTrusted) { + case Trusted: + out << (uint8_t)1; + break; + case NotTrusted: + out << (uint8_t)2; + break; + default: + assert(false); + }; + } +} + + ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _) { return ContentAddress::parse(readString(from)); @@ -56,12 +90,12 @@ void write(const Store & store, Sink & out, const ContentAddress & ca) DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _) { auto s = readString(from); - return DerivedPath::parse(store, s); + return DerivedPath::parseLegacy(store, s); } void write(const Store & store, Sink & out, const DerivedPath & req) { - out << req.to_string(store); + out << req.to_string_legacy(store); } @@ -226,6 +260,13 @@ void RemoteStore::initConnection(Connection & conn) conn.daemonNixVersion = readString(conn.from); } + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { + conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom<std::optional<TrustedFlag>> {}); + } else { + // We don't know the answer; protocol to old. + conn.remoteTrustsUs = std::nullopt; + } + auto ex = conn.processStderr(); if (ex) std::rethrow_exception(ex); } @@ -1086,6 +1127,11 @@ unsigned int RemoteStore::getProtocol() return conn->daemonVersion; } +std::optional<TrustedFlag> RemoteStore::isTrustedClient() +{ + auto conn(getConnection()); + return conn->remoteTrustsUs; +} void RemoteStore::flushBadConnections() { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3ff22ed66..3babf8e21 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <limits> #include <string> @@ -31,8 +32,10 @@ struct RemoteStoreConfig : virtual StoreConfig "Maximum age of a connection before it is closed."}; }; -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ +/** + * \todo RemoteStore is a misnomer - should be something like + * DaemonStore. + */ class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore, @@ -68,7 +71,9 @@ public: void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) override; - /* Add a content-addressable store path. `dump` will be drained. */ + /** + * Add a content-addressable store path. `dump` will be drained. + */ ref<const ValidPathInfo> addCAToStore( Source & dump, std::string_view name, @@ -77,7 +82,9 @@ public: const StorePathSet & references, RepairFlag repair); - /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ + /** + * Add a content-addressable store path. Does not support references. `dump` will be drained. + */ StorePath addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; @@ -144,6 +151,8 @@ public: unsigned int getProtocol() override; + std::optional<TrustedFlag> isTrustedClient() override; + void flushBadConnections(); struct Connection @@ -151,6 +160,7 @@ public: FdSink to; FdSource from; unsigned int daemonVersion; + std::optional<TrustedFlag> remoteTrustsUs; std::optional<std::string> daemonNixVersion; std::chrono::time_point<std::chrono::steady_clock> startTime; diff --git a/src/libstore/repair-flag.hh b/src/libstore/repair-flag.hh index a13cda312..f412d6a20 100644 --- a/src/libstore/repair-flag.hh +++ b/src/libstore/repair-flag.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index ac82147ee..d2fc6abaf 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -509,6 +509,16 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return paths; } + /** + * For now, we conservatively say we don't know. + * + * \todo try to expose our S3 authentication status. + */ + std::optional<TrustedFlag> isTrustedClient() override + { + return std::nullopt; + } + static std::set<std::string> uriSchemes() { return {"s3"}; } }; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index bce828b11..c62ea5147 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "binary-cache-store.hh" diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index cdb3e5908..f0aeb3bed 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -1,4 +1,5 @@ #pragma once +///@file #if ENABLE_S3 diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 3f76baa82..553fd3a09 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -1,4 +1,5 @@ #pragma once +///@file namespace nix { diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 871f2f3be..df334c23c 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -239,14 +239,11 @@ SQLiteTxn::~SQLiteTxn() } } -void handleSQLiteBusy(const SQLiteBusy & e) +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) { - static std::atomic<time_t> lastWarned{0}; - time_t now = time(0); - - if (now > lastWarned + 10) { - lastWarned = now; + if (now > nextWarning) { + nextWarning = now + 10; logWarning({ .msg = hintfmt(e.what()) }); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 1853731a2..6e14852cb 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <functional> #include <string> @@ -10,7 +11,9 @@ struct sqlite3_stmt; namespace nix { -/* RAII wrapper to close a SQLite database automatically. */ +/** + * RAII wrapper to close a SQLite database automatically. + */ struct SQLite { sqlite3 * db = 0; @@ -22,7 +25,9 @@ struct SQLite ~SQLite(); operator sqlite3 * () { return db; } - /* Disable synchronous mode, set truncate journal mode. */ + /** + * Disable synchronous mode, set truncate journal mode. + */ void isCache(); void exec(const std::string & stmt); @@ -30,7 +35,9 @@ struct SQLite uint64_t getLastInsertedRowId(); }; -/* RAII wrapper to create and destroy SQLite prepared statements. */ +/** + * RAII wrapper to create and destroy SQLite prepared statements. + */ struct SQLiteStmt { sqlite3 * db = 0; @@ -42,7 +49,9 @@ struct SQLiteStmt ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } - /* Helper for binding / executing statements. */ + /** + * Helper for binding / executing statements. + */ class Use { friend struct SQLiteStmt; @@ -55,7 +64,9 @@ struct SQLiteStmt ~Use(); - /* Bind the next parameter. */ + /** + * Bind the next parameter. + */ Use & operator () (std::string_view value, bool notNull = true); Use & operator () (const unsigned char * data, size_t len, bool notNull = true); Use & operator () (int64_t value, bool notNull = true); @@ -63,11 +74,15 @@ struct SQLiteStmt int step(); - /* Execute a statement that does not return rows. */ + /** + * Execute a statement that does not return rows. + */ void exec(); - /* For statements that return 0 or more rows. Returns true iff - a row is available. */ + /** + * For statements that return 0 or more rows. Returns true iff + * a row is available. + */ bool next(); std::string getStr(int col); @@ -81,8 +96,10 @@ struct SQLiteStmt } }; -/* RAII helper that ensures transactions are aborted unless explicitly - committed. */ +/** + * RAII helper that ensures transactions are aborted unless explicitly + * committed. + */ struct SQLiteTxn { bool active = false; @@ -122,18 +139,22 @@ protected: MakeError(SQLiteBusy, SQLiteError); -void handleSQLiteBusy(const SQLiteBusy & e); +void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning); -/* Convenience function for retrying a SQLite transaction when the - database is busy. */ +/** + * Convenience function for retrying a SQLite transaction when the + * database is busy. + */ template<typename T, typename F> T retrySQLite(F && fun) { + time_t nextWarning = time(0) + 1; + while (true) { try { return fun(); } catch (SQLiteBusy & e) { - handleSQLiteBusy(e); + handleSQLiteBusy(e, nextWarning); } } } diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index c4232df34..c27a5d00f 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "store-api.hh" namespace nix { diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 69bfe3418..6f6deda51 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,4 +1,5 @@ #include "ssh.hh" +#include "finally.hh" namespace nix { @@ -35,6 +36,9 @@ void SSHMaster::addCommonSSHOpts(Strings & args) } if (compress) args.push_back("-C"); + + args.push_back("-oPermitLocalCommand=yes"); + args.push_back("-oLocalCommand=echo started"); } std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command) @@ -49,6 +53,11 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string ProcessOptions options; options.dieWithParent = false; + if (!fakeSSH && !useMaster) { + logger->pause(); + } + Finally cleanup = [&]() { logger->resume(); }; + conn->sshPid = startProcess([&]() { restoreProcessContext(); @@ -86,6 +95,18 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string in.readSide = -1; out.writeSide = -1; + // Wait for the SSH connection to be established, + // So that we don't overwrite the password prompt with our progress bar. + if (!fakeSSH && !useMaster) { + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } + + if (reply != "started") + throw Error("failed to start SSH connection to '%s'", host); + } + conn->out = std::move(out.readSide); conn->in = std::move(in.writeSide); @@ -109,6 +130,9 @@ Path SSHMaster::startMaster() ProcessOptions options; options.dieWithParent = false; + logger->pause(); + Finally cleanup = [&]() { logger->resume(); }; + state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -117,11 +141,7 @@ Path SSHMaster::startMaster() if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("duping over stdout"); - Strings args = - { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath - , "-o", "LocalCommand=echo started" - , "-o", "PermitLocalCommand=yes" - }; + Strings args = { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath }; if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index dabbcedda..c86a8a986 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "util.hh" #include "sync.hh" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 78b0d907e..5bee1af9f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -527,6 +527,57 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) return outputPaths; } + +void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) +{ + if (!settings.useSubstitutes) return; + for (auto & sub : getDefaultSubstituters()) { + for (auto & path : paths) { + if (infos.count(path.first)) + // Choose first succeeding substituter. + continue; + + auto subPath(path.first); + + // Recompute store path so that we can use a different store root. + if (path.second) { + subPath = makeFixedOutputPathFromCA( + path.first.name(), + ContentAddressWithReferences::withoutRefs(*path.second)); + if (sub->storeDir == storeDir) + assert(subPath == path.first); + if (subPath != path.first) + debug("replaced path '%s' with '%s' for substituter '%s'", printStorePath(path.first), sub->printStorePath(subPath), sub->getUri()); + } else if (sub->storeDir != storeDir) continue; + + debug("checking substituter '%s' for path '%s'", sub->getUri(), sub->printStorePath(subPath)); + try { + auto info = sub->queryPathInfo(subPath); + + if (sub->storeDir != storeDir && !(info->isContentAddressed(*sub) && info->references.empty())) + continue; + + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + infos.insert_or_assign(path.first, SubstitutablePathInfo{ + .deriver = info->deriver, + .references = info->references, + .downloadSize = narInfo ? narInfo->fileSize : 0, + .narSize = info->narSize, + }); + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + logError(e.info()); + else + throw; + } + } + } +} + + bool Store::isValidPath(const StorePath & storePath) { { @@ -1125,7 +1176,8 @@ std::map<StorePath, StorePath> copyPaths( return storePathForDst; }; - uint64_t total = 0; + // total is accessed by each copy, which are each handled in separate threads + std::atomic<uint64_t> total = 0; for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5edcc0f36..74f50a00d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "nar-info.hh" #include "realisation.hh" @@ -88,6 +89,7 @@ const uint32_t exportMagic = 0x4558494e; enum BuildMode { bmNormal, bmRepair, bmCheck }; +enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; @@ -403,17 +405,17 @@ public: { unsupported("queryReferrers"); } /** - * @return all currently valid derivations that have `path' as an + * @return all currently valid derivations that have `path` as an * output. * - * (Note that the result of `queryDeriver()' is the derivation that - * was actually used to produce `path', which may not exist + * (Note that the result of `queryDeriver()` is the derivation that + * was actually used to produce `path`, which may not exist * anymore.) */ virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /** - * Query the outputs of the derivation denoted by `path'. + * Query the outputs of the derivation denoted by `path`. */ virtual StorePathSet queryDerivationOutputs(const StorePath & path); @@ -449,7 +451,7 @@ public: * resulting ‘infos’ map. */ virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, - SubstitutablePathInfos & infos) { return; }; + SubstitutablePathInfos & infos); /** * Import a path into the store. @@ -505,7 +507,7 @@ public: /** * Like addToStore(), but the contents of the path are contained - * in `dump', which is either a NAR serialisation (if recursive == + * in `dump`, which is either a NAR serialisation (if recursive == * true) or simply the contents of a regular file (if recursive == * false). * `dump` may be drained @@ -626,8 +628,8 @@ public: /** * @return a string representing information about the path that - * can be loaded into the database using `nix-store --load-db' or - * `nix-store --register-validity'. + * can be loaded into the database using `nix-store --load-db` or + * `nix-store --register-validity`. */ std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); @@ -670,8 +672,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref<FSAccessor> getFSAccessor() - { unsupported("getFSAccessor"); } + virtual ref<FSAccessor> getFSAccessor() = 0; /** * Repair the contents of the given path by redownloading it using @@ -707,12 +708,12 @@ public: /** * @param [out] out Place in here the set of all store paths in the - * file system closure of `storePath'; that is, all paths than can - * be directly or indirectly reached from it. `out' is not cleared. + * file system closure of `storePath`; that is, all paths than can + * be directly or indirectly reached from it. `out` is not cleared. * * @param flipDirection If true, the set of paths that can reach - * `storePath' is returned; that is, the closures under the - * `referrers' relation instead of the `references' relation is + * `storePath` is returned; that is, the closures under the + * `referrers` relation instead of the `references` relation is * returned. */ virtual void computeFSClosure(const StorePathSet & paths, @@ -808,6 +809,17 @@ public: return 0; }; + /** + * @return/ whether store trusts *us*. + * + * `std::nullopt` means we do not know. + * + * @note This is the opposite of the StoreConfig::isTrusted + * store setting. That is about whether *we* trust the store. + */ + virtual std::optional<TrustedFlag> isTrustedClient() = 0; + + virtual Path toRealPath(const Path & storePath) { return storePath; diff --git a/src/libstore/store-cast.hh b/src/libstore/store-cast.hh index ff62fc359..2473e72c5 100644 --- a/src/libstore/store-cast.hh +++ b/src/libstore/store-cast.hh @@ -1,9 +1,17 @@ #pragma once +///@file #include "store-api.hh" namespace nix { +/** + * Helper to try downcasting a Store with a nice method if it fails. + * + * This is basically an alternative to the user-facing part of + * Store::unsupported that allows us to still have a nice message but + * better interface design. + */ template<typename T> T & require(Store & store) { diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 4dd14dcce..1cab68e06 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -11,15 +11,29 @@ class DerivationTest : public LibStoreTest { }; -#define TEST_JSON(TYPE, NAME, STR, VAL, ...) \ - TEST_F(DerivationTest, TYPE ## _ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (TYPE { VAL }).toJSON(*store __VA_OPT__(,) __VA_ARGS__)); \ +#define TEST_JSON(NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (DerivationOutput { VAL }).toJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME)); \ + } \ + \ + TEST_F(DerivationTest, DerivationOutput_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + DerivationOutput { VAL }, \ + DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + STR ## _json)); \ } -TEST_JSON(DerivationOutput, inputAddressed, +TEST_JSON(inputAddressed, R"({ "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", @@ -28,7 +42,7 @@ TEST_JSON(DerivationOutput, inputAddressed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFixed, +TEST_JSON(caFixed, R"({ "hashAlgo": "r:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -45,7 +59,7 @@ TEST_JSON(DerivationOutput, caFixed, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFixedText, +TEST_JSON(caFixedText, R"({ "hashAlgo": "text:sha256", "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", @@ -61,7 +75,7 @@ TEST_JSON(DerivationOutput, caFixedText, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, caFloating, +TEST_JSON(caFloating, R"({ "hashAlgo": "r:sha256" })", @@ -71,12 +85,12 @@ TEST_JSON(DerivationOutput, caFloating, }), "drv-name", "output-name") -TEST_JSON(DerivationOutput, deferred, +TEST_JSON(deferred, R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") -TEST_JSON(DerivationOutput, impure, +TEST_JSON(impure, R"({ "hashAlgo": "r:sha256", "impure": true @@ -87,8 +101,28 @@ TEST_JSON(DerivationOutput, impure, }), "drv-name", "output-name") -TEST_JSON(Derivation, impure, +#undef TEST_JSON + +#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (Derivation { VAL }).toJSON(*store)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + Derivation { VAL }, \ + Derivation::fromJSON( \ + *store, \ + STR ## _json)); \ + } + +TEST_JSON(simple, R"({ + "name": "my-derivation", "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], @@ -111,6 +145,7 @@ TEST_JSON(Derivation, impure, })", ({ Derivation drv; + drv.name = "my-derivation"; drv.inputSrcs = { store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; @@ -136,7 +171,8 @@ TEST_JSON(Derivation, impure, }, }; drv; - })) + }), + "drv-name") #undef TEST_JSON diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index d1ac2c5e7..e6d32dbd0 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -53,6 +53,14 @@ TEST_F(DerivedPathTest, force_init) RC_GTEST_FIXTURE_PROP( DerivedPathTest, + prop_legacy_round_rip, + (const DerivedPath & o)) +{ + RC_ASSERT(o == DerivedPath::parseLegacy(*store, o.to_string_legacy(*store))); +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathTest, prop_round_rip, (const DerivedPath & o)) { diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 3bc812440..506f3ccb1 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <rapidcheck/gen/Arbitrary.h> diff --git a/src/libstore/tests/libstore.hh b/src/libstore/tests/libstore.hh index 05397659b..ef93457b5 100644 --- a/src/libstore/tests/libstore.hh +++ b/src/libstore/tests/libstore.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include <gtest/gtest.h> #include <gmock/gmock.h> diff --git a/src/libstore/tests/outputs-spec.hh b/src/libstore/tests/outputs-spec.hh index 2d455c817..ded331b33 100644 --- a/src/libstore/tests/outputs-spec.hh +++ b/src/libstore/tests/outputs-spec.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <rapidcheck/gen/Arbitrary.h> diff --git a/src/libstore/tests/path.hh b/src/libstore/tests/path.hh index d7f1a8988..21cb62310 100644 --- a/src/libstore/tests/path.hh +++ b/src/libstore/tests/path.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include <rapidcheck/gen/Arbitrary.h> diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index caa452919..bd1dcb67c 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "remote-store.hh" #include "local-fs-store.hh" diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 87088a3ac..c7a6f8688 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "store-api.hh" #include "serialise.hh" @@ -9,11 +10,15 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 34) +#define PROTOCOL_VERSION (1 << 8 | 35) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) +/** + * Enumeration of all the request types for the "worker protocol", used + * by unix:// and ssh-ng:// stores. + */ typedef enum { wopIsValidPath = 1, wopHasSubstitutes = 3, @@ -74,7 +79,12 @@ typedef enum { class Store; struct Source; -/* To guide overloading */ +/** + * Used to guide overloading + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ template<typename T> struct Phantom {}; @@ -93,6 +103,7 @@ MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); MAKE_WORKER_PROTO(, BuildResult); +MAKE_WORKER_PROTO(, std::optional<TrustedFlag>); MAKE_WORKER_PROTO(template<typename T>, std::vector<T>); MAKE_WORKER_PROTO(template<typename T>, std::set<T>); @@ -103,18 +114,19 @@ MAKE_WORKER_PROTO(X_, Y_); #undef X_ #undef Y_ -/* These use the empty string for the null case, relying on the fact - that the underlying types never serialize to the empty string. - - We do this instead of a generic std::optional<T> instance because - ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - the same reason, we don't have a std::variant<T..> instances (ordinal - tags 0...n). - - We could the generic instances and then these as specializations for - compatability, but that's proven a bit finnicky, and also makes the - worker protocol harder to implement in other languages where such - specializations may not be allowed. +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional<T> instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant<T..> instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. */ MAKE_WORKER_PROTO(, std::optional<StorePath>); MAKE_WORKER_PROTO(, std::optional<ContentAddress>); |