diff options
Diffstat (limited to 'src/libstore/store-api.hh')
| -rw-r--r-- | src/libstore/store-api.hh | 371 |
1 files changed, 182 insertions, 189 deletions
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 81014763d..591140874 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,12 +2,14 @@ #include "path.hh" #include "hash.hh" +#include "content-address.hh" #include "serialise.hh" -#include "crypto.hh" #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" #include "config.hh" +#include "derivations.hh" +#include "path-info.hh" #include <atomic> #include <limits> @@ -17,10 +19,36 @@ #include <memory> #include <string> #include <chrono> +#include <variant> namespace nix { +/** + * About the class hierarchy of the store implementations: + * + * Each store type `Foo` consists of two classes: + * + * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration + * for the store + * + * It should only contain members of type `const Setting<T>` (or subclasses + * of it) and inherit the constructors of `StoreConfig` + * (`using StoreConfig::StoreConfig`). + * + * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the + * implementation of the store. + * + * This class is expected to have a constructor `Foo(const Params & params)` + * that calls `StoreConfig(params)` (otherwise you're gonna encounter an + * `assertion failure` when trying to instantiate it). + * + * You can then register the new store using: + * + * ``` + * cpp static RegisterStoreImplementation<Foo, FooConfig> regStore; + * ``` + */ MakeError(SubstError, Error); MakeError(BuildError, Error); // denotes a permanent build failure @@ -28,23 +56,20 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(NotInStore, Error); +MakeError(BadStorePath, Error); +MakeError(InvalidStoreURI, Error); -struct BasicDerivation; -struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; class JSONPlaceholder; -enum RepairFlag : bool { NoRepair = false, Repair = true }; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; - /* Magic header of exportPath() output (obsolete). */ const uint32_t exportMagic = 0x4558494e; @@ -86,7 +111,7 @@ struct GCOptions StorePathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ - unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; + uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; }; @@ -98,106 +123,9 @@ struct GCResults /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the number of bytes that would be or was freed. */ - unsigned long long bytesFreed = 0; -}; - - -struct SubstitutablePathInfo -{ - std::optional<StorePath> deriver; - StorePathSet references; - unsigned long long downloadSize; /* 0 = unknown or inapplicable */ - unsigned long long narSize; /* 0 = unknown */ -}; - -typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; - - -struct ValidPathInfo -{ - StorePath path; - std::optional<StorePath> deriver; - Hash narHash; - StorePathSet references; - time_t registrationTime = 0; - uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only - - /* Whether the path is ultimately trusted, that is, it's a - derivation output that was built locally. */ - bool ultimate = false; - - StringSet sigs; // note: not necessarily verified - - /* If non-empty, an assertion that the path is content-addressed, - i.e., that the store path is computed from a cryptographic hash - of the contents of the path, plus some other bits of data like - the "name" part of the path. Such a path doesn't need - signatures, since we don't have to trust anybody's claim that - the path is the output of a particular derivation. (In the - extensional store model, we have to trust that the *contents* - of an output path of a derivation were actually produced by - that derivation. In the intensional model, we have to trust - that a particular output path was produced by a derivation; the - path then implies the contents.) - - Ideally, the content-addressability assertion would just be a - Boolean, and the store path would be computed from - the name component, ‘narHash’ and ‘references’. However, - 1) we've accumulated several types of content-addressed paths - over the years; and 2) fixed-output derivations support - multiple hash algorithms and serialisation methods (flat file - vs NAR). Thus, ‘ca’ has one of the following forms: - - * ‘text:sha256:<sha256 hash of file contents>’: For paths - computed by makeTextPath() / addTextToStore(). - - * ‘fixed:<r?>:<ht>:<h>’: For paths computed by - makeFixedOutputPath() / addToStore(). - */ - std::string ca; - - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } - - /* Return a fingerprint of the store path to be used in binary - cache signatures. It contains the store path, the base-32 - SHA-256 hash of the NAR serialisation of the path, the size of - the NAR, and the sorted references. The size field is strictly - speaking superfluous, but might prevent endless/excessive data - attacks. */ - std::string fingerprint(const Store & store) const; - - void sign(const Store & store, const SecretKey & secretKey); - - /* Return true iff the path is verifiably content-addressed. */ - bool isContentAddressed(const Store & store) const; - - static const size_t maxSigs = std::numeric_limits<size_t>::max(); - - /* Return the number of signatures on this .narinfo that were - produced by one of the specified keys, or maxSigs if the path - is content-addressed. */ - size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; - - /* Verify a single signature. */ - bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; - - Strings shortRefs() const; - - ValidPathInfo(StorePath && path) : path(std::move(path)) { } - explicit ValidPathInfo(const ValidPathInfo & other); - - virtual ~ValidPathInfo() { } + uint64_t bytesFreed = 0; }; -typedef list<ValidPathInfo> ValidPathInfos; - enum BuildMode { bmNormal, bmRepair, bmCheck }; @@ -242,12 +170,31 @@ struct BuildResult } }; - -class Store : public std::enable_shared_from_this<Store>, public Config +struct StoreConfig : public Config { -public: - - typedef std::map<std::string, std::string> Params; + using Config::Config; + + /** + * When constructing a store implementation, we pass in a map `params` of + * parameters that's supposed to initialize the associated config. + * To do that, we must use the `StoreConfig(StringMap & params)` + * constructor, so we'd like to `delete` its default constructor to enforce + * it. + * + * However, actually deleting it means that all the subclasses of + * `StoreConfig` will have their default constructor deleted (because it's + * supposed to call the deleted default constructor of `StoreConfig`). But + * because we're always using virtual inheritance, the constructors of + * child classes will never implicitely call this one, so deleting it will + * be more painful than anything else. + * + * So we `assert(false)` here to ensure at runtime that the right + * constructor is always called without having to redefine a custom + * constructor for each `*Config` class. + */ + StoreConfig() { assert(false); } + + virtual const std::string name() = 0; const PathSetting storeDir_{this, false, settings.nixStore, "store", "path to the Nix store"}; @@ -261,6 +208,18 @@ public: Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; + Setting<StringSet> systemFeatures{this, settings.systemFeatures, + "system-features", + "Optional features that the system this store builds on implements (like \"kvm\")."}; + +}; + +class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig +{ +public: + + typedef std::map<std::string, std::string> Params; + protected: struct PathInfoCacheValue { @@ -294,6 +253,11 @@ protected: Store(const Params & params); public: + /** + * Perform any necessary effectful operation to make the store up and + * running + */ + virtual void init() {}; virtual ~Store() { } @@ -327,9 +291,9 @@ public: the Nix store. */ bool isStorePath(std::string_view path) const; - /* Chop off the parts after the top-level store name, e.g., - /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ - Path toStorePath(const Path & path) const; + /* Split a path like /nix/store/<hash>-<name>/<bla> into + /nix/store/<hash>-<name> and /<bla>. */ + std::pair<StorePath, Path> toStorePath(const Path & path) const; /* Follow symlinks until we end up with a path in the Nix store. */ Path followLinksToStore(std::string_view path) const; @@ -341,25 +305,31 @@ public: StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const; /* Constructs a unique store path name. */ - StorePath makeStorePath(const string & type, + StorePath makeStorePath(std::string_view type, + std::string_view hash, std::string_view name) const; + StorePath makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const; - StorePath makeOutputPath(const string & id, + StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(bool recursive, + StorePath makeFixedOutputPath(FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references = {}, bool hasSelfReference = false) const; StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const; + const StorePathSet & references = {}) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, + const StorePathSet & references = {}, + bool hasSelfReference = false) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, - const Path & srcPath, bool recursive = true, + const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). @@ -394,13 +364,16 @@ public: SubstituteFlag maybeSubstitute = NoSubstitute); /* Query the set of all valid paths. Note that for some store - backends, the name part of store paths may be omitted - (i.e. you'll get /nix/store/<hash> rather than + backends, the name part of store paths may be replaced by 'x' + (i.e. you'll get /nix/store/<hash>-x rather than /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the - full store path. */ + full store path. FIXME: should return a set of + std::variant<StorePath, HashPart> to get rid of this hack. */ virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } + constexpr static const char * MissingName = "x"; + /* Query information about a valid path. It is permitted to omit the name part of the store path. */ ref<const ValidPathInfo> queryPathInfo(const StorePath & path); @@ -428,12 +401,17 @@ public: virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /* Query the outputs of the derivation denoted by `path'. */ - virtual StorePathSet queryDerivationOutputs(const StorePath & path) - { unsupported("queryDerivationOutputs"); } + virtual StorePathSet queryDerivationOutputs(const StorePath & path); - /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const StorePath & path) - { unsupported("queryDerivationOutputNames"); } + /* Query the mapping outputName => outputPath for the given derivation. All + outputs are mentioned so ones mising the mapping are mapped to + `std::nullopt`. */ + virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) + { unsupported("queryPartialDerivationOutputMap"); } + + /* Query the mapping outputName=>outputPath for the given derivation. + Assume every output has a mapping and throw an exception otherwise. */ + OutputPathMap queryDerivationOutputMap(const StorePath & path); /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ @@ -443,32 +421,39 @@ public: virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download - sizes) of a set of paths. If a path does not have substitute - info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const StorePathSet & paths, + sizes) of a map of paths to their optional ca values. If a path + does not have substitute info, it's omitted from the resulting + ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const StorePathCAMap & paths, SubstitutablePathInfos & infos) { return; }; /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0); - - // FIXME: remove - virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0); + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); + /* Copy the contents of a path to the store and register the + validity the resulting path, using a constant amount of + memory. */ + ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + std::optional<Hash> expectedCAHash = {}); + + /* Like addToStore(), but the contents of the path are contained + in `dump', which is either a NAR serialisation (if recursive == + true) or simply the contents of a regular file (if recursive == + false). + `dump` may be drained */ // FIXME: remove? - virtual StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + virtual StorePath addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); } @@ -593,6 +578,9 @@ public: ensurePath(). */ Derivation derivationFromPath(const StorePath & drvPath); + /* Read a derivation (which must already be valid). */ + Derivation readDerivation(const StorePath & drvPath); + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly or indirectly reached from it. `out' is not cleared. If @@ -613,7 +601,7 @@ public: that will be substituted. */ virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize); + uint64_t & downloadSize, uint64_t & narSize); /* Sort a set of paths topologically under the references relation. If p refers to q, then p precedes q in this list. */ @@ -629,8 +617,7 @@ public: the Nix store. Optionally, the contents of the NARs are preloaded into the specified FS accessor to speed up subsequent access. */ - StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, - CheckSigsFlag checkSigs = CheckSigs); + StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); struct Stats { @@ -698,22 +685,25 @@ protected: }; - -class LocalFSStore : public virtual Store +struct LocalFSStoreConfig : virtual StoreConfig { -public: - - // FIXME: the (Store*) cast works around a bug in gcc that causes + using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes // it to omit the call to the Setting constructor. Clang works fine // either way. - const PathSetting rootDir{(Store*) this, true, "", + const PathSetting rootDir{(StoreConfig*) this, true, "", "root", "directory prefixed to all other paths"}; - const PathSetting stateDir{(Store*) this, false, + const PathSetting stateDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, "state", "directory where Nix will store state"}; - const PathSetting logDir{(Store*) this, false, + const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store state"}; +}; + +class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +{ +public: const static string drvsLogDir; @@ -723,8 +713,7 @@ public: ref<FSAccessor> getFSAccessor() override; /* Register a permanent GC root. */ - Path addPermRoot(const StorePath & storePath, - const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); + Path addPermRoot(const StorePath & storePath, const Path & gcRoot); virtual Path getRealStoreDir() { return storeDir; } @@ -738,21 +727,19 @@ public: }; -/* Extract the hash part of the given store path. */ -string storePathToHash(const Path & path); - - /* Copy a path from one store to another. */ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); /* Copy store paths from one store to another. The paths may be copied - in parallel. They are copied in a topologically sorted order - (i.e. if A is a reference of B, then A is copied before B), but - the set of store paths is not automatically closed; use - copyClosure() for that. */ -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, + in parallel. They are copied in a topologically sorted order (i.e. + if A is a reference of B, then A is copied before B), but the set + of store paths is not automatically closed; use copyClosure() for + that. Returns a map of what each path was copied to the dstStore + as. */ +std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, + const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -805,39 +792,49 @@ ref<Store> openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); -enum StoreType { - tDaemon, - tLocal, - tOther -}; - - -StoreType getStoreType(const std::string & uri = settings.storeUri.get(), - const std::string & stateDir = settings.nixStateDir); - /* Return the default substituter stores, defined by the ‘substituters’ option and various legacy options. */ std::list<ref<Store>> getDefaultSubstituters(); +struct StoreFactory +{ + std::set<std::string> uriSchemes; + std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create; + std::function<std::shared_ptr<StoreConfig> ()> getConfig; +}; +struct Implementations +{ + static std::vector<StoreFactory> * registered; -/* Store implementation registration. */ -typedef std::function<std::shared_ptr<Store>( - const std::string & uri, const Store::Params & params)> OpenStore; + template<typename T, typename TConfig> + static void add() + { + if (!registered) registered = new std::vector<StoreFactory>(); + StoreFactory factory{ + .uriSchemes = T::uriSchemes(), + .create = + ([](const std::string & scheme, const std::string & uri, const Store::Params & params) + -> std::shared_ptr<Store> + { return std::make_shared<T>(scheme, uri, params); }), + .getConfig = + ([]() + -> std::shared_ptr<StoreConfig> + { return std::make_shared<TConfig>(StringMap({})); }) + }; + registered->push_back(factory); + } +}; +template<typename T, typename TConfig> struct RegisterStoreImplementation { - typedef std::vector<OpenStore> Implementations; - static Implementations * implementations; - - RegisterStoreImplementation(OpenStore fun) + RegisterStoreImplementation() { - if (!implementations) implementations = new Implementations; - implementations->push_back(fun); + Implementations::add<T, TConfig>(); } }; - /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ string showPaths(const PathSet & paths); @@ -846,15 +843,11 @@ string showPaths(const PathSet & paths); std::optional<ValidPathInfo> decodeValidPathInfo( const Store & store, std::istream & str, - bool hashGiven = false); - - -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(bool recursive, const Hash & hash); - + std::optional<HashResult> hashGiven = std::nullopt); /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); +std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv); + } |