diff options
30 files changed, 442 insertions, 227 deletions
diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 5e7cfda3e..9ec8b39c3 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -6,7 +6,7 @@ namespace nix { class Store; -struct StorePath; +class StorePath; } namespace nix::flake { diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index be71b786b..89b1e6e7d 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -23,7 +23,7 @@ struct InputScheme; struct Input { - friend class InputScheme; + friend struct InputScheme; std::shared_ptr<InputScheme> scheme; // note: can be null Attrs attrs; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5433fe50d..34f844a18 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -22,7 +22,8 @@ namespace nix { BinaryCacheStore::BinaryCacheStore(const Params & params) - : Store(params) + : BinaryCacheStoreConfig(params) + , Store(params) { if (secretKeyFile != "") secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile))); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 9bcdf5901..4b779cdd4 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -11,17 +11,21 @@ namespace nix { struct NarInfo; -class BinaryCacheStore : public Store +struct BinaryCacheStoreConfig : virtual StoreConfig { -public: - - const Setting<std::string> compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; - const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; - const Setting<bool> writeDebugInfo{this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; - const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; - const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; - const Setting<bool> parallelCompression{this, false, "parallel-compression", + using StoreConfig::StoreConfig; + + const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; + const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; + const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression", "enable multi-threading compression, available for xz only currently"}; +}; + +class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig +{ private: @@ -58,7 +62,7 @@ public: public: - virtual void init(); + virtual void init() override; private: diff --git a/src/libstore/build.cc b/src/libstore/build.cc index ce973862d..6e55f83d5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2872,18 +2872,23 @@ void DerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } +struct RestrictedStoreConfig : LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + const std::string name() { return "Restricted Store"; } +}; /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore +struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig { ref<LocalStore> next; DerivationGoal & goal; RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal) - : Store(params), LocalFSStore(params), next(next), goal(goal) + : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) { } Path getRealStoreDir() override diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 7a5744bc1..128832e60 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -2,17 +2,27 @@ namespace nix { -static std::string uriScheme = "dummy://"; +struct DummyStoreConfig : virtual StoreConfig { + using StoreConfig::StoreConfig; -struct DummyStore : public Store + const std::string name() override { return "Dummy Store"; } +}; + +struct DummyStore : public Store, public virtual DummyStoreConfig { - DummyStore(const Params & params) - : Store(params) + DummyStore(const std::string scheme, const std::string uri, const Params & params) + : DummyStore(params) { } + DummyStore(const Params & params) + : StoreConfig(params) + , Store(params) + { + } + string getUri() override { - return uriScheme; + return *uriSchemes().begin(); } void queryPathInfoUncached(const StorePath & path, @@ -21,6 +31,10 @@ struct DummyStore : public Store callback(nullptr); } + static std::set<std::string> uriSchemes() { + return {"dummy"}; + } + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } @@ -48,12 +62,6 @@ struct DummyStore : public Store { unsupported("buildDerivation"); } }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (uri != uriScheme) return nullptr; - return std::make_shared<DummyStore>(params); -}); +static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regStore; } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4a5971c3f..491c664db 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -162,11 +162,6 @@ template<> std::string BaseSetting<SandboxMode>::to_string() const else abort(); } -template<> nlohmann::json BaseSetting<SandboxMode>::toJSON() -{ - return AbstractSetting::toJSON(); -} - template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category) { args.addFlag({ diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8a2d3ff75..02721285a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -2,6 +2,7 @@ #include "types.hh" #include "config.hh" +#include "abstractsettingtojson.hh" #include "util.hh" #include <map> diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 1733239fb..f4ab15a10 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -7,7 +7,14 @@ namespace nix { MakeError(UploadToHTTP, Error); -class HttpBinaryCacheStore : public BinaryCacheStore +struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + const std::string name() override { return "Http Binary Cache Store"; } +}; + +class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig { private: @@ -24,9 +31,12 @@ private: public: HttpBinaryCacheStore( - const Params & params, const Path & _cacheUri) - : BinaryCacheStore(params) - , cacheUri(_cacheUri) + const std::string & scheme, + const Path & _cacheUri, + const Params & params) + : StoreConfig(params) + , BinaryCacheStore(params) + , cacheUri(scheme + "://" + _cacheUri) { if (cacheUri.back() == '/') cacheUri.pop_back(); @@ -55,6 +65,13 @@ public: } } + static std::set<std::string> uriSchemes() + { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set<std::string>({"http", "https"}); + if (forceHttp) ret.insert("file"); + return ret; + } protected: void maybeDisable() @@ -162,18 +179,6 @@ protected: }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - if (std::string(uri, 0, 7) != "http://" && - std::string(uri, 0, 8) != "https://" && - (!forceHttp || std::string(uri, 0, 7) != "file://")) - return 0; - auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); - store->init(); - return store; -}); +static RegisterStoreImplementation<HttpBinaryCacheStore, HttpBinaryCacheStoreConfig> regStore; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index dc03313f0..e9478c1d5 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -9,18 +9,24 @@ namespace nix { -static std::string uriScheme = "ssh://"; - -struct LegacySSHStore : public Store +struct LegacySSHStoreConfig : virtual StoreConfig { - const Setting<int> maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; - const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; - const Setting<bool> compress{this, false, "compress", "whether to compress the connection"}; - const Setting<Path> remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; - const Setting<std::string> remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; + using StoreConfig::StoreConfig; + const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"}; + const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; + const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; + const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + + const std::string name() override { return "Legacy SSH Store"; } +}; +struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig +{ // Hack for getting remote build log output. - const Setting<int> logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; struct Connection { @@ -37,8 +43,11 @@ struct LegacySSHStore : public Store SSHMaster master; - LegacySSHStore(const string & host, const Params & params) - : Store(params) + static std::set<std::string> uriSchemes() { return {"ssh"}; } + + LegacySSHStore(const string & scheme, const string & host, const Params & params) + : StoreConfig(params) + , Store(params) , host(host) , connections(make_ref<Pool<Connection>>( std::max(1, (int) maxConnections), @@ -84,7 +93,7 @@ struct LegacySSHStore : public Store string getUri() override { - return uriScheme + host; + return *uriSchemes().begin() + "://" + host; } void queryPathInfoUncached(const StorePath & path, @@ -325,12 +334,6 @@ public: } }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<LegacySSHStore>(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regStore; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 87d8334d7..b5744448e 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -4,7 +4,14 @@ namespace nix { -class LocalBinaryCacheStore : public BinaryCacheStore +struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + const std::string name() override { return "Local Binary Cache Store"; } +}; + +class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig { private: @@ -13,8 +20,11 @@ private: public: LocalBinaryCacheStore( - const Params & params, const Path & binaryCacheDir) - : BinaryCacheStore(params) + const std::string scheme, + const Path & binaryCacheDir, + const Params & params) + : StoreConfig(params) + , BinaryCacheStore(params) , binaryCacheDir(binaryCacheDir) { } @@ -26,6 +36,8 @@ public: return "file://" + binaryCacheDir; } + static std::set<std::string> uriSchemes(); + protected: bool fileExists(const std::string & path) override; @@ -85,16 +97,14 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) return pathExists(binaryCacheDir + "/" + path); } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> +std::set<std::string> LocalBinaryCacheStore::uriSchemes() { - if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || - std::string(uri, 0, 7) != "file://") - return 0; - auto store = std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); - store->init(); - return store; -}); + if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1") + return {}; + else + return {"file"}; +} + +static RegisterStoreImplementation<LocalBinaryCacheStore, LocalBinaryCacheStoreConfig> regStore; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0086bb13e..c618203f0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -42,7 +42,8 @@ namespace nix { LocalStore::LocalStore(const Params & params) - : Store(params) + : StoreConfig(params) + , Store(params) , LocalFSStore(params) , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", "physical path to the Nix store"} diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index d5e6d68ef..e7c9d1605 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -30,8 +30,19 @@ struct OptimiseStats uint64_t blocksFreed = 0; }; +struct LocalStoreConfig : virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + + Setting<bool> requireSigs{(StoreConfig*) this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + + const std::string name() override { return "Local Store"; } +}; + -class LocalStore : public LocalFSStore +class LocalStore : public LocalFSStore, public virtual LocalStoreConfig { private: @@ -95,10 +106,6 @@ public: private: - Setting<bool> requireSigs{(Store*) this, - settings.requireSigs, - "require-sigs", "whether store paths should have a trusted signature on import"}; - const PublicKeys & getPublicKeys(); public: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index dc61951d3..1abe236f7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -94,6 +94,7 @@ void write(const Store & store, Sink & out, const std::optional<StorePath> & sto /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : Store(params) + , RemoteStoreConfig(params) , connections(make_ref<Pool<Connection>>( std::max(1, (int) maxConnections), [this]() { return openConnectionWrapper(); }, @@ -123,19 +124,21 @@ ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper() UDSRemoteStore::UDSRemoteStore(const Params & params) - : Store(params) + : StoreConfig(params) + , Store(params) , LocalFSStore(params) , RemoteStore(params) { } -UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) - : Store(params) - , LocalFSStore(params) - , RemoteStore(params) - , path(socket_path) +UDSRemoteStore::UDSRemoteStore( + const std::string scheme, + std::string socket_path, + const Params & params) + : UDSRemoteStore(params) { + path.emplace(socket_path); } @@ -982,14 +985,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -static std::string uriScheme = "unix://"; - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<UDSRemoteStore>(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation<UDSRemoteStore, UDSRemoteStoreConfig> regStore; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 6f05f2197..a23690830 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -16,18 +16,22 @@ struct FdSource; template<typename T> class Pool; struct ConnectionHandle; - -/* FIXME: RemoteStore is a misnomer - should be something like - DaemonStore. */ -class RemoteStore : public virtual Store +struct RemoteStoreConfig : virtual StoreConfig { -public: + using StoreConfig::StoreConfig; - const Setting<int> maxConnections{(Store*) this, 1, + const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent connections to the Nix daemon"}; - const Setting<unsigned int> maxConnectionAge{(Store*) this, std::numeric_limits<unsigned int>::max(), + const Setting<unsigned int> maxConnectionAge{(StoreConfig*) this, std::numeric_limits<unsigned int>::max(), "max-connection-age", "number of seconds to reuse a connection"}; +}; + +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public virtual Store, public virtual RemoteStoreConfig +{ +public: virtual bool sameMachine() = 0; @@ -141,15 +145,35 @@ private: }; -class UDSRemoteStore : public LocalFSStore, public RemoteStore +struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +{ + UDSRemoteStoreConfig(const Store::Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + { + } + + UDSRemoteStoreConfig() + : UDSRemoteStoreConfig(Store::Params({})) + { + } + + const std::string name() override { return "Local Daemon Store"; } +}; + +class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig { public: UDSRemoteStore(const Params & params); - UDSRemoteStore(std::string path, const Params & params); + UDSRemoteStore(const std::string scheme, std::string path, const Params & params); std::string getUri() override; + static std::set<std::string> uriSchemes() + { return {"unix"}; } + bool sameMachine() override { return true; } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index a0a446bd3..d43f267e0 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -172,20 +172,26 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } -struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { - const Setting<std::string> profile{this, "", "profile", "The name of the AWS configuration profile to use."}; - const Setting<std::string> region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; - const Setting<std::string> scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; - const Setting<std::string> endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; - const Setting<std::string> narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; - const Setting<std::string> lsCompression{this, "", "ls-compression", "compression method for .ls files"}; - const Setting<std::string> logCompression{this, "", "log-compression", "compression method for log/* files"}; + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + const Setting<std::string> profile{(StoreConfig*) this, "", "profile", "The name of the AWS configuration profile to use."}; + const Setting<std::string> region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting<std::string> scheme{(StoreConfig*) this, "", "scheme", "The scheme to use for S3 requests, https by default."}; + const Setting<std::string> endpoint{(StoreConfig*) this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; + const Setting<std::string> narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting<std::string> lsCompression{(StoreConfig*) this, "", "ls-compression", "compression method for .ls files"}; + const Setting<std::string> logCompression{(StoreConfig*) this, "", "log-compression", "compression method for log/* files"}; const Setting<bool> multipartUpload{ - this, false, "multipart-upload", "whether to use multi-part uploads"}; + (StoreConfig*) this, false, "multipart-upload", "whether to use multi-part uploads"}; const Setting<uint64_t> bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + const std::string name() override { return "S3 Binary Cache Store"; } +}; + +struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig +{ std::string bucketName; Stats stats; @@ -193,8 +199,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore S3Helper s3Helper; S3BinaryCacheStoreImpl( - const Params & params, const std::string & bucketName) - : S3BinaryCacheStore(params) + const std::string & scheme, + const std::string & bucketName, + const Params & params) + : StoreConfig(params) + , S3BinaryCacheStore(params) , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) { @@ -426,17 +435,11 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore return paths; } + static std::set<std::string> uriSchemes() { return {"s3"}; } + }; -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, 5) != "s3://") return 0; - auto store = std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5)); - store->init(); - return store; -}); +static RegisterStoreImplementation<S3BinaryCacheStoreImpl, S3BinaryCacheStoreConfig> regStore; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 6cb97c1f1..8b6e48fb0 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -8,19 +8,25 @@ namespace nix { -static std::string uriScheme = "ssh-ng://"; +struct SSHStoreConfig : virtual RemoteStoreConfig +{ + using RemoteStoreConfig::RemoteStoreConfig; + + const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"}; + const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; + const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"}; + + const std::string name() override { return "SSH Store"; } +}; -class SSHStore : public RemoteStore +class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig { public: - const Setting<Path> sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; - const Setting<bool> compress{(Store*) this, false, "compress", "whether to compress the connection"}; - const Setting<Path> remoteProgram{(Store*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"}; - const Setting<std::string> remoteStore{(Store*) this, "", "remote-store", "URI of the store on the remote system"}; - - SSHStore(const std::string & host, const Params & params) - : Store(params) + SSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , Store(params) , RemoteStore(params) , host(host) , master( @@ -32,9 +38,11 @@ public: { } + static std::set<std::string> uriSchemes() { return {"ssh-ng"}; } + std::string getUri() override { - return uriScheme + host; + return *uriSchemes().begin() + "://" + host; } bool sameMachine() override @@ -76,12 +84,6 @@ ref<RemoteStore::Connection> SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared<SSHStore>(std::string(uri, uriScheme.size()), params); -}); +static RegisterStoreImplementation<SSHStore, SSHStoreConfig> regStore; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b3877487c..76cbc0605 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -346,7 +346,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, Store::Store(const Params & params) - : Config(params) + : StoreConfig(params) , state({(size_t) pathInfoCacheSize}) { } @@ -1009,7 +1009,6 @@ Derivation Store::readDerivation(const StorePath & drvPath) } } - } @@ -1019,9 +1018,6 @@ Derivation Store::readDerivation(const StorePath & drvPath) namespace nix { - -RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; - /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_) { @@ -1035,24 +1031,6 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_ return {uri, params}; } -ref<Store> openStore(const std::string & uri_, - const Store::Params & extraParams) -{ - auto [uri, uriParams] = splitUriAndParams(uri_); - auto params = extraParams; - params.insert(uriParams.begin(), uriParams.end()); - - for (auto fun : *RegisterStoreImplementation::implementations) { - auto store = fun(uri, params); - if (store) { - store->warnUnknownSettings(); - return ref<Store>(store); - } - } - - throw Error("don't know how to open Nix store '%s'", uri); -} - static bool isNonUriPath(const std::string & spec) { return // is not a URL @@ -1080,10 +1058,7 @@ StoreType getStoreType(const std::string & uri, const std::string & stateDir) } } - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr<Store> +std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Params & params) { switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) { case tDaemon: @@ -1098,8 +1073,41 @@ static RegisterStoreImplementation regStore([]( default: return nullptr; } -}); +} +ref<Store> openStore(const std::string & uri_, + const Store::Params & extraParams) +{ + auto params = extraParams; + try { + auto parsedUri = parseURL(uri_); + params.insert(parsedUri.query.begin(), parsedUri.query.end()); + + auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; + + for (auto implem : *Implementations::registered) { + if (implem.uriSchemes.count(parsedUri.scheme)) { + auto store = implem.create(parsedUri.scheme, baseURI, params); + if (store) { + store->init(); + store->warnUnknownSettings(); + return ref<Store>(store); + } + } + } + } + catch (BadURL &) { + auto [uri, uriParams] = splitUriAndParams(uri_); + params.insert(uriParams.begin(), uriParams.end()); + + if (auto store = openFromNonUri(uri, params)) { + store->warnUnknownSettings(); + return ref<Store>(store); + } + } + + throw Error("don't know how to open Nix store '%s'", uri_); +} std::list<ref<Store>> getDefaultSubstituters() { @@ -1133,5 +1141,6 @@ std::list<ref<Store>> getDefaultSubstituters() return stores; } +std::vector<StoreFactory> * Implementations::registered = 0; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8b42aa6d2..1bea4837b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -24,6 +24,31 @@ namespace nix { +/** + * About the class hierarchy of the store implementations: + * + * Each store type `Foo` consists of two classes: + * + * 1. A class `FooConfig : virtual StoreConfig` that contains the configuration + * for the store + * + * It should only contain members of type `const Setting<T>` (or subclasses + * of it) and inherit the constructors of `StoreConfig` + * (`using StoreConfig::StoreConfig`). + * + * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the + * implementation of the store. + * + * This class is expected to have a constructor `Foo(const Params & params)` + * that calls `StoreConfig(params)` (otherwise you're gonna encounter an + * `assertion failure` when trying to instantiate it). + * + * You can then register the new store using: + * + * ``` + * cpp static RegisterStoreImplementation<Foo, FooConfig> regStore; + * ``` + */ MakeError(SubstError, Error); MakeError(BuildError, Error); // denotes a permanent build failure @@ -33,6 +58,7 @@ MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); MakeError(BadStorePath, Error); +MakeError(InvalidStoreURI, Error); class FSAccessor; class NarInfoDiskCache; @@ -144,12 +170,31 @@ struct BuildResult } }; - -class Store : public std::enable_shared_from_this<Store>, public Config +struct StoreConfig : public Config { -public: - - typedef std::map<std::string, std::string> Params; + using Config::Config; + + /** + * When constructing a store implementation, we pass in a map `params` of + * parameters that's supposed to initialize the associated config. + * To do that, we must use the `StoreConfig(StringMap & params)` + * constructor, so we'd like to `delete` its default constructor to enforce + * it. + * + * However, actually deleting it means that all the subclasses of + * `StoreConfig` will have their default constructor deleted (because it's + * supposed to call the deleted default constructor of `StoreConfig`). But + * because we're always using virtual inheritance, the constructors of + * child classes will never implicitely call this one, so deleting it will + * be more painful than anything else. + * + * So we `assert(false)` here to ensure at runtime that the right + * constructor is always called without having to redefine a custom + * constructor for each `*Config` class. + */ + StoreConfig() { assert(false); } + + virtual const std::string name() = 0; const PathSetting storeDir_{this, false, settings.nixStore, "store", "path to the Nix store"}; @@ -167,6 +212,14 @@ public: "system-features", "Optional features that the system this store builds on implements (like \"kvm\")."}; +}; + +class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig +{ +public: + + typedef std::map<std::string, std::string> Params; + protected: struct PathInfoCacheValue { @@ -200,6 +253,11 @@ protected: Store(const Params & params); public: + /** + * Perform any necessary effectful operation to make the store up and + * running + */ + virtual void init() {}; virtual ~Store() { } @@ -626,22 +684,25 @@ protected: }; - -class LocalFSStore : public virtual Store +struct LocalFSStoreConfig : virtual StoreConfig { -public: - - // FIXME: the (Store*) cast works around a bug in gcc that causes + using StoreConfig::StoreConfig; + // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes // it to omit the call to the Setting constructor. Clang works fine // either way. - const PathSetting rootDir{(Store*) this, true, "", + const PathSetting rootDir{(StoreConfig*) this, true, "", "root", "directory prefixed to all other paths"}; - const PathSetting stateDir{(Store*) this, false, + const PathSetting stateDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, "state", "directory where Nix will store state"}; - const PathSetting logDir{(Store*) this, false, + const PathSetting logDir{(StoreConfig*) this, false, rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store state"}; +}; + +class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +{ +public: const static string drvsLogDir; @@ -744,25 +805,45 @@ StoreType getStoreType(const std::string & uri = settings.storeUri.get(), ‘substituters’ option and various legacy options. */ std::list<ref<Store>> getDefaultSubstituters(); +struct StoreFactory +{ + std::set<std::string> uriSchemes; + std::function<std::shared_ptr<Store> (const std::string & scheme, const std::string & uri, const Store::Params & params)> create; + std::function<std::shared_ptr<StoreConfig> ()> getConfig; +}; +struct Implementations +{ + static std::vector<StoreFactory> * registered; -/* Store implementation registration. */ -typedef std::function<std::shared_ptr<Store>( - const std::string & uri, const Store::Params & params)> OpenStore; + template<typename T, typename TConfig> + static void add() + { + if (!registered) registered = new std::vector<StoreFactory>(); + StoreFactory factory{ + .uriSchemes = T::uriSchemes(), + .create = + ([](const std::string & scheme, const std::string & uri, const Store::Params & params) + -> std::shared_ptr<Store> + { return std::make_shared<T>(scheme, uri, params); }), + .getConfig = + ([]() + -> std::shared_ptr<StoreConfig> + { return std::make_shared<TConfig>(StringMap({})); }) + }; + registered->push_back(factory); + } +}; +template<typename T, typename TConfig> struct RegisterStoreImplementation { - typedef std::vector<OpenStore> Implementations; - static Implementations * implementations; - - RegisterStoreImplementation(OpenStore fun) + RegisterStoreImplementation() { - if (!implementations) implementations = new Implementations; - implementations->push_back(fun); + Implementations::add<T, TConfig>(); } }; - /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ string showPaths(const PathSet & paths); diff --git a/src/libutil/abstractsettingtojson.hh b/src/libutil/abstractsettingtojson.hh new file mode 100644 index 000000000..b3fbc84f7 --- /dev/null +++ b/src/libutil/abstractsettingtojson.hh @@ -0,0 +1,15 @@ +#pragma once + +#include <nlohmann/json.hpp> +#include "config.hh" + +namespace nix { +template<typename T> +std::map<std::string, nlohmann::json> BaseSetting<T>::toJSONObject() +{ + auto obj = AbstractSetting::toJSONObject(); + obj.emplace("value", value); + obj.emplace("defaultValue", defaultValue); + return obj; +} +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 3cf720bce..309d23b40 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,5 +1,6 @@ #include "config.hh" #include "args.hh" +#include "abstractsettingtojson.hh" #include <nlohmann/json.hpp> @@ -137,11 +138,7 @@ nlohmann::json Config::toJSON() auto res = nlohmann::json::object(); for (auto & s : _settings) if (!s.second.isAlias) { - auto obj = nlohmann::json::object(); - obj.emplace("description", s.second.setting->description); - obj.emplace("aliases", s.second.setting->aliases); - obj.emplace("value", s.second.setting->toJSON()); - res.emplace(s.first, obj); + res.emplace(s.first, s.second.setting->toJSON()); } return res; } @@ -168,17 +165,19 @@ void AbstractSetting::setDefault(const std::string & str) nlohmann::json AbstractSetting::toJSON() { - return to_string(); + return nlohmann::json(toJSONObject()); } -void AbstractSetting::convertToArg(Args & args, const std::string & category) +std::map<std::string, nlohmann::json> AbstractSetting::toJSONObject() { + std::map<std::string, nlohmann::json> obj; + obj.emplace("description", description); + obj.emplace("aliases", aliases); + return obj; } -template<typename T> -nlohmann::json BaseSetting<T>::toJSON() +void AbstractSetting::convertToArg(Args & args, const std::string & category) { - return value; } template<typename T> @@ -259,11 +258,6 @@ template<> std::string BaseSetting<Strings>::to_string() const return concatStringsSep(" ", value); } -template<> nlohmann::json BaseSetting<Strings>::toJSON() -{ - return value; -} - template<> void BaseSetting<StringSet>::set(const std::string & str) { value = tokenizeString<StringSet>(str); @@ -274,11 +268,6 @@ template<> std::string BaseSetting<StringSet>::to_string() const return concatStringsSep(" ", value); } -template<> nlohmann::json BaseSetting<StringSet>::toJSON() -{ - return value; -} - template class BaseSetting<int>; template class BaseSetting<unsigned int>; template class BaseSetting<long>; diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 2b4265806..1f5f4e7b9 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -206,7 +206,9 @@ protected: virtual std::string to_string() const = 0; - virtual nlohmann::json toJSON(); + nlohmann::json toJSON(); + + virtual std::map<std::string, nlohmann::json> toJSONObject(); virtual void convertToArg(Args & args, const std::string & category); @@ -220,6 +222,7 @@ class BaseSetting : public AbstractSetting protected: T value; + const T defaultValue; public: @@ -229,6 +232,7 @@ public: const std::set<std::string> & aliases = {}) : AbstractSetting(name, description, aliases) , value(def) + , defaultValue(def) { } operator const T &() const { return value; } @@ -251,7 +255,7 @@ public: void convertToArg(Args & args, const std::string & category) override; - nlohmann::json toJSON() override; + std::map<std::string, nlohmann::json> toJSONObject() override; }; template<typename T> diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index c5abefe11..c7777a21f 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -161,7 +161,7 @@ namespace nix { Setting<std::string> setting{&config, "", "name-of-the-setting", "description"}; setting.assign("value"); - ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"description":"description\n","value":"value"}})#"); + ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#"); } TEST(Config, setSettingAlias) { diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 2ef88ef2a..1f716ba10 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -31,7 +31,7 @@ ParsedURL parseURL(const std::string & url); // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z+]+)"; +const static std::string schemeRegex = "(?:[a-z+.-]+)"; const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])"; diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc new file mode 100644 index 000000000..0cc2d9337 --- /dev/null +++ b/src/nix/describe-stores.cc @@ -0,0 +1,44 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +using namespace nix; + +struct CmdDescribeStores : Command, MixJSON +{ + std::string description() override + { + return "show registered store types and their available options"; + } + + Category category() override { return catUtility; } + + void run() override + { + auto res = nlohmann::json::object(); + for (auto & implem : *Implementations::registered) { + auto storeConfig = implem.getConfig(); + auto storeName = storeConfig->name(); + res[storeName] = storeConfig->toJSON(); + } + if (json) { + std::cout << res; + } else { + for (auto & [storeName, storeConfig] : res.items()) { + std::cout << "## " << storeName << std::endl << std::endl; + for (auto & [optionName, optionDesc] : storeConfig.items()) { + std::cout << "### " << optionName << std::endl << std::endl; + std::cout << optionDesc["description"].get<std::string>() << std::endl; + std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl; + if (!optionDesc["aliases"].empty()) + std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl; + } + } + } + } +}; + +static auto r1 = registerCommand<CmdDescribeStores>("describe-stores"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index a2ce9c8c1..f29fa71d2 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -392,7 +392,7 @@ struct CmdDevelop : Common, MixEnvironment auto bashInstallable = std::make_shared<InstallableFlake>( state, - std::move(installable->nixpkgsFlakeRef()), + installable->nixpkgsFlakeRef(), Strings{"bashInteractive"}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, lockFlags); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 4199dae0f..0dc99d05e 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -81,7 +81,7 @@ void printClosureDiff( auto beforeSize = totalSize(beforeVersions); auto afterSize = totalSize(afterVersions); auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize; - auto showDelta = abs(sizeDelta) >= 8 * 1024; + auto showDelta = std::abs(sizeDelta) >= 8 * 1024; std::set<std::string> removed, unchanged; for (auto & [version, _] : beforeVersions) diff --git a/src/nix/installables.hh b/src/nix/installables.hh index 41c75a4ed..c7c2f8981 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -69,7 +69,7 @@ struct Installable virtual FlakeRef nixpkgsFlakeRef() const { - return std::move(FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}})); + return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); } }; diff --git a/tests/describe-stores.sh b/tests/describe-stores.sh new file mode 100644 index 000000000..3fea61483 --- /dev/null +++ b/tests/describe-stores.sh @@ -0,0 +1,8 @@ +source common.sh + +# Query an arbitrary value in `nix describe-stores --json`'s output just to +# check that it has the right structure +[[ $(nix --experimental-features 'nix-command flakes' describe-stores --json | jq '.["SSH Store"]["compress"]["defaultValue"]') == false ]] + +# Ensure that the output of `nix describe-stores` isn't empty +[[ -n $(nix --experimental-features 'nix-command flakes' describe-stores) ]] diff --git a/tests/local.mk b/tests/local.mk index 5d25de019..594901504 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -32,6 +32,7 @@ nix_tests = \ post-hook.sh \ function-trace.sh \ recursive.sh \ + describe-stores.sh \ flakes.sh \ content-addressed.sh # parallel.sh |