diff options
| -rw-r--r-- | src/libstore/binary-cache-store.cc | 152 | ||||
| -rw-r--r-- | src/libstore/binary-cache-store.hh | 17 | ||||
| -rw-r--r-- | src/libstore/export-import.cc | 5 | ||||
| -rw-r--r-- | src/libstore/local-store.cc | 2 | ||||
| -rw-r--r-- | src/libstore/local-store.hh | 9 | ||||
| -rw-r--r-- | src/libstore/remote-store.cc | 2 | ||||
| -rw-r--r-- | src/libstore/remote-store.hh | 3 | ||||
| -rw-r--r-- | src/libstore/store-api.hh | 14 | ||||
| -rw-r--r-- | src/nix-daemon/nix-daemon.cc | 2 |
9 files changed, 97 insertions, 109 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 7692da742..4125af118 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -17,6 +17,66 @@ namespace nix { +/* Given requests for a path /nix/store/<x>/<y>, this accessor will + first download the NAR for /nix/store/<x> from the binary cache, + build a NAR accessor for that NAR, and use that to access <y>. */ +struct BinaryCacheStoreAccessor : public FSAccessor +{ + ref<BinaryCacheStore> store; + + std::map<Path, ref<FSAccessor>> nars; + + BinaryCacheStoreAccessor(ref<BinaryCacheStore> store) + : store(store) + { + } + + std::pair<ref<FSAccessor>, Path> fetch(const Path & path_) + { + auto path = canonPath(path_); + + auto storePath = store->toStorePath(path); + std::string restPath = std::string(path, storePath.size()); + + if (!store->isValidPath(storePath)) + throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath); + + auto i = nars.find(storePath); + if (i != nars.end()) return {i->second, restPath}; + + StringSink sink; + store->narFromPath(storePath, sink); + + auto accessor = makeNarAccessor(sink.s); + nars.emplace(storePath, accessor); + return {accessor, restPath}; + } + + Stat stat(const Path & path) override + { + auto res = fetch(path); + return res.first->stat(res.second); + } + + StringSet readDirectory(const Path & path) override + { + auto res = fetch(path); + return res.first->readDirectory(res.second); + } + + std::string readFile(const Path & path) override + { + auto res = fetch(path); + return res.first->readFile(res.second); + } + + std::string readLink(const Path & path) override + { + auto res = fetch(path); + return res.first->readLink(res.second); + } +}; + BinaryCacheStore::BinaryCacheStore(const Params & params) : Store(params) , compression(get(params, "compression", "xz")) @@ -82,7 +142,7 @@ Path BinaryCacheStore::narInfoFileFor(const Path & storePath) } void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs) + bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) { if (!repair && isValidPath(info.path)) return; @@ -109,6 +169,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str if (info.narHash && info.narHash != narInfo->narHash) throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path); + auto accessor_ = std::dynamic_pointer_cast<BinaryCacheStoreAccessor>(accessor); + /* Optionally write a JSON file containing a listing of the contents of the NAR. */ if (writeNARListing) { @@ -118,12 +180,15 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str JSONObject jsonRoot(jsonOut); jsonRoot.attr("version", 1); - auto accessor = makeNarAccessor(nar); + auto narAccessor = makeNarAccessor(nar); + + if (accessor_) + accessor_->nars.emplace(info.path, narAccessor); std::function<void(const Path &, JSONPlaceholder &)> recurse; recurse = [&](const Path & path, JSONPlaceholder & res) { - auto st = accessor->stat(path); + auto st = narAccessor->stat(path); auto obj = res.object(); @@ -138,7 +203,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str obj.attr("type", "directory"); { auto res2 = obj.object("entries"); - for (auto & name : accessor->readDirectory(path)) { + for (auto & name : narAccessor->readDirectory(path)) { auto res3 = res2.placeholder(name); recurse(path + "/" + name, res3); } @@ -146,7 +211,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str break; case FSAccessor::Type::tSymlink: obj.attr("type", "symlink"); - obj.attr("target", accessor->readLink(path)); + obj.attr("target", narAccessor->readLink(path)); break; default: abort(); @@ -162,6 +227,11 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str upsertFile(storePathToHash(info.path) + ".ls.xz", *compress("xz", jsonOut.str())); } + else { + if (accessor_) + accessor_->nars.emplace(info.path, makeNarAccessor(nar)); + } + /* Compress the NAR. */ narInfo->compression = compression; auto now1 = std::chrono::steady_clock::now(); @@ -286,7 +356,7 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, ValidPathInfo info; info.path = makeFixedOutputPath(recursive, h, name); - addToStore(info, sink.s, repair); + addToStore(info, sink.s, repair, false, 0); return info.path; } @@ -301,84 +371,16 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToStore(info, sink.s, repair); + addToStore(info, sink.s, repair, false, 0); } return info.path; } -/* Given requests for a path /nix/store/<x>/<y>, this accessor will - first download the NAR for /nix/store/<x> from the binary cache, - build a NAR accessor for that NAR, and use that to access <y>. */ -struct BinaryCacheStoreAccessor : public FSAccessor -{ - ref<BinaryCacheStore> store; - - std::map<Path, ref<FSAccessor>> nars; - - BinaryCacheStoreAccessor(ref<BinaryCacheStore> store) - : store(store) - { - } - - std::pair<ref<FSAccessor>, Path> fetch(const Path & path_) - { - auto path = canonPath(path_); - - auto storePath = store->toStorePath(path); - std::string restPath = std::string(path, storePath.size()); - - if (!store->isValidPath(storePath)) - throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath); - - auto i = nars.find(storePath); - if (i != nars.end()) return {i->second, restPath}; - - StringSink sink; - store->narFromPath(storePath, sink); - - auto accessor = makeNarAccessor(sink.s); - nars.emplace(storePath, accessor); - return {accessor, restPath}; - } - - Stat stat(const Path & path) override - { - auto res = fetch(path); - return res.first->stat(res.second); - } - - StringSet readDirectory(const Path & path) override - { - auto res = fetch(path); - return res.first->readDirectory(res.second); - } - - std::string readFile(const Path & path) override - { - auto res = fetch(path); - return res.first->readFile(res.second); - } - - std::string readLink(const Path & path) override - { - auto res = fetch(path); - return res.first->readLink(res.second); - } -}; - ref<FSAccessor> BinaryCacheStore::getFSAccessor() { return make_ref<BinaryCacheStoreAccessor>(ref<BinaryCacheStore>( std::dynamic_pointer_cast<BinaryCacheStore>(shared_from_this()))); } -void BinaryCacheStore::addPathToAccessor(ref<FSAccessor> accessor, - const Path & storePath, const ref<std::string> & data) -{ - auto accessor_ = accessor.dynamic_pointer_cast<BinaryCacheStoreAccessor>(); - if (accessor_) - accessor_->nars.emplace(storePath, makeNarAccessor(data)); -} - } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 333cf0856..31878bbb2 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -93,22 +93,23 @@ public: bool wantMassQuery() override { return wantMassQuery_; } void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair = false, bool dontCheckSigs = false) override; + bool repair, bool dontCheckSigs, + std::shared_ptr<FSAccessor> accessor) override; Path addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false) override; + bool recursive, HashType hashAlgo, + PathFilter & filter, bool repair) override; Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false) override; + const PathSet & references, bool repair) override; void narFromPath(const Path & path, Sink & sink) override; - void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override + void buildPaths(const PathSet & paths, BuildMode buildMode) override { notImpl(); } BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, - BuildMode buildMode = bmNormal) override + BuildMode buildMode) override { notImpl(); } void ensurePath(const Path & path) override @@ -137,10 +138,6 @@ public: ref<FSAccessor> getFSAccessor() override; -private: - - void addPathToAccessor(ref<FSAccessor>, const Path & storePath, const ref<std::string> & data) override; - public: void addSignatures(const Path & storePath, const StringSet & sigs) override diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 883b87217..c5618c826 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -117,10 +117,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, if (readInt(source) == 1) readString(source); - addToStore(info, tee.data, false, dontCheckSigs); - - if (accessor) - addPathToAccessor(ref<FSAccessor>(accessor), info.path, tee.data); + addToStore(info, tee.data, false, dontCheckSigs, accessor); res.push_back(info.path); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2c2475afc..d3a641fd9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -910,7 +910,7 @@ void LocalStore::invalidatePath(State & state, const Path & path) void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs) + bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) { Hash h = hashString(htSHA256, *nar); if (h != info.narHash) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 86f03526d..511209d84 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -126,11 +126,12 @@ public: SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs) override; + bool repair, bool dontCheckSigs, + std::shared_ptr<FSAccessor> accessor) override; Path addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, bool repair = false) override; + bool recursive, HashType hashAlgo, + PathFilter & filter, bool repair) override; /* Like addToStore(), but the contents of the path are contained in `dump', which is either a NAR serialisation (if recursive == @@ -140,7 +141,7 @@ public: bool recursive = true, HashType hashAlgo = htSHA256, bool repair = false); Path addTextToStore(const string & name, const string & s, - const PathSet & references, bool repair = false) override; + const PathSet & references, bool repair) override; void buildPaths(const PathSet & paths, BuildMode buildMode) override; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index de51716db..7a041c6e0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -333,7 +333,7 @@ Path RemoteStore::queryPathFromHashPart(const string & hashPart) void RemoteStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs) + bool repair, bool dontCheckSigs, std::shared_ptr<FSAccessor> accessor) { throw Error("RemoteStore::addToStore() not implemented"); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 5932fda95..b14ce4a97 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -54,7 +54,8 @@ public: SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair, bool dontCheckSigs) override; + bool repair, bool dontCheckSigs, + std::shared_ptr<FSAccessor> accessor) override; Path addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d6e181196..94dc8ab1a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -366,7 +366,8 @@ public: /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - bool repair = false, bool dontCheckSigs = false) = 0; + bool repair = false, bool dontCheckSigs = false, + std::shared_ptr<FSAccessor> accessor = 0) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. @@ -461,17 +462,6 @@ public: /* Return an object to access files in the Nix store. */ virtual ref<FSAccessor> getFSAccessor() = 0; -private: - - /* Inform an accessor about the NAR contents of a store path. Used - by importPaths() to speed up subsequent access to the imported - paths when used with binary cache stores. */ - virtual void addPathToAccessor(ref<FSAccessor>, const Path & storePath, const ref<std::string> & data) - { - } - -public: - /* Add signatures to the specified store path. The signatures are not verified. */ virtual void addSignatures(const Path & storePath, const StringSet & sigs) = 0; diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 5ed38871f..41c847dad 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -304,7 +304,7 @@ static void performOp(ref<LocalStore> store, bool trusted, unsigned int clientVe string s = readString(from); PathSet refs = readStorePaths<PathSet>(*store, from); startWork(); - Path path = store->addTextToStore(suffix, s, refs); + Path path = store->addTextToStore(suffix, s, refs, false); stopWork(); to << path; break; |