From ca580bec35ea4d1984e36864158d7be99cfcb34b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Oct 2017 21:15:33 +0200 Subject: BinaryCacheStore: Support local caching of NARs This speeds up commands like "nix cat-store". For example: $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nar-cache /nix/store/i60yncmq6w9dyv37zd2k454g0fkl3arl-systemd-234/etc/udev/udev.conf real 0m4.336s $ time nix cat-store --store https://cache.nixos.org?local-nar-cache=/tmp/nar-cache /nix/store/i60yncmq6w9dyv37zd2k454g0fkl3arl-systemd-234/etc/udev/udev.conf real 0m0.045s The primary motivation is to allow hydra-server to serve files from S3 binary caches. Previously Hydra had a hack to do "nix-store -r ", but that fetches the entire closure so is prohibitively expensive. There is no garbage collection of the NAR cache yet. Also, the entire NAR is read when accessing a single member file. We could generate the NAR listing to provide random access. Note: the NAR cache is indexed by the store path hash, not the content hash, so NAR caches should not be shared between binary caches, unless you're sure that all your builds are binary-reproducible. --- src/libstore/remote-fs-accessor.cc | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) (limited to 'src/libstore/remote-fs-accessor.cc') diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 098151f8c..da4e30b22 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -3,10 +3,12 @@ namespace nix { - -RemoteFSAccessor::RemoteFSAccessor(ref store) +RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) : store(store) + , cacheDir(cacheDir) { + if (cacheDir != "") + createDirs(cacheDir); } std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) @@ -23,7 +25,21 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) if (i != nars.end()) return {i->second, restPath}; StringSink sink; - store->narFromPath(storePath, sink); + + Path cacheFile = cacheDir != "" ? fmt("%s/%s.nar", cacheDir, storePathToHash(storePath)) : ""; + + try { + if (cacheFile != "") + *sink.s = nix::readFile(cacheFile); + } catch (SysError &) { } + + if (sink.s->empty()) { + store->narFromPath(storePath, sink); + + if (cacheFile != "") + /* FIXME: do this asynchronously. */ + writeFile(cacheFile, *sink.s); + } auto accessor = makeNarAccessor(sink.s); nars.emplace(storePath, accessor); -- cgit v1.2.3