aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libstore/binary-cache-store.cc292
-rw-r--r--src/libstore/binary-cache-store.hh170
-rw-r--r--src/libstore/local-binary-cache-store.cc44
-rw-r--r--src/libstore/local-binary-cache-store.hh31
-rw-r--r--src/libstore/store-api.cc18
-rw-r--r--src/libstore/store-api.hh24
-rw-r--r--src/libutil/lru-cache.hh84
7 files changed, 658 insertions, 5 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
new file mode 100644
index 000000000..6bea0e6c9
--- /dev/null
+++ b/src/libstore/binary-cache-store.cc
@@ -0,0 +1,292 @@
+#include "binary-cache-store.hh"
+#include "sync.hh"
+
+#include "archive.hh"
+#include "compression.hh"
+#include "derivations.hh"
+#include "globals.hh"
+#include "nar-info.hh"
+#include "worker-protocol.hh"
+
+#include <chrono>
+
+namespace nix {
+
+BinaryCacheStore::BinaryCacheStore(std::shared_ptr<Store> localStore,
+ const Path & secretKeyFile, const Path & publicKeyFile)
+ : localStore(localStore)
+{
+ if (secretKeyFile != "")
+ secretKey = std::unique_ptr<SecretKey>(new SecretKey(readFile(secretKeyFile)));
+
+ if (publicKeyFile != "") {
+ publicKeys = std::unique_ptr<PublicKeys>(new PublicKeys);
+ auto key = PublicKey(readFile(publicKeyFile));
+ publicKeys->emplace(key.name, key);
+ }
+}
+
+void BinaryCacheStore::init()
+{
+ std::string cacheInfoFile = "nix-cache-info";
+ if (!fileExists(cacheInfoFile))
+ upsertFile(cacheInfoFile, "StoreDir: " + settings.nixStore + "\n");
+}
+
+const BinaryCacheStore::Stats & BinaryCacheStore::getStats()
+{
+ return stats;
+}
+
+Path BinaryCacheStore::narInfoFileFor(const Path & storePath)
+{
+ assertStorePath(storePath);
+ return storePathToHash(storePath) + ".narinfo";
+}
+
+void BinaryCacheStore::addToCache(const ValidPathInfo & info,
+ const string & nar)
+{
+ auto narInfoFile = narInfoFileFor(info.path);
+ if (fileExists(narInfoFile)) return;
+
+ auto narInfo = make_ref<NarInfo>(info);
+
+ narInfo->narSize = nar.size();
+ narInfo->narHash = hashString(htSHA256, nar);
+
+ if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash)
+ throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
+
+ /* Compress the NAR. */
+ narInfo->compression = "xz";
+ auto now1 = std::chrono::steady_clock::now();
+ string narXz = compressXZ(nar);
+ auto now2 = std::chrono::steady_clock::now();
+ narInfo->fileHash = hashString(htSHA256, narXz);
+ narInfo->fileSize = narXz.size();
+
+ auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
+ printMsg(lvlTalkative, format("copying path ‘%1%’ (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache")
+ % narInfo->path % narInfo->narSize
+ % ((1.0 - (double) narXz.size() / nar.size()) * 100.0)
+ % duration);
+
+ /* Atomically write the NAR file. */
+ narInfo->url = "nar/" + printHash32(narInfo->fileHash) + ".nar.xz";
+ if (!fileExists(narInfo->url)) {
+ stats.narWrite++;
+ upsertFile(narInfo->url, narXz);
+ } else
+ stats.narWriteAverted++;
+
+ stats.narWriteBytes += nar.size();
+ stats.narWriteCompressedBytes += narXz.size();
+ stats.narWriteCompressionTimeMs += duration;
+
+ /* Atomically write the NAR info file.*/
+ if (secretKey) narInfo->sign(*secretKey);
+
+ upsertFile(narInfoFile, narInfo->to_string());
+
+ {
+ auto state_(state.lock());
+ state_->narInfoCache.upsert(narInfo->path, narInfo);
+ stats.narInfoCacheSize = state_->narInfoCache.size();
+ }
+
+ stats.narInfoWrite++;
+}
+
+NarInfo BinaryCacheStore::readNarInfo(const Path & storePath)
+{
+ {
+ auto state_(state.lock());
+ auto res = state_->narInfoCache.get(storePath);
+ if (res) {
+ stats.narInfoReadAverted++;
+ return **res;
+ }
+ }
+
+ auto narInfoFile = narInfoFileFor(storePath);
+ auto narInfo = make_ref<NarInfo>(getFile(narInfoFile), narInfoFile);
+ assert(narInfo->path == storePath);
+
+ stats.narInfoRead++;
+
+ if (publicKeys) {
+ if (!narInfo->checkSignature(*publicKeys))
+ throw Error(format("invalid signature on NAR info file ‘%1%’") % narInfoFile);
+ }
+
+ {
+ auto state_(state.lock());
+ state_->narInfoCache.upsert(storePath, narInfo);
+ stats.narInfoCacheSize = state_->narInfoCache.size();
+ }
+
+ return *narInfo;
+}
+
+bool BinaryCacheStore::isValidPath(const Path & storePath)
+{
+ return fileExists(narInfoFileFor(storePath));
+}
+
+void BinaryCacheStore::exportPath(const Path & storePath, bool sign, Sink & sink)
+{
+ assert(!sign);
+
+ auto res = readNarInfo(storePath);
+
+ auto nar = getFile(res.url);
+
+ stats.narRead++;
+ stats.narReadCompressedBytes += nar.size();
+
+ /* Decompress the NAR. FIXME: would be nice to have the remote
+ side do this. */
+ if (res.compression == "none")
+ ;
+ else if (res.compression == "xz")
+ nar = decompressXZ(nar);
+ else
+ throw Error(format("unknown NAR compression type ‘%1%’") % nar);
+
+ stats.narReadBytes += nar.size();
+
+ printMsg(lvlTalkative, format("exporting path ‘%1%’ (%2% bytes)") % storePath % nar.size());
+
+ assert(nar.size() % 8 == 0);
+
+ sink((unsigned char *) nar.c_str(), nar.size());
+
+ // FIXME: check integrity of NAR.
+
+ sink << exportMagic << storePath << res.references << res.deriver << 0;
+}
+
+Paths BinaryCacheStore::importPaths(bool requireSignature, Source & source)
+{
+ assert(!requireSignature);
+ Paths res;
+ while (true) {
+ unsigned long long n = readLongLong(source);
+ if (n == 0) break;
+ if (n != 1) throw Error("input doesn't look like something created by ‘nix-store --export’");
+ res.push_back(importPath(source));
+ }
+ return res;
+}
+
+struct TeeSource : Source
+{
+ Source & readSource;
+ std::string data;
+ TeeSource(Source & readSource) : readSource(readSource)
+ {
+ }
+ size_t read(unsigned char * data, size_t len)
+ {
+ size_t n = readSource.read(data, len);
+ this->data.append((char *) data, n);
+ return n;
+ }
+};
+
+struct NopSink : ParseSink
+{
+};
+
+Path BinaryCacheStore::importPath(Source & source)
+{
+ /* FIXME: some cut&paste of LocalStore::importPath(). */
+
+ /* Extract the NAR from the source. */
+ TeeSource tee(source);
+ NopSink sink;
+ parseDump(sink, tee);
+
+ uint32_t magic = readInt(source);
+ if (magic != exportMagic)
+ throw Error("Nix archive cannot be imported; wrong format");
+
+ ValidPathInfo info;
+ info.path = readStorePath(source);
+
+ info.references = readStorePaths<PathSet>(source);
+
+ readString(source); // deriver, don't care
+
+ bool haveSignature = readInt(source) == 1;
+ assert(!haveSignature);
+
+ addToCache(info, tee.data);
+
+ return info.path;
+}
+
+ValidPathInfo BinaryCacheStore::queryPathInfo(const Path & storePath)
+{
+ return ValidPathInfo(readNarInfo(storePath));
+}
+
+void BinaryCacheStore::querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos)
+{
+ PathSet left;
+
+ if (!localStore) return;
+
+ for (auto & storePath : paths) {
+ if (!localStore->isValidPath(storePath)) {
+ left.insert(storePath);
+ continue;
+ }
+ ValidPathInfo info = localStore->queryPathInfo(storePath);
+ SubstitutablePathInfo sub;
+ sub.references = info.references;
+ sub.downloadSize = 0;
+ sub.narSize = info.narSize;
+ infos.emplace(storePath, sub);
+ }
+
+ if (settings.useSubstitutes)
+ localStore->querySubstitutablePathInfos(left, infos);
+}
+
+void BinaryCacheStore::buildPaths(const PathSet & paths, BuildMode buildMode)
+{
+ for (auto & storePath : paths) {
+ assert(!isDerivation(storePath));
+
+ if (isValidPath(storePath)) continue;
+
+ if (!localStore)
+ throw Error(format("don't know how to realise path ‘%1%’ in a binary cache") % storePath);
+
+ localStore->addTempRoot(storePath);
+
+ if (!localStore->isValidPath(storePath))
+ localStore->ensurePath(storePath);
+
+ ValidPathInfo info = localStore->queryPathInfo(storePath);
+
+ for (auto & ref : info.references)
+ if (ref != storePath)
+ ensurePath(ref);
+
+ StringSink sink;
+ dumpPath(storePath, sink);
+
+ addToCache(info, sink.s);
+ }
+}
+
+void BinaryCacheStore::ensurePath(const Path & path)
+{
+ buildPaths({path});
+}
+
+}
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
new file mode 100644
index 000000000..d02f46dea
--- /dev/null
+++ b/src/libstore/binary-cache-store.hh
@@ -0,0 +1,170 @@
+#pragma once
+
+#include "crypto.hh"
+#include "store-api.hh"
+
+#include "lru-cache.hh"
+#include "sync.hh"
+#include "pool.hh"
+
+#include <atomic>
+
+namespace nix {
+
+struct NarInfo;
+
+class BinaryCacheStore : public Store
+{
+private:
+
+ std::unique_ptr<SecretKey> secretKey;
+ std::unique_ptr<PublicKeys> publicKeys;
+
+ std::shared_ptr<Store> localStore;
+
+ struct State
+ {
+ LRUCache<Path, ref<NarInfo>> narInfoCache{32 * 1024};
+ };
+
+ Sync<State> state;
+
+protected:
+
+ BinaryCacheStore(std::shared_ptr<Store> localStore,
+ const Path & secretKeyFile, const Path & publicKeyFile);
+
+ virtual bool fileExists(const std::string & path) = 0;
+
+ virtual void upsertFile(const std::string & path, const std::string & data) = 0;
+
+ virtual std::string getFile(const std::string & path) = 0;
+
+public:
+
+ virtual void init();
+
+ struct Stats
+ {
+ std::atomic<uint64_t> narInfoRead{0};
+ std::atomic<uint64_t> narInfoReadAverted{0};
+ std::atomic<uint64_t> narInfoWrite{0};
+ std::atomic<uint64_t> narInfoCacheSize{0};
+ std::atomic<uint64_t> narRead{0};
+ std::atomic<uint64_t> narReadBytes{0};
+ std::atomic<uint64_t> narReadCompressedBytes{0};
+ std::atomic<uint64_t> narWrite{0};
+ std::atomic<uint64_t> narWriteAverted{0};
+ std::atomic<uint64_t> narWriteBytes{0};
+ std::atomic<uint64_t> narWriteCompressedBytes{0};
+ std::atomic<uint64_t> narWriteCompressionTimeMs{0};
+ };
+
+ const Stats & getStats();
+
+private:
+
+ Stats stats;
+
+ std::string narInfoFileFor(const Path & storePath);
+
+ void addToCache(const ValidPathInfo & info, const string & nar);
+
+protected:
+
+ NarInfo readNarInfo(const Path & storePath);
+
+public:
+
+ bool isValidPath(const Path & path) override;
+
+ PathSet queryValidPaths(const PathSet & paths) override
+ { abort(); }
+
+ PathSet queryAllValidPaths() override
+ { abort(); }
+
+ ValidPathInfo queryPathInfo(const Path & path) override;
+
+ Hash queryPathHash(const Path & path) override
+ { abort(); }
+
+ void queryReferrers(const Path & path,
+ PathSet & referrers) override
+ { abort(); }
+
+ Path queryDeriver(const Path & path) override
+ { abort(); }
+
+ PathSet queryValidDerivers(const Path & path) override
+ { abort(); }
+
+ PathSet queryDerivationOutputs(const Path & path) override
+ { abort(); }
+
+ StringSet queryDerivationOutputNames(const Path & path) override
+ { abort(); }
+
+ Path queryPathFromHashPart(const string & hashPart) override
+ { abort(); }
+
+ PathSet querySubstitutablePaths(const PathSet & paths) override
+ { abort(); }
+
+ void querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos) override;
+
+ Path addToStore(const string & name, const Path & srcPath,
+ bool recursive = true, HashType hashAlgo = htSHA256,
+ PathFilter & filter = defaultPathFilter, bool repair = false) override
+ { abort(); }
+
+ Path addTextToStore(const string & name, const string & s,
+ const PathSet & references, bool repair = false) override
+ { abort(); }
+
+ void exportPath(const Path & path, bool sign,
+ Sink & sink) override;
+
+ Paths importPaths(bool requireSignature, Source & source) override;
+
+ Path importPath(Source & source);
+
+ void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal) override;
+
+ BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv,
+ BuildMode buildMode = bmNormal) override
+ { abort(); }
+
+ void ensurePath(const Path & path) override;
+
+ void addTempRoot(const Path & path) override
+ { abort(); }
+
+ void addIndirectRoot(const Path & path) override
+ { abort(); }
+
+ void syncWithGC() override
+ { }
+
+ Roots findRoots() override
+ { abort(); }
+
+ void collectGarbage(const GCOptions & options, GCResults & results) override
+ { abort(); }
+
+ PathSet queryFailedPaths() override
+ { return PathSet(); }
+
+ void clearFailedPaths(const PathSet & paths) override
+ { }
+
+ void optimiseStore() override
+ { }
+
+ bool verifyStore(bool checkContents, bool repair) override
+ { return true; }
+
+};
+
+}
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
new file mode 100644
index 000000000..5714688e0
--- /dev/null
+++ b/src/libstore/local-binary-cache-store.cc
@@ -0,0 +1,44 @@
+#include "local-binary-cache-store.hh"
+
+namespace nix {
+
+LocalBinaryCacheStore::LocalBinaryCacheStore(std::shared_ptr<Store> localStore,
+ const Path & secretKeyFile, const Path & publicKeyFile,
+ const Path & binaryCacheDir)
+ : BinaryCacheStore(localStore, secretKeyFile, publicKeyFile)
+ , binaryCacheDir(binaryCacheDir)
+{
+}
+
+void LocalBinaryCacheStore::init()
+{
+ createDirs(binaryCacheDir + "/nar");
+ BinaryCacheStore::init();
+}
+
+static void atomicWrite(const Path & path, const std::string & s)
+{
+ Path tmp = path + ".tmp." + std::to_string(getpid());
+ AutoDelete del(tmp, false);
+ writeFile(tmp, s);
+ if (rename(tmp.c_str(), path.c_str()))
+ throw SysError(format("renaming ‘%1%’ to ‘%2%’") % tmp % path);
+ del.cancel();
+}
+
+bool LocalBinaryCacheStore::fileExists(const std::string & path)
+{
+ return pathExists(binaryCacheDir + "/" + path);
+}
+
+void LocalBinaryCacheStore::upsertFile(const std::string & path, const std::string & data)
+{
+ atomicWrite(binaryCacheDir + "/" + path, data);
+}
+
+std::string LocalBinaryCacheStore::getFile(const std::string & path)
+{
+ return readFile(binaryCacheDir + "/" + path);
+}
+
+}
diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh
new file mode 100644
index 000000000..0303ebe73
--- /dev/null
+++ b/src/libstore/local-binary-cache-store.hh
@@ -0,0 +1,31 @@
+#pragma once
+
+#include "binary-cache-store.hh"
+
+namespace nix {
+
+class LocalBinaryCacheStore : public BinaryCacheStore
+{
+private:
+
+ Path binaryCacheDir;
+
+public:
+
+ LocalBinaryCacheStore(std::shared_ptr<Store> localStore,
+ const Path & secretKeyFile, const Path & publicKeyFile,
+ const Path & binaryCacheDir);
+
+ void init() override;
+
+protected:
+
+ bool fileExists(const std::string & path) override;
+
+ void upsertFile(const std::string & path, const std::string & data) override;
+
+ std::string getFile(const std::string & path) override;
+
+};
+
+}
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 82872cc33..f9c876770 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -313,18 +313,24 @@ void Store::exportPaths(const Paths & paths,
#include "local-store.hh"
-#include "serialise.hh"
#include "remote-store.hh"
+#include "local-binary-cache-store.hh"
namespace nix {
-ref<Store> openStore(bool reserveSpace)
+ref<Store> openStoreAt(const std::string & uri, bool reserveSpace)
{
+ if (std::string(uri, 0, 7) == "file://") {
+ return make_ref<LocalBinaryCacheStore>(std::shared_ptr<Store>(0),
+ "", "", // FIXME: allow the signing key to be set
+ std::string(uri, 7));
+ }
+
enum { mDaemon, mLocal, mAuto } mode;
- mode = getEnv("NIX_REMOTE") == "daemon" ? mDaemon : mAuto;
+ mode = uri == "daemon" ? mDaemon : mAuto;
if (mode == mAuto) {
if (LocalStore::haveWriteAccess())
@@ -341,4 +347,10 @@ ref<Store> openStore(bool reserveSpace)
}
+ref<Store> openStore(bool reserveSpace)
+{
+ return openStoreAt(getEnv("NIX_REMOTE"), reserveSpace);
+}
+
+
}
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 6f50e3c55..5fe42c973 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -419,8 +419,28 @@ Path computeStorePathForText(const string & name, const string & s,
void removeTempRoots();
-/* Factory method: open the Nix database, either through the local or
- remote implementation. */
+/* Return a Store object to access the Nix store denoted by
+ ‘uri’ (slight misnomer...). Supported values are:
+
+ * ‘direct’: The Nix store in /nix/store and database in
+ /nix/var/nix/db, accessed directly.
+
+ * ‘daemon’: The Nix store accessed via a Unix domain socket
+ connection to nix-daemon.
+
+ * ‘file://<path>’: A binary cache stored in <path>.
+
+ If ‘uri’ is empty, it defaults to ‘direct’ or ‘daemon’ depending on
+ whether the user has write access to the local Nix store/database.
+
+ The Boolean ‘reserveSpace’ denotes whether some disk space should
+ be reserved to enable future garbage collector runs. It should be
+ set to true *unless* you're going to collect garbage.
+*/
+ref<Store> openStoreAt(const std::string & uri, bool reserveSpace = true);
+
+
+/* Open the store indicated by the ‘NIX_REMOTE’ environment variable. */
ref<Store> openStore(bool reserveSpace = true);
diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh
new file mode 100644
index 000000000..3c6569cf4
--- /dev/null
+++ b/src/libutil/lru-cache.hh
@@ -0,0 +1,84 @@
+#pragma once
+
+#include <map>
+#include <list>
+
+namespace nix {
+
+/* A simple least-recently used cache. Not thread-safe. */
+template<typename Key, typename Value>
+class LRUCache
+{
+private:
+
+ size_t maxSize;
+
+ // Stupid wrapper to get around circular dependency between Data
+ // and LRU.
+ struct LRUIterator;
+
+ using Data = std::map<Key, std::pair<LRUIterator, Value>>;
+ using LRU = std::list<typename Data::iterator>;
+
+ struct LRUIterator { typename LRU::iterator it; };
+
+ Data data;
+ LRU lru;
+
+public:
+
+ LRUCache(size_t maxSize) : maxSize(maxSize) { }
+
+ /* Insert or upsert an item in the cache. */
+ void upsert(const Key & key, const Value & value)
+ {
+ erase(key);
+
+ if (data.size() >= maxSize) {
+ /* Retire the oldest item. */
+ auto oldest = lru.begin();
+ data.erase(*oldest);
+ lru.erase(oldest);
+ }
+
+ auto res = data.emplace(key, std::make_pair(LRUIterator(), value));
+ assert(res.second);
+ auto & i(res.first);
+
+ auto j = lru.insert(lru.end(), i);
+
+ i->second.first.it = j;
+ }
+
+ bool erase(const Key & key)
+ {
+ auto i = data.find(key);
+ if (i == data.end()) return false;
+ lru.erase(i->second.first.it);
+ data.erase(i);
+ return true;
+ }
+
+ /* Look up an item in the cache. If it's exists, it becomes the
+ most recently used item. */
+ // FIXME: use boost::optional?
+ Value * get(const Key & key)
+ {
+ auto i = data.find(key);
+ if (i == data.end()) return 0;
+
+ /* Move this item to the back of the LRU list. */
+ lru.erase(i->second.first.it);
+ auto j = lru.insert(lru.end(), i);
+ i->second.first.it = j;
+
+ return &i->second.second;
+ }
+
+ size_t size()
+ {
+ return data.size();
+ }
+};
+
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 12:55:42 +0000