aboutsummaryrefslogtreecommitdiff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/binary-cache-store.cc8
-rw-r--r--src/libstore/local-store.cc4
-rw-r--r--src/libstore/nar-info-disk-cache.cc51
-rw-r--r--src/libstore/nar-info-disk-cache.hh5
-rw-r--r--src/libstore/s3-binary-cache-store.cc35
-rw-r--r--src/libstore/store-api.cc34
-rw-r--r--src/libstore/store-api.hh11
7 files changed, 103 insertions, 45 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index dacdeb3f5..4445bfd27 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -94,13 +94,15 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
upsertFile(narInfoFile, narInfo->to_string());
+ auto hashPart = storePathToHash(narInfo->path);
+
{
auto state_(state.lock());
- state_->pathInfoCache.upsert(narInfo->path, std::shared_ptr<NarInfo>(narInfo));
+ state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo));
}
if (diskCache)
- diskCache->upsertNarInfo(getUri(), std::shared_ptr<NarInfo>(narInfo));
+ diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
stats.narInfoWrite++;
}
@@ -197,8 +199,6 @@ std::shared_ptr<ValidPathInfo> BinaryCacheStore::queryPathInfoUncached(const Pat
if (!data) return 0;
auto narInfo = make_ref<NarInfo>(*data, narInfoFile);
- if (narInfo->path != storePath)
- throw Error(format("NAR info file for store path ‘%1%’ does not match ‘%2%’") % narInfo->path % storePath);
stats.narInfoRead++;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 9bc164e19..baf3a528b 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -579,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state,
{
auto state_(Store::state.lock());
- state_->pathInfoCache.upsert(info.path, std::make_shared<ValidPathInfo>(info));
+ state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared<ValidPathInfo>(info));
}
return id;
@@ -1067,7 +1067,7 @@ void LocalStore::invalidatePath(State & state, const Path & path)
{
auto state_(Store::state.lock());
- state_->pathInfoCache.erase(path);
+ state_->pathInfoCache.erase(storePathToHash(path));
}
}
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 30ef7b36c..d8b0815bf 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -20,7 +20,8 @@ create table if not exists BinaryCaches (
create table if not exists NARs (
cache integer not null,
- storePath text not null,
+ hashPart text not null,
+ namePart text not null,
url text,
compression text,
fileHash text,
@@ -31,7 +32,7 @@ create table if not exists NARs (
deriver text,
sigs text,
timestamp integer not null,
- primary key (cache, storePath),
+ primary key (cache, hashPart),
foreign key (cache) references BinaryCaches(id) on delete cascade
);
@@ -66,7 +67,7 @@ public:
{
auto state(_state.lock());
- Path dbPath = getCacheDir() + "/nix/binary-cache-v3.sqlite";
+ Path dbPath = getCacheDir() + "/nix/binary-cache-v4.sqlite";
createDirs(dirOf(dbPath));
if (sqlite3_open_v2(dbPath.c_str(), &state->db.db,
@@ -92,11 +93,11 @@ public:
"select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?");
state->insertNAR.create(state->db,
- "insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, "
- "narSize, refs, deriver, sigs, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
+ "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, "
+ "narSize, refs, deriver, sigs, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
state->queryNAR.create(state->db,
- "select * from NARs where cache = ? and storePath = ?");
+ "select * from NARs where cache = ? and hashPart = ?");
state->insertNARExistence.create(state->db,
"insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)");
@@ -141,13 +142,13 @@ public:
}
std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
- const std::string & uri, const Path & storePath) override
+ const std::string & uri, const std::string & hashPart) override
{
auto state(_state.lock());
auto queryNAR(state->queryNAR.use()
(uriToInt(*state, uri))
- (baseNameOf(storePath)));
+ (hashPart));
if (!queryNAR.next())
// FIXME: check NARExistence
@@ -157,26 +158,29 @@ public:
// FIXME: implement TTL.
- narInfo->path = storePath;
- narInfo->url = queryNAR.getStr(2);
- narInfo->compression = queryNAR.getStr(3);
- if (!queryNAR.isNull(4))
- narInfo->fileHash = parseHash(queryNAR.getStr(4));
- narInfo->fileSize = queryNAR.getInt(5);
- narInfo->narHash = parseHash(queryNAR.getStr(6));
- narInfo->narSize = queryNAR.getInt(7);
- for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
+ auto namePart = queryNAR.getStr(2);
+ narInfo->path = settings.nixStore + "/" +
+ hashPart + (namePart.empty() ? "" : "-" + namePart);
+ narInfo->url = queryNAR.getStr(3);
+ narInfo->compression = queryNAR.getStr(4);
+ if (!queryNAR.isNull(5))
+ narInfo->fileHash = parseHash(queryNAR.getStr(5));
+ narInfo->fileSize = queryNAR.getInt(6);
+ narInfo->narHash = parseHash(queryNAR.getStr(7));
+ narInfo->narSize = queryNAR.getInt(8);
+ for (auto & r : tokenizeString<Strings>(queryNAR.getStr(9), " "))
narInfo->references.insert(settings.nixStore + "/" + r);
- if (!queryNAR.isNull(9))
- narInfo->deriver = settings.nixStore + "/" + queryNAR.getStr(9);
- for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
+ if (!queryNAR.isNull(10))
+ narInfo->deriver = settings.nixStore + "/" + queryNAR.getStr(10);
+ for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(11), " "))
narInfo->sigs.insert(sig);
return {oValid, narInfo};
}
void upsertNarInfo(
- const std::string & uri, std::shared_ptr<ValidPathInfo> info) override
+ const std::string & uri, const std::string & hashPart,
+ std::shared_ptr<ValidPathInfo> info) override
{
auto state(_state.lock());
@@ -184,9 +188,12 @@ public:
auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
+ assert(hashPart == storePathToHash(info->path));
+
state->insertNAR.use()
(uriToInt(*state, uri))
- (baseNameOf(info->path))
+ (hashPart)
+ (storePathToName(info->path))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh
index ee1aafc63..f4e3fbbdc 100644
--- a/src/libstore/nar-info-disk-cache.hh
+++ b/src/libstore/nar-info-disk-cache.hh
@@ -15,10 +15,11 @@ public:
virtual bool cacheExists(const std::string & uri) = 0;
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
- const std::string & uri, const Path & storePath) = 0;
+ const std::string & uri, const std::string & hashPart) = 0;
virtual void upsertNarInfo(
- const std::string & uri, std::shared_ptr<ValidPathInfo> narInfo) = 0;
+ const std::string & uri, const std::string & hashPart,
+ std::shared_ptr<ValidPathInfo> info) = 0;
};
/* Return a singleton cache object that can be used concurrently by
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 9ac79cf41..6fadf7be2 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -10,6 +10,7 @@
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <aws/s3/model/PutObjectRequest.h>
+#include <aws/s3/model/ListObjectsRequest.h>
namespace nix {
@@ -164,7 +165,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
std::shared_ptr<std::string> getFile(const std::string & path)
{
- printMsg(lvlDebug, format("fetching ‘s3://%1%/%2%’...") % bucketName % path);
+ debug(format("fetching ‘s3://%1%/%2%’...") % bucketName % path);
auto request =
Aws::S3::Model::GetObjectRequest()
@@ -204,6 +205,38 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
}
}
+ PathSet queryAllValidPaths() override
+ {
+ PathSet paths;
+ std::string marker;
+
+ do {
+ debug(format("listing bucket ‘s3://%s’ from key ‘%s’...") % bucketName % marker);
+
+ auto res = checkAws(format("AWS error listing bucket ‘%s’") % bucketName,
+ client->ListObjects(
+ Aws::S3::Model::ListObjectsRequest()
+ .WithBucket(bucketName)
+ .WithDelimiter("/")
+ .WithMarker(marker)));
+
+ auto & contents = res.GetContents();
+
+ debug(format("got %d keys, next marker ‘%s’")
+ % contents.size() % res.GetNextMarker());
+
+ for (auto object : contents) {
+ auto & key = object.GetKey();
+ if (!hasSuffix(key, ".narinfo")) continue;
+ paths.insert(settings.nixStore + "/" + std::string(key, 0, key.size() - 8));
+ }
+
+ marker = res.GetNextMarker();
+ } while (!marker.empty());
+
+ return paths;
+ }
+
};
static RegisterStoreImplementation regStore([](const std::string & uri) -> std::shared_ptr<Store> {
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index cac137a99..9870e36b5 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -63,13 +63,16 @@ Path followLinksToStorePath(const Path & path)
string storePathToName(const Path & path)
{
assertStorePath(path);
- return string(path, settings.nixStore.size() + storePathHashLen + 2);
+ auto l = settings.nixStore.size() + 1 + storePathHashLen;
+ assert(path.size() >= l);
+ return path.size() == l ? "" : string(path, l + 1);
}
string storePathToHash(const Path & path)
{
assertStorePath(path);
+ assert(path.size() >= settings.nixStore.size() + 1 + storePathHashLen);
return string(path, settings.nixStore.size() + 1, storePathHashLen);
}
@@ -234,9 +237,11 @@ std::string Store::getUri()
bool Store::isValidPath(const Path & storePath)
{
+ auto hashPart = storePathToHash(storePath);
+
{
auto state_(state.lock());
- auto res = state_->pathInfoCache.get(storePath);
+ auto res = state_->pathInfoCache.get(hashPart);
if (res) {
stats.narInfoReadAverted++;
return *res != 0;
@@ -244,10 +249,11 @@ bool Store::isValidPath(const Path & storePath)
}
if (diskCache) {
- auto res = diskCache->lookupNarInfo(getUri(), storePath);
+ auto res = diskCache->lookupNarInfo(getUri(), hashPart);
if (res.first != NarInfoDiskCache::oUnknown) {
+ stats.narInfoReadAverted++;
auto state_(state.lock());
- state_->pathInfoCache.upsert(storePath,
+ state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
return res.first == NarInfoDiskCache::oValid;
}
@@ -261,9 +267,11 @@ bool Store::isValidPath(const Path & storePath)
ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
{
+ auto hashPart = storePathToHash(storePath);
+
{
auto state_(state.lock());
- auto res = state_->pathInfoCache.get(storePath);
+ auto res = state_->pathInfoCache.get(hashPart);
if (res) {
stats.narInfoReadAverted++;
if (!*res)
@@ -273,12 +281,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
}
if (diskCache) {
- auto res = diskCache->lookupNarInfo(getUri(), storePath);
+ auto res = diskCache->lookupNarInfo(getUri(), hashPart);
if (res.first != NarInfoDiskCache::oUnknown) {
+ stats.narInfoReadAverted++;
auto state_(state.lock());
- state_->pathInfoCache.upsert(storePath,
+ state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
- if (res.first == NarInfoDiskCache::oInvalid)
+ if (res.first == NarInfoDiskCache::oInvalid ||
+ (res.second->path != storePath && storePathToName(storePath) != ""))
throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
return ref<ValidPathInfo>(res.second);
}
@@ -287,14 +297,16 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
auto info = queryPathInfoUncached(storePath);
if (diskCache && info)
- diskCache->upsertNarInfo(getUri(), info);
+ diskCache->upsertNarInfo(getUri(), hashPart, info);
{
auto state_(state.lock());
- state_->pathInfoCache.upsert(storePath, info);
+ state_->pathInfoCache.upsert(hashPart, info);
}
- if (!info) {
+ if (!info
+ || (info->path != storePath && storePathToName(storePath) != ""))
+ {
stats.narInfoMissing++;
throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
}
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index eab6da91e..e0cc32296 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -181,7 +181,7 @@ protected:
struct State
{
- LRUCache<Path, std::shared_ptr<ValidPathInfo>> pathInfoCache{64 * 1024};
+ LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache{64 * 1024};
};
Sync<State> state;
@@ -206,10 +206,15 @@ public:
/* Query which of the given paths is valid. */
virtual PathSet queryValidPaths(const PathSet & paths) = 0;
- /* Query the set of all valid paths. */
+ /* Query the set of all valid paths. Note that for some store
+ backends, the name part of store paths may be omitted
+ (i.e. you'll get /nix/store/<hash> rather than
+ /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the
+ full store path. */
virtual PathSet queryAllValidPaths() = 0;
- /* Query information about a valid path. */
+ /* Query information about a valid path. It is permitted to omit
+ the name part of the store path. */
ref<const ValidPathInfo> queryPathInfo(const Path & path);
protected:
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 03:40:14 +0000