diff options
| -rw-r--r-- | src/libstore/binary-cache-store.cc | 29 | ||||
| -rw-r--r-- | src/libstore/nar-info-disk-cache.cc | 100 | ||||
| -rw-r--r-- | src/libstore/nar-info-disk-cache.hh | 10 | ||||
| -rw-r--r-- | tests/ca/substitute.sh | 13 |
4 files changed, 149 insertions, 3 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 09e1c254b..312630520 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -450,18 +450,43 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id) { + if (diskCache) { + auto [cacheOutcome, maybeCachedRealisation] = + diskCache->lookupRealisation(getUri(), id); + switch (cacheOutcome) { + case (NarInfoDiskCache::oValid): + debug("Returning a cached realisation for %s", id.to_string()); + return *maybeCachedRealisation; + case (NarInfoDiskCache::oInvalid): + debug("Returning a cached missing realisation for %s", id.to_string()); + return {}; + case (NarInfoDiskCache::oUnknown): + break; + } + } + auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; auto rawOutputInfo = getFile(outputInfoFilePath); if (rawOutputInfo) { - return {Realisation::fromJSON( - nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)}; + auto realisation = Realisation::fromJSON( + nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath); + + if (diskCache) + diskCache->upsertRealisation( + getUri(), realisation); + + return {realisation}; } else { + if (diskCache) + diskCache->upsertAbsentRealisation(getUri(), id); return std::nullopt; } } void BinaryCacheStore::registerDrvOutput(const Realisation& info) { + if (diskCache) + diskCache->upsertRealisation(getUri(), info); auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; upsertFile(filePath, info.toJSON().dump(), "application/json"); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 1d8d2d57e..84ce7972f 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -4,6 +4,7 @@ #include "globals.hh" #include <sqlite3.h> +#include <nlohmann/json.hpp> namespace nix { @@ -38,6 +39,16 @@ create table if not exists NARs ( foreign key (cache) references BinaryCaches(id) on delete cascade ); +create table if not exists Realisations ( + cache integer not null, + outputId text not null, + content blob, -- Json serialisation of the realisation, or empty if present is true + timestamp integer not null, + present integer not null, + primary key (cache, outputId), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + create table if not exists LastPurge ( dummy text primary key, value integer @@ -63,7 +74,9 @@ public: struct State { SQLite db; - SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; + SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, + queryNAR, insertRealisation, insertMissingRealisation, + queryRealisation, purgeCache; std::map<std::string, Cache> caches; }; @@ -98,6 +111,26 @@ public: state->queryNAR.create(state->db, "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + state->insertRealisation.create(state->db, + R"( + insert or replace into Realisations(cache, outputId, content, timestamp, present) + values (?, ?, ?, ?, 1) + )"); + + state->insertMissingRealisation.create(state->db, + R"( + insert or replace into Realisations(cache, outputId, timestamp, present) + values (?, ?, ?, 0) + )"); + + state->queryRealisation.create(state->db, + R"( + select present, content from Realisations + where cache = ? and outputId = ? and + ((present = 0 and timestamp > ?) or + (present = 1 and timestamp > ?)) + )"); + /* Periodically purge expired entries from the database. */ retrySQLite<void>([&]() { auto now = time(0); @@ -212,6 +245,38 @@ public: }); } + std::pair<Outcome, std::shared_ptr<Realisation>> lookupRealisation( + const std::string & uri, const DrvOutput & id) override + { + return retrySQLite<std::pair<Outcome, std::shared_ptr<Realisation>>>( + [&]() -> std::pair<Outcome, std::shared_ptr<Realisation>> { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + auto now = time(0); + + auto queryRealisation(state->queryRealisation.use() + (cache.id) + (id.to_string()) + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache)); + + if (!queryRealisation.next()) + return {oUnknown, 0}; + + if (queryRealisation.getInt(0) == 0) + return {oInvalid, 0}; + + auto realisation = + std::make_shared<Realisation>(Realisation::fromJSON( + nlohmann::json::parse(queryRealisation.getStr(1)), + "Local disk cache")); + + return {oValid, realisation}; + }); + } + void upsertNarInfo( const std::string & uri, const std::string & hashPart, std::shared_ptr<const ValidPathInfo> info) override @@ -251,6 +316,39 @@ public: } }); } + + void upsertRealisation( + const std::string & uri, + const Realisation & realisation) override + { + retrySQLite<void>([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + state->insertRealisation.use() + (cache.id) + (realisation.id.to_string()) + (realisation.toJSON().dump()) + (time(0)).exec(); + }); + + } + + virtual void upsertAbsentRealisation( + const std::string & uri, + const DrvOutput & id) override + { + retrySQLite<void>([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + state->insertMissingRealisation.use() + (cache.id) + (id.to_string()) + (time(0)).exec(); + }); + } }; ref<NarInfoDiskCache> getNarInfoDiskCache() diff --git a/src/libstore/nar-info-disk-cache.hh b/src/libstore/nar-info-disk-cache.hh index 04de2c5eb..2dcaa76a4 100644 --- a/src/libstore/nar-info-disk-cache.hh +++ b/src/libstore/nar-info-disk-cache.hh @@ -2,6 +2,7 @@ #include "ref.hh" #include "nar-info.hh" +#include "realisation.hh" namespace nix { @@ -29,6 +30,15 @@ public: virtual void upsertNarInfo( const std::string & uri, const std::string & hashPart, std::shared_ptr<const ValidPathInfo> info) = 0; + + virtual void upsertRealisation( + const std::string & uri, + const Realisation & realisation) = 0; + virtual void upsertAbsentRealisation( + const std::string & uri, + const DrvOutput & id) = 0; + virtual std::pair<Outcome, std::shared_ptr<Realisation>> lookupRealisation( + const std::string & uri, const DrvOutput & id) = 0; }; /* Return a singleton cache object that can be used concurrently by diff --git a/tests/ca/substitute.sh b/tests/ca/substitute.sh index 737c851a5..dfc4ea68e 100644 --- a/tests/ca/substitute.sh +++ b/tests/ca/substitute.sh @@ -45,3 +45,16 @@ if [[ -z "$(ls "$REMOTE_STORE_DIR/realisations")" ]]; then echo "Realisations not rebuilt" exit 1 fi + +# Test the local realisation disk cache +buildDrvs --post-build-hook ../push-to-store.sh +clearStore +# Add the realisations of rootCA to the cachecache +clearCacheCache +export _NIX_FORCE_HTTP=1 +buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +# Try rebuilding, but remove the realisations from the remote cache to force +# using the cachecache +clearStore +rm $REMOTE_STORE_DIR/realisations/* +buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 |