From d38f860c3ef001a456d4d447f89219de5e3c830c Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 10 Jun 2020 11:20:52 +0200 Subject: Add a way to get all the outputs of a derivation with their label Generalize `queryDerivationOutputNames` and `queryDerivationOutputs` by adding a `queryDerivationOutputMap` that returns the map `outputName=>outputPath` (not that this is not equivalent to merging the results of `queryDerivationOutputs` and `queryDerivationOutputNames` as sets don't preserve the order, so we would end up with an incorrect mapping). squash! Add a way to get all the outputs of a derivation with their label Rename StorePathMap to OutputPathMap --- src/libstore/store-api.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src/libstore/store-api.cc') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e4a4ae11e..c0a8bc9f6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -242,6 +242,16 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } +StorePathSet Store::queryDerivationOutputs(const StorePath & path) +{ + auto outputMap = this->queryDerivationOutputMap(path); + StorePathSet outputPaths; + for (auto & i: outputMap) { + outputPaths.emplace(std::move(i.second)); + } + return outputPaths; +} + bool Store::isValidPath(const StorePath & storePath) { std::string hashPart(storePath.hashPart()); -- cgit v1.2.3 From 5dff49f661cd221fc457d1a4660cd36f28266dc5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jul 2020 13:21:37 +0200 Subject: Factor out commonality between nix-prefetch-url and nix-store --add-fixed --- src/libstore/store-api.cc | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'src/libstore/store-api.cc') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c0a8bc9f6..46587a49a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "json.hh" #include "derivations.hh" #include "url.hh" +#include "archive.hh" #include @@ -221,6 +222,37 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } +ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, + FileIngestionMethod method, HashType hashAlgo, + std::optional expectedCAHash) +{ + /* FIXME: inefficient: we're reading/hashing 'tmpFile' three + times. */ + + auto hash = method == FileIngestionMethod::Recursive + ? hashPath(hashAlgo, srcPath).first + : hashFile(hashAlgo, srcPath); + + if (expectedCAHash && expectedCAHash != hash) + throw Error("hash mismatch for '%s'", srcPath); + + auto [narHash, narSize] = hashPath(htSHA256, srcPath); + ValidPathInfo info(makeFixedOutputPath(method, hash, name)); + info.narHash = narHash; + info.narSize = narSize; + info.ca = FixedOutputHash { .method = method, .hash = hash }; + + if (!isValidPath(info.path)) { + auto source = sinkToSource([&](Sink & sink) { + dumpPath(srcPath, sink); + }); + addToStore(info, *source); + } + + return info; +} + + Store::Store(const Params & params) : Config(params) , state({(size_t) pathInfoCacheSize}) -- cgit v1.2.3 From 8efa23bb996161af74f89401902450e51e9d4b54 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jul 2020 15:56:24 +0200 Subject: Avoid a redundant hash --- src/libstore/store-api.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src/libstore/store-api.cc') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 46587a49a..8d46bb436 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -229,14 +229,17 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, /* FIXME: inefficient: we're reading/hashing 'tmpFile' three times. */ + auto [narHash, narSize] = hashPath(htSHA256, srcPath); + auto hash = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath).first + ? hashAlgo == htSHA256 + ? narHash + : hashPath(hashAlgo, srcPath).first : hashFile(hashAlgo, srcPath); if (expectedCAHash && expectedCAHash != hash) throw Error("hash mismatch for '%s'", srcPath); - auto [narHash, narSize] = hashPath(htSHA256, srcPath); ValidPathInfo info(makeFixedOutputPath(method, hash, name)); info.narHash = narHash; info.narSize = narSize; -- cgit v1.2.3 From 1d01ae816b80eaefb0996a9605d00a3031ecd4d9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jul 2020 14:35:01 +0200 Subject: Fix 'nix verify --all' on a binary cache and add a test --- src/libstore/store-api.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'src/libstore/store-api.cc') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8d46bb436..0c6788b69 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -390,7 +390,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { + {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future> fut) { try { auto info = fut.get(); @@ -403,9 +403,15 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - if (!info || info->path != parseStorePath(storePath)) { + auto storePath = parseStorePath(storePathS); + + if (!info + || info->path.hashPart() != storePath.hashPart() + || (storePath.name() != MissingName && info->path.name() != storePath.name()) + ) + { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePath); + throw InvalidPath("path '%s' is not valid", storePathS); } (*callbackPtr)(ref(info)); -- cgit v1.2.3 From c0dd05131e08102d560a737ff68c66bc8314f5fa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jul 2020 16:19:37 +0200 Subject: toStorePath(): Return a StorePath and the suffix --- src/libstore/store-api.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/libstore/store-api.cc') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0c6788b69..e67f4e359 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -21,15 +21,15 @@ bool Store::isInStore(const Path & path) const } -Path Store::toStorePath(const Path & path) const +std::pair Store::toStorePath(const Path & path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); Path::size_type slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) - return path; + return {parseStorePath(path), ""}; else - return Path(path, 0, slash); + return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)}; } @@ -42,14 +42,14 @@ Path Store::followLinksToStore(std::string_view _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw NotInStore("path '%1%' is not in the Nix store", path); + throw BadStorePath("path '%1%' is not in the Nix store", path); return path; } StorePath Store::followLinksToStorePath(std::string_view path) const { - return parseStorePath(toStorePath(followLinksToStore(path))); + return toStorePath(followLinksToStore(path)).first; } -- cgit v1.2.3 From 400f1a9b59d95861b4dd171c45e2dc6cf263ab99 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Jul 2020 18:55:24 +0200 Subject: Store::pathInfoToJSON(): Use consistent format for downloadHash --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/store-api.cc') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e67f4e359..bc3fafa35 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -556,7 +556,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); + jsonPath.attr("downloadHash", narInfo->fileHash.to_string(hashBase, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) -- cgit v1.2.3 From 43b8e96d30fa2b4e5f2f5144dc905c2d944a4e15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jul 2020 20:17:00 +0200 Subject: Fix 'nix verify --all' on a binary cache (cached case) --- src/libstore/store-api.cc | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'src/libstore/store-api.cc') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index bc3fafa35..5554befa4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -351,6 +351,14 @@ ref Store::queryPathInfo(const StorePath & storePath) } +static bool goodStorePath(const StorePath & expected, const StorePath & actual) +{ + return + expected.hashPart() == actual.hashPart() + && (expected.name() == Store::MissingName || expected.name() == actual.name()); +} + + void Store::queryPathInfo(const StorePath & storePath, Callback> callback) noexcept { @@ -378,7 +386,7 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || - res.second->path != storePath) + !goodStorePath(storePath, res.second->path)) throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } return callback(ref(res.second)); @@ -405,11 +413,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto storePath = parseStorePath(storePathS); - if (!info - || info->path.hashPart() != storePath.hashPart() - || (storePath.name() != MissingName && info->path.name() != storePath.name()) - ) - { + if (!info || !goodStorePath(storePath, info->path)); { stats.narInfoMissing++; throw InvalidPath("path '%s' is not valid", storePathS); } -- cgit v1.2.3 From 926c3a6664a9dfb288ca35af8aae40c4e4a2badb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Jul 2020 11:55:54 +0200 Subject: Doh --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/store-api.cc') diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5554befa4..5b9f79049 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -413,7 +413,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)); { + if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; throw InvalidPath("path '%s' is not valid", storePathS); } -- cgit v1.2.3