aboutsummaryrefslogtreecommitdiff
path: root/src/libstore/store-api.cc
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore/store-api.cc')
-rw-r--r--src/libstore/store-api.cc236
1 files changed, 200 insertions, 36 deletions
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 95b1c1c3b..62feb388a 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -7,6 +7,7 @@
#include "json.hh"
#include "derivations.hh"
#include "url.hh"
+#include "archive.hh"
#include <future>
@@ -20,15 +21,15 @@ bool Store::isInStore(const Path & path) const
}
-Path Store::toStorePath(const Path & path) const
+std::pair<StorePath, Path> Store::toStorePath(const Path & path) const
{
if (!isInStore(path))
throw Error("path '%1%' is not in the Nix store", path);
Path::size_type slash = path.find('/', storeDir.size() + 1);
if (slash == Path::npos)
- return path;
+ return {parseStorePath(path), ""};
else
- return Path(path, 0, slash);
+ return {parseStorePath(std::string_view(path).substr(0, slash)), path.substr(slash)};
}
@@ -41,14 +42,14 @@ Path Store::followLinksToStore(std::string_view _path) const
path = absPath(target, dirOf(path));
}
if (!isInStore(path))
- throw NotInStore("path '%1%' is not in the Nix store", path);
+ throw BadStorePath("path '%1%' is not in the Nix store", path);
return path;
}
StorePath Store::followLinksToStorePath(std::string_view path) const
{
- return parseStorePath(toStorePath(followLinksToStore(path)));
+ return toStorePath(followLinksToStore(path)).first;
}
@@ -192,6 +193,23 @@ StorePath Store::makeFixedOutputPath(
}
}
+// FIXME Put this somewhere?
+template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
+template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
+
+StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca,
+ const StorePathSet & references, bool hasSelfReference) const
+{
+ // New template
+ return std::visit(overloaded {
+ [&](TextHash th) {
+ return makeTextPath(name, th.hash, references);
+ },
+ [&](FixedOutputHash fsh) {
+ return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
+ }
+ }, ca);
+}
StorePath Store::makeTextPath(std::string_view name, const Hash & hash,
const StorePathSet & references) const
@@ -221,6 +239,93 @@ StorePath Store::computeStorePathForText(const string & name, const string & s,
}
+/*
+The aim of this function is to compute in one pass the correct ValidPathInfo for
+the files that we are trying to add to the store. To accomplish that in one
+pass, given the different kind of inputs that we can take (normal nar archives,
+nar archives with non SHA-256 hashes, and flat files), we set up a net of sinks
+and aliases. Also, since the dataflow is obfuscated by this, we include here a
+graphviz diagram:
+
+digraph graphname {
+ node [shape=box]
+ fileSource -> narSink
+ narSink [style=dashed]
+ narSink -> unsualHashTee [style = dashed, label = "Recursive && !SHA-256"]
+ narSink -> narHashSink [style = dashed, label = "else"]
+ unsualHashTee -> narHashSink
+ unsualHashTee -> caHashSink
+ fileSource -> parseSink
+ parseSink [style=dashed]
+ parseSink-> fileSink [style = dashed, label = "Flat"]
+ parseSink -> blank [style = dashed, label = "Recursive"]
+ fileSink -> caHashSink
+}
+*/
+ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
+ FileIngestionMethod method, HashType hashAlgo,
+ std::optional<Hash> expectedCAHash)
+{
+ HashSink narHashSink { htSHA256 };
+ HashSink caHashSink { hashAlgo };
+
+ /* Note that fileSink and unusualHashTee must be mutually exclusive, since
+ they both write to caHashSink. Note that that requisite is currently true
+ because the former is only used in the flat case. */
+ RetrieveRegularNARSink fileSink { caHashSink };
+ TeeSink unusualHashTee { narHashSink, caHashSink };
+
+ auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256
+ ? static_cast<Sink &>(unusualHashTee)
+ : narHashSink;
+
+ /* Functionally, this means that fileSource will yield the content of
+ srcPath. The fact that we use scratchpadSink as a temporary buffer here
+ is an implementation detail. */
+ auto fileSource = sinkToSource([&](Sink & scratchpadSink) {
+ dumpPath(srcPath, scratchpadSink);
+ });
+
+ /* tapped provides the same data as fileSource, but we also write all the
+ information to narSink. */
+ TeeSource tapped { *fileSource, narSink };
+
+ ParseSink blank;
+ auto & parseSink = method == FileIngestionMethod::Flat
+ ? fileSink
+ : blank;
+
+ /* The information that flows from tapped (besides being replicated in
+ narSink), is now put in parseSink. */
+ parseDump(parseSink, tapped);
+
+ /* We extract the result of the computation from the sink by calling
+ finish. */
+ auto [narHash, narSize] = narHashSink.finish();
+
+ auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256
+ ? narHash
+ : caHashSink.finish().first;
+
+ if (expectedCAHash && expectedCAHash != hash)
+ throw Error("hash mismatch for '%s'", srcPath);
+
+ ValidPathInfo info(makeFixedOutputPath(method, hash, name));
+ info.narHash = narHash;
+ info.narSize = narSize;
+ info.ca = FixedOutputHash { .method = method, .hash = hash };
+
+ if (!isValidPath(info.path)) {
+ auto source = sinkToSource([&](Sink & scratchpadSink) {
+ dumpPath(srcPath, scratchpadSink);
+ });
+ addToStore(info, *source);
+ }
+
+ return info;
+}
+
+
Store::Store(const Params & params)
: Config(params)
, state({(size_t) pathInfoCacheSize})
@@ -242,6 +347,16 @@ bool Store::PathInfoCacheValue::isKnownNow()
return std::chrono::steady_clock::now() < time_point + ttl;
}
+StorePathSet Store::queryDerivationOutputs(const StorePath & path)
+{
+ auto outputMap = this->queryDerivationOutputMap(path);
+ StorePathSet outputPaths;
+ for (auto & i: outputMap) {
+ outputPaths.emplace(std::move(i.second));
+ }
+ return outputPaths;
+}
+
bool Store::isValidPath(const StorePath & storePath)
{
std::string hashPart(storePath.hashPart());
@@ -306,6 +421,14 @@ ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath)
}
+static bool goodStorePath(const StorePath & expected, const StorePath & actual)
+{
+ return
+ expected.hashPart() == actual.hashPart()
+ && (expected.name() == Store::MissingName || expected.name() == actual.name());
+}
+
+
void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept
{
@@ -333,7 +456,7 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart,
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid ||
- res.second->path != storePath)
+ !goodStorePath(storePath, res.second->path))
throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
return callback(ref<const ValidPathInfo>(res.second));
@@ -345,7 +468,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath,
- {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
+ {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try {
auto info = fut.get();
@@ -358,9 +481,11 @@ void Store::queryPathInfo(const StorePath & storePath,
state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
}
- if (!info || info->path != parseStorePath(storePath)) {
+ auto storePath = parseStorePath(storePathS);
+
+ if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++;
- throw InvalidPath("path '%s' is not valid", storePath);
+ throw InvalidPath("path '%s' is not valid", storePathS);
}
(*callbackPtr)(ref<const ValidPathInfo>(info));
@@ -430,7 +555,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
auto info = queryPathInfo(i);
if (showHash) {
- s += info->narHash.to_string(Base16, false) + "\n";
+ s += info->narHash->to_string(Base16, false) + "\n";
s += (format("%1%\n") % info->narSize).str();
}
@@ -462,7 +587,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
auto info = queryPathInfo(storePath);
jsonPath
- .attr("narHash", info->narHash.to_string(hashBase, true))
+ .attr("narHash", info->narHash->to_string(hashBase, true))
.attr("narSize", info->narSize);
{
@@ -505,7 +630,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store
if (!narInfo->url.empty())
jsonPath.attr("url", narInfo->url);
if (narInfo->fileHash)
- jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true));
+ jsonPath.attr("downloadHash", narInfo->fileHash->to_string(hashBase, true));
if (narInfo->fileSize)
jsonPath.attr("downloadSize", narInfo->fileSize);
if (showClosureSize)
@@ -581,6 +706,15 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
uint64_t total = 0;
+ // recompute store path on the chance dstStore does it differently
+ if (info->ca && info->references.empty()) {
+ auto info2 = make_ref<ValidPathInfo>(*info);
+ info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(info->path == info2->path);
+ info = info2;
+ }
+
if (!info->narHash) {
StringSink sink;
srcStore->narFromPath({storePath}, sink);
@@ -616,16 +750,20 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
}
-void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
auto valid = dstStore->queryValidPaths(storePaths, substitute);
- PathSet missing;
+ StorePathSet missing;
for (auto & path : storePaths)
- if (!valid.count(path)) missing.insert(srcStore->printStorePath(path));
+ if (!valid.count(path)) missing.insert(path);
- if (missing.empty()) return;
+ std::map<StorePath, StorePath> pathsMap;
+ for (auto & path : storePaths)
+ pathsMap.insert_or_assign(path, path);
+
+ if (missing.empty()) return pathsMap;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
@@ -640,30 +778,49 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
ThreadPool pool;
- processGraph<Path>(pool,
- PathSet(missing.begin(), missing.end()),
+ processGraph<StorePath>(pool,
+ StorePathSet(missing.begin(), missing.end()),
+
+ [&](const StorePath & storePath) {
+ auto info = srcStore->queryPathInfo(storePath);
+ auto storePathForDst = storePath;
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(storePathForDst == storePath);
+ if (storePathForDst != storePath)
+ debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ }
+ pathsMap.insert_or_assign(storePath, storePathForDst);
- [&](const Path & storePath) {
- if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) {
+ if (dstStore->isValidPath(storePath)) {
nrDone++;
showProgress();
- return PathSet();
+ return StorePathSet();
}
- auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath));
-
bytesExpected += info->narSize;
act.setExpected(actCopyPath, bytesExpected);
- return srcStore->printStorePathSet(info->references);
+ return info->references;
},
- [&](const Path & storePathS) {
+ [&](const StorePath & storePath) {
checkInterrupt();
- auto storePath = dstStore->parseStorePath(storePathS);
+ auto info = srcStore->queryPathInfo(storePath);
- if (!dstStore->isValidPath(storePath)) {
+ auto storePathForDst = storePath;
+ if (info->ca && info->references.empty()) {
+ storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore->storeDir == srcStore->storeDir)
+ assert(storePathForDst == storePath);
+ if (storePathForDst != storePath)
+ debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ }
+ pathsMap.insert_or_assign(storePath, storePathForDst);
+
+ if (!dstStore->isValidPath(storePathForDst)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress();
try {
@@ -672,7 +829,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
nrFailed++;
if (!settings.keepGoing)
throw e;
- logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what()));
+ logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what()));
showProgress();
return;
}
@@ -681,6 +838,8 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st
nrDone++;
showProgress();
});
+
+ return pathsMap;
}
@@ -760,7 +919,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
store.printStorePath(path));
return
"1;" + store.printStorePath(path) + ";"
- + narHash.to_string(Base32, true) + ";"
+ + narHash->to_string(Base32, true) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf()));
}
@@ -771,10 +930,6 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey)
sigs.insert(secretKey.signDetached(fingerprint(store)));
}
-// FIXME Put this somewhere?
-template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
-template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
-
bool ValidPathInfo::isContentAddressed(const Store & store) const
{
if (! ca) return false;
@@ -868,12 +1023,20 @@ ref<Store> openStore(const std::string & uri_,
throw Error("don't know how to open Nix store '%s'", uri);
}
+static bool isNonUriPath(const std::string & spec) {
+ return
+ // is not a URL
+ spec.find("://") == std::string::npos
+ // Has at least one path separator, and so isn't a single word that
+ // might be special like "auto"
+ && spec.find("/") != std::string::npos;
+}
StoreType getStoreType(const std::string & uri, const std::string & stateDir)
{
if (uri == "daemon") {
return tDaemon;
- } else if (uri == "local" || hasPrefix(uri, "/")) {
+ } else if (uri == "local" || isNonUriPath(uri)) {
return tLocal;
} else if (uri == "" || uri == "auto") {
if (access(stateDir.c_str(), R_OK | W_OK) == 0)
@@ -897,8 +1060,9 @@ static RegisterStoreImplementation regStore([](
return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params));
case tLocal: {
Store::Params params2 = params;
- if (hasPrefix(uri, "/"))
- params2["root"] = uri;
+ if (isNonUriPath(uri)) {
+ params2["root"] = absPath(uri);
+ }
return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2));
}
default:
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 09:16:40 +0000