aboutsummaryrefslogtreecommitdiff
path: root/src/libstore/store-api.cc
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore/store-api.cc')
-rw-r--r--src/libstore/store-api.cc273
1 files changed, 207 insertions, 66 deletions
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index e6a4f33ba..49bc91614 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -9,6 +9,7 @@
#include "url.hh"
#include "archive.hh"
#include "callback.hh"
+#include "remote-store.hh"
#include <regex>
@@ -53,13 +54,6 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
}
-StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
-{
- auto [path2, outputs] = nix::parsePathWithOutputs(path);
- return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) };
-}
-
-
/* Store paths have the following form:
<realized-path> = <store>/<h>-<name>
@@ -205,10 +199,10 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress
{
// New template
return std::visit(overloaded {
- [&](TextHash th) {
+ [&](const TextHash & th) {
return makeTextPath(name, th.hash, references);
},
- [&](FixedOutputHash fsh) {
+ [&](const FixedOutputHash & fsh) {
return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference);
}
}, ca);
@@ -256,6 +250,20 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath,
}
+void Store::addMultipleToStore(
+ Source & source,
+ RepairFlag repair,
+ CheckSigsFlag checkSigs)
+{
+ auto expected = readNum<uint64_t>(source);
+ for (uint64_t i = 0; i < expected; ++i) {
+ auto info = ValidPathInfo::read(source, *this, 16);
+ info.ultimate = false;
+ addToStore(info, source, repair, checkSigs);
+ }
+}
+
+
/*
The aim of this function is to compute in one pass the correct ValidPathInfo for
the files that we are trying to add to the store. To accomplish that in one
@@ -344,6 +352,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
return info;
}
+StringSet StoreConfig::getDefaultSystemFeatures()
+{
+ auto res = settings.systemFeatures.get();
+ if (settings.isExperimentalFeatureEnabled("ca-derivations"))
+ res.insert("ca-derivations");
+ return res;
+}
Store::Store(const Params & params)
: StoreConfig(params)
@@ -536,10 +551,10 @@ void Store::queryPathInfo(const StorePath & storePath,
void Store::substitutePaths(const StorePathSet & paths)
{
- std::vector<StorePathWithOutputs> paths2;
+ std::vector<DerivedPath> paths2;
for (auto & path : paths)
if (!path.isDerivation())
- paths2.push_back({path});
+ paths2.push_back(DerivedPath::Opaque{path});
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
queryMissing(paths2,
@@ -547,8 +562,8 @@ void Store::substitutePaths(const StorePathSet & paths)
if (!willSubstitute.empty())
try {
- std::vector<StorePathWithOutputs> subs;
- for (auto & p : willSubstitute) subs.push_back({p});
+ std::vector<DerivedPath> subs;
+ for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p});
buildPaths(subs);
} catch (Error & e) {
logWarning(e.info());
@@ -634,6 +649,42 @@ string Store::makeValidityRegistration(const StorePathSet & paths,
}
+StorePathSet Store::exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths)
+{
+ StorePathSet paths;
+
+ for (auto & storePath : storePaths) {
+ if (!inputPaths.count(storePath))
+ throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", printStorePath(storePath));
+
+ computeFSClosure({storePath}, paths);
+ }
+
+ /* If there are derivations in the graph, then include their
+ outputs as well. This is useful if you want to do things
+ like passing all build-time dependencies of some path to a
+ derivation that builds a NixOS DVD image. */
+ auto paths2 = paths;
+
+ for (auto & j : paths2) {
+ if (j.isDerivation()) {
+ Derivation drv = derivationFromPath(j);
+ for (auto & k : drv.outputsAndOptPaths(*this)) {
+ if (!k.second.second)
+ /* FIXME: I am confused why we are calling
+ `computeFSClosure` on the output path, rather than
+ derivation itself. That doesn't seem right to me, so I
+ won't try to implemented this for CA derivations. */
+ throw UnimplementedError("exportReferences on CA derivations is not yet implemented");
+ computeFSClosure(*k.second.second, paths);
+ }
+ }
+ }
+
+ return paths;
+}
+
+
void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize,
Base hashBase,
@@ -734,30 +785,43 @@ const Store::Stats & Store::getStats()
}
-void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
- const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
+static std::string makeCopyPathMessage(
+ std::string_view srcUri,
+ std::string_view dstUri,
+ std::string_view storePath)
{
- auto srcUri = srcStore->getUri();
- auto dstUri = dstStore->getUri();
+ return srcUri == "local" || srcUri == "daemon"
+ ? fmt("copying path '%s' to '%s'", storePath, dstUri)
+ : dstUri == "local" || dstUri == "daemon"
+ ? fmt("copying path '%s' from '%s'", storePath, srcUri)
+ : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri);
+}
+
+void copyStorePath(
+ Store & srcStore,
+ Store & dstStore,
+ const StorePath & storePath,
+ RepairFlag repair,
+ CheckSigsFlag checkSigs)
+{
+ auto srcUri = srcStore.getUri();
+ auto dstUri = dstStore.getUri();
+ auto storePathS = srcStore.printStorePath(storePath);
Activity act(*logger, lvlInfo, actCopyPath,
- srcUri == "local" || srcUri == "daemon"
- ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri)
- : dstUri == "local" || dstUri == "daemon"
- ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri)
- : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri),
- {srcStore->printStorePath(storePath), srcUri, dstUri});
+ makeCopyPathMessage(srcUri, dstUri, storePathS),
+ {storePathS, srcUri, dstUri});
PushActivity pact(act.id);
- auto info = srcStore->queryPathInfo(storePath);
+ auto info = srcStore.queryPathInfo(storePath);
uint64_t total = 0;
// recompute store path on the chance dstStore does it differently
if (info->ca && info->references.empty()) {
auto info2 = make_ref<ValidPathInfo>(*info);
- info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca);
- if (dstStore->storeDir == srcStore->storeDir)
+ info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca);
+ if (dstStore.storeDir == srcStore.storeDir)
assert(info->path == info2->path);
info = info2;
}
@@ -774,64 +838,90 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
act.progress(total, info->narSize);
});
TeeSink tee { sink, progressSink };
- srcStore->narFromPath(storePath, tee);
+ srcStore.narFromPath(storePath, tee);
}, [&]() {
- throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri());
+ throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(storePath), srcStore.getUri());
});
- dstStore->addToStore(*info, *source, repair, checkSigs);
+ dstStore.addToStore(*info, *source, repair, checkSigs);
}
-void copyStorePathAdapter(ref<Store> srcStore, ref<Store> dstStore,
+void copyStorePathAdapter(Store & srcStore, Store & dstStore,
const ValidPathInfo & info, RepairFlag repair, CheckSigsFlag checkSigs)
{
copyStorePath(srcStore, dstStore, info.path, repair, checkSigs);
}
-void copyOrBuildStorePath(ref<Store> srcStore, ref<Store> dstStore,
+void copyOrBuildStorePath(Store & srcStore, Store & dstStore,
const ValidPathInfo & info, RepairFlag repair, CheckSigsFlag checkSigs)
{
auto storePath = info.path;
- if (dstStore->isTrusting || info.ca) {
+ if (dstStore.isTrusting || info.ca) {
copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
- } else if (info.deriver && dstStore->storeDir == srcStore->storeDir) {
+ } else if (info.deriver && dstStore.storeDir == srcStore.storeDir) {
auto drvPath = *info.deriver;
- auto outputMap = srcStore->queryDerivationOutputMap(drvPath);
+ auto outputMap = srcStore.queryDerivationOutputMap(drvPath);
auto p = std::find_if(outputMap.begin(), outputMap.end(), [&](auto & i) {
return i.second == storePath;
});
// drv file is always CA
- srcStore->ensurePath(drvPath);
+ srcStore.ensurePath(drvPath);
copyStorePath(srcStore, dstStore, drvPath, repair, checkSigs);
- dstStore->buildPaths({{
- drvPath,
- p != outputMap.end() ? StringSet { p->first } : StringSet {},
- }});
+ dstStore.buildPaths({
+ DerivedPath::Built {
+ .drvPath = drvPath,
+ .outputs = p != outputMap.end() ? StringSet { p->first } : StringSet {},
+ },
+ });
} else {
- dstStore->ensurePath(storePath);
+ dstStore.ensurePath(storePath);
}
}
-std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths,
- RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute,
- std::function<void(ref<Store>, ref<Store>, const ValidPathInfo &, RepairFlag, CheckSigsFlag)> copyStorePathImpl)
+std::map<StorePath, StorePath> copyPaths(
+ Store & srcStore,
+ Store & dstStore,
+ const RealisedPath::Set & paths,
+ RepairFlag repair,
+ CheckSigsFlag checkSigs,
+ SubstituteFlag substitute,
+ std::function<void(Store &, Store &, const ValidPathInfo &, RepairFlag, CheckSigsFlag)> copyStorePathImpl)
{
StorePathSet storePaths;
- std::set<Realisation> realisations;
+ std::set<Realisation> toplevelRealisations;
for (auto & path : paths) {
storePaths.insert(path.path());
if (auto realisation = std::get_if<Realisation>(&path.raw)) {
settings.requireExperimentalFeature("ca-derivations");
- realisations.insert(*realisation);
+ toplevelRealisations.insert(*realisation);
}
}
auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute, copyStorePathImpl);
+
+ ThreadPool pool;
+
try {
- for (auto & realisation : realisations) {
- dstStore->registerDrvOutput(realisation);
- }
+ // Copy the realisation closure
+ processGraph<Realisation>(
+ pool, Realisation::closure(srcStore, toplevelRealisations),
+ [&](const Realisation & current) -> std::set<Realisation> {
+ std::set<Realisation> children;
+ for (const auto & [drvOutput, _] : current.dependentRealisations) {
+ auto currentChild = srcStore.queryRealisation(drvOutput);
+ if (!currentChild)
+ throw Error(
+ "incomplete realisation closure: '%s' is a "
+ "dependency of '%s' but isn't registered",
+ drvOutput.to_string(), current.id.to_string());
+ children.insert(*currentChild);
+ }
+ return children;
+ },
+ [&](const Realisation& current) -> void {
+ dstStore.registerDrvOutput(current, checkSigs);
+ });
} catch (MissingExperimentalFeature & e) {
// Don't fail if the remote doesn't support CA derivations is it might
// not be within our control to change that, and we might still want
@@ -845,11 +935,16 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
return pathsMap;
}
-std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
- RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute,
- std::function<void(ref<Store>, ref<Store>, const ValidPathInfo &, RepairFlag, CheckSigsFlag)> copyStorePathImpl)
+std::map<StorePath, StorePath> copyPaths(
+ Store & srcStore,
+ Store & dstStore,
+ const StorePathSet & storePaths,
+ RepairFlag repair,
+ CheckSigsFlag checkSigs,
+ SubstituteFlag substitute,
+ std::function<void(Store &, Store &, const ValidPathInfo &, RepairFlag, CheckSigsFlag)> copyStorePathImpl)
{
- auto valid = dstStore->queryValidPaths(storePaths, substitute);
+ auto valid = dstStore.queryValidPaths(storePaths, substitute);
StorePathSet missing;
for (auto & path : storePaths)
@@ -859,9 +954,31 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
for (auto & path : storePaths)
pathsMap.insert_or_assign(path, path);
-
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
+ auto sorted = srcStore.topoSortPaths(missing);
+ std::reverse(sorted.begin(), sorted.end());
+
+ auto source = sinkToSource([&](Sink & sink) {
+ sink << sorted.size();
+ for (auto & storePath : sorted) {
+ auto srcUri = srcStore.getUri();
+ auto dstUri = dstStore.getUri();
+ auto storePathS = srcStore.printStorePath(storePath);
+ Activity act(*logger, lvlInfo, actCopyPath,
+ makeCopyPathMessage(srcUri, dstUri, storePathS),
+ {storePathS, srcUri, dstUri});
+ PushActivity pact(act.id);
+
+ auto info = srcStore.queryPathInfo(storePath);
+ info->write(sink, srcStore, 16);
+ srcStore.narFromPath(storePath, sink);
+ }
+ });
+
+ dstStore.addMultipleToStore(*source, repair, checkSigs);
+
+ #if 0
std::atomic<size_t> nrDone{0};
std::atomic<size_t> nrFailed{0};
std::atomic<uint64_t> bytesExpected{0};
@@ -877,18 +994,21 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
StorePathSet(missing.begin(), missing.end()),
[&](const StorePath & storePath) {
- auto info = srcStore->queryPathInfo(storePath);
+ auto info = srcStore.queryPathInfo(storePath);
auto storePathForDst = storePath;
if (info->ca && info->references.empty()) {
- storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
- if (dstStore->storeDir == srcStore->storeDir)
+ storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore.storeDir == srcStore.storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
- debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ debug("replaced path '%s' to '%s' for substituter '%s'",
+ srcStore.printStorePath(storePath),
+ dstStore.printStorePath(storePathForDst),
+ dstStore.getUri());
}
pathsMap.insert_or_assign(storePath, storePathForDst);
- if (dstStore->isValidPath(storePath)) {
+ if (dstStore.isValidPath(storePath)) {
nrDone++;
showProgress();
return StorePathSet();
@@ -903,19 +1023,22 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
[&](const StorePath & storePath) {
checkInterrupt();
- auto info = srcStore->queryPathInfo(storePath);
+ auto info = srcStore.queryPathInfo(storePath);
auto storePathForDst = storePath;
if (info->ca && info->references.empty()) {
- storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca);
- if (dstStore->storeDir == srcStore->storeDir)
+ storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca);
+ if (dstStore.storeDir == srcStore.storeDir)
assert(storePathForDst == storePath);
if (storePathForDst != storePath)
- debug("replaced path '%s' to '%s' for substituter '%s'", srcStore->printStorePath(storePath), dstStore->printStorePath(storePathForDst), dstStore->getUri());
+ debug("replaced path '%s' to '%s' for substituter '%s'",
+ srcStore.printStorePath(storePath),
+ dstStore.printStorePath(storePathForDst),
+ dstStore.getUri());
}
pathsMap.insert_or_assign(storePath, storePathForDst);
- if (!dstStore->isValidPath(storePathForDst)) {
+ if (!dstStore.isValidPath(storePathForDst)) {
MaintainCount<decltype(nrRunning)> mc(nrRunning);
showProgress();
try {
@@ -924,7 +1047,7 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
nrFailed++;
if (!settings.keepGoing)
throw e;
- logger->log(lvlError, fmt("could not copy %s: %s", dstStore->printStorePath(storePath), e.what()));
+ logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what()));
showProgress();
return;
}
@@ -933,9 +1056,27 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
nrDone++;
showProgress();
});
+ #endif
+
return pathsMap;
}
+void copyClosure(
+ Store & srcStore,
+ Store & dstStore,
+ const RealisedPath::Set & paths,
+ RepairFlag repair,
+ CheckSigsFlag checkSigs,
+ SubstituteFlag substitute)
+{
+ if (&srcStore == &dstStore) return;
+
+ RealisedPath::Set closure;
+ RealisedPath::closure(srcStore, paths, closure);
+
+ copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
+}
+
std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
{
std::string path;
@@ -1008,10 +1149,10 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const
if (! ca) return false;
auto caPath = std::visit(overloaded {
- [&](TextHash th) {
+ [&](const TextHash & th) {
return store.makeTextPath(path.name(), th.hash, references);
},
- [&](FixedOutputHash fsh) {
+ [&](const FixedOutputHash & fsh) {
auto refs = references;
bool hasSelfReference = false;
if (refs.count(path)) {
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 15:43:49 +0000