aboutsummaryrefslogtreecommitdiff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/build/derivation-goal.cc11
-rw-r--r--src/libstore/build/local-derivation-goal.cc12
-rw-r--r--src/libstore/builtins/unpack-channel.cc3
-rw-r--r--src/libstore/daemon.cc2
-rw-r--r--src/libstore/filetransfer.cc4
-rw-r--r--src/libstore/gc.cc15
-rw-r--r--src/libstore/globals.hh19
-rw-r--r--src/libstore/local-binary-cache-store.cc3
-rw-r--r--src/libstore/local-store.cc44
-rw-r--r--src/libstore/local-store.hh2
-rw-r--r--src/libstore/nar-accessor.cc3
-rw-r--r--src/libstore/optimise-store.cc6
-rw-r--r--src/libstore/remote-store.cc19
-rw-r--r--src/libstore/remote-store.hh6
-rw-r--r--src/libstore/sandbox-network.sb4
-rw-r--r--src/libstore/store-api.cc212
-rw-r--r--src/libstore/store-api.hh11
17 files changed, 231 insertions, 145 deletions
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 3fff2385f..41d2e2a1c 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -344,7 +344,7 @@ void DerivationGoal::gaveUpOnSubstitution()
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
/* Ensure that pure, non-fixed-output derivations don't
depend on impure derivations. */
- if (drv->type().isPure() && !drv->type().isFixed()) {
+ if (settings.isExperimentalFeatureEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) {
auto inputDrv = worker.evalStore.readDerivation(i.first);
if (!inputDrv.type().isPure())
throw Error("pure derivation '%s' depends on impure derivation '%s'",
@@ -705,8 +705,7 @@ static void movePath(const Path & src, const Path & dst)
if (changePerm)
chmod_(src, st.st_mode | S_IWUSR);
- if (rename(src.c_str(), dst.c_str()))
- throw SysError("renaming '%1%' to '%2%'", src, dst);
+ renameFile(src, dst);
if (changePerm)
chmod_(dst, st.st_mode);
@@ -914,12 +913,6 @@ void DerivationGoal::buildDone()
outputPaths
);
- if (buildMode == bmCheck) {
- cleanupPostOutputsRegisteredModeCheck();
- done(BuildResult::Built, std::move(builtOutputs));
- return;
- }
-
cleanupPostOutputsRegisteredModeNonCheck();
/* Repeat the build if necessary. */
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 79a241ae0..18b682e13 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -223,8 +223,7 @@ static void movePath(const Path & src, const Path & dst)
if (changePerm)
chmod_(src, st.st_mode | S_IWUSR);
- if (rename(src.c_str(), dst.c_str()))
- throw SysError("renaming '%1%' to '%2%'", src, dst);
+ renameFile(src, dst);
if (changePerm)
chmod_(dst, st.st_mode);
@@ -311,7 +310,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
if (buildMode != bmCheck && status.known->isValid()) continue;
auto p = worker.store.printStorePath(status.known->path);
if (pathExists(chrootRootDir + p))
- rename((chrootRootDir + p).c_str(), p.c_str());
+ renameFile((chrootRootDir + p), p);
}
return diskFull;
@@ -2375,10 +2374,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
if (*scratchPath != finalPath) {
// Also rewrite the output path
auto source = sinkToSource([&](Sink & nextSink) {
- StringSink sink;
- dumpPath(actualPath, sink);
RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink);
- rsink2(sink.s);
+ dumpPath(actualPath, rsink2);
rsink2.flush();
});
Path tmpPath = actualPath + ".tmp";
@@ -2625,8 +2622,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
Path prev = path + checkSuffix;
deletePath(prev);
Path dst = path + checkSuffix;
- if (rename(path.c_str(), dst.c_str()))
- throw SysError("renaming '%s' to '%s'", path, dst);
+ renameFile(path, dst);
}
}
diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc
index 426d58a53..ba04bb16c 100644
--- a/src/libstore/builtins/unpack-channel.cc
+++ b/src/libstore/builtins/unpack-channel.cc
@@ -22,8 +22,7 @@ void builtinUnpackChannel(const BasicDerivation & drv)
auto entries = readDirectory(out);
if (entries.size() != 1)
throw Error("channel tarball '%s' contains more than one file", src);
- if (rename((out + "/" + entries[0].name).c_str(), (out + "/" + channelName).c_str()) == -1)
- throw SysError("renaming channel directory");
+ renameFile((out + "/" + entries[0].name), (out + "/" + channelName));
}
}
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index de69b50ee..48dd5c247 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -239,6 +239,8 @@ struct ClientSettings
else if (trusted
|| name == settings.buildTimeout.name
|| name == settings.buildRepeat.name
+ || name == settings.maxSilentTime.name
+ || name == settings.pollInterval.name
|| name == "connect-timeout"
|| (name == "builders" && value == ""))
settings.set(name, value);
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index 8454ad7d2..5746c32a3 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -308,6 +308,9 @@ struct curlFileTransfer : public FileTransfer
curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders);
+ if (settings.downloadSpeed.get() > 0)
+ curl_easy_setopt(req, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t) (settings.downloadSpeed.get() * 1024));
+
if (request.head)
curl_easy_setopt(req, CURLOPT_NOBODY, 1);
@@ -319,7 +322,6 @@ struct curlFileTransfer : public FileTransfer
}
if (request.verifyTLS) {
- debug("verify TLS: Nix CA file = '%s'", settings.caFile);
if (settings.caFile != "")
curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str());
} else {
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index d58ed78b1..9ef8972f3 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -39,9 +39,7 @@ static void makeSymlink(const Path & link, const Path & target)
createSymlink(target, tempLink);
/* Atomically replace the old one. */
- if (rename(tempLink.c_str(), link.c_str()) == -1)
- throw SysError("cannot rename '%1%' to '%2%'",
- tempLink , link);
+ renameFile(tempLink, link);
}
@@ -621,6 +619,17 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
Path path = storeDir + "/" + std::string(baseName);
Path realPath = realStoreDir + "/" + std::string(baseName);
+ /* There may be temp directories in the store that are still in use
+ by another process. We need to be sure that we can acquire an
+ exclusive lock before deleting them. */
+ if (baseName.find("tmp-", 0) == 0) {
+ AutoCloseFD tmpDirFd = open(realPath.c_str(), O_RDONLY | O_DIRECTORY);
+ if (tmpDirFd.get() == -1 || !lockFile(tmpDirFd.get(), ltWrite, false)) {
+ debug("skipping locked tempdir '%s'", realPath);
+ return;
+ }
+ }
+
printInfo("deleting '%1%'", path);
results.paths.insert(path);
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index a4db3bf08..3dcf3d479 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -560,9 +560,15 @@ public:
R"(
If set to `true` (the default), any non-content-addressed path added
or copied to the Nix store (e.g. when substituting from a binary
- cache) must have a valid signature, that is, be signed using one of
- the keys listed in `trusted-public-keys` or `secret-key-files`. Set
- to `false` to disable signature checking.
+ cache) must have a signature by a trusted key. A trusted key is one
+ listed in `trusted-public-keys`, or a public key counterpart to a
+ private key stored in a file listed in `secret-key-files`.
+
+ Set to `false` to disable signature checking and trust all
+ non-content-addressed paths unconditionally.
+
+ (Content-addressed paths are inherently trustworthy and thus
+ unaffected by this configuration option.)
)"};
Setting<StringSet> extraPlatforms{
@@ -754,6 +760,13 @@ public:
/nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23`.
)"};
+ Setting<unsigned int> downloadSpeed {
+ this, 0, "download-speed",
+ R"(
+ Specify the maximum transfer rate in kilobytes per second you want
+ Nix to use for downloads.
+ )"};
+
Setting<std::string> netrcFile{
this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file",
R"(
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index ba4416f6d..f20b1fa02 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -57,8 +57,7 @@ protected:
AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream);
writeFile(tmp, source);
- if (rename(tmp.c_str(), path2.c_str()))
- throw SysError("renaming '%1%' to '%2%'", tmp, path2);
+ renameFile(tmp, path2);
del.cancel();
}
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index eba3b0fa5..d374d4558 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -158,7 +158,7 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
txn.commit();
}
- writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
+ writeFile(schemaPath, fmt("%d", nixCASchemaVersion), 0666, true);
lockFile(lockFd.get(), ltRead, true);
}
}
@@ -281,7 +281,7 @@ LocalStore::LocalStore(const Params & params)
else if (curSchema == 0) { /* new store */
curSchema = nixSchemaVersion;
openDB(*state, true);
- writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
+ writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true);
}
else if (curSchema < nixSchemaVersion) {
@@ -329,7 +329,7 @@ LocalStore::LocalStore(const Params & params)
txn.commit();
}
- writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
+ writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str(), 0666, true);
lockFile(globalLock.get(), ltRead, true);
}
@@ -751,7 +751,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info))
registerDrvOutput(info);
else
- throw Error("cannot register realisation '%s' because it lacks a valid signature", info.outPath.to_string());
+ throw Error("cannot register realisation '%s' because it lacks a signature by a trusted key", info.outPath.to_string());
}
void LocalStore::registerDrvOutput(const Realisation & info)
@@ -1266,7 +1266,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs)
{
if (checkSigs && pathInfoIsUntrusted(info))
- throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path));
+ throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path));
addTempRoot(info.path);
@@ -1382,13 +1382,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
std::unique_ptr<AutoDelete> delTempDir;
Path tempPath;
+ Path tempDir;
+ AutoCloseFD tempDirFd;
if (!inMemory) {
/* Drain what we pulled so far, and then keep on pulling */
StringSource dumpSource { dump };
ChainSource bothSource { dumpSource, source };
- auto tempDir = createTempDir(realStoreDir, "add");
+ std::tie(tempDir, tempDirFd) = createTempDirInStore();
delTempDir = std::make_unique<AutoDelete>(tempDir);
tempPath = tempDir + "/x";
@@ -1430,8 +1432,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
writeFile(realPath, dumpSource);
} else {
/* Move the temporary path we restored above. */
- if (rename(tempPath.c_str(), realPath.c_str()))
- throw Error("renaming '%s' to '%s'", tempPath, realPath);
+ moveFile(tempPath, realPath);
}
/* For computing the nar hash. In recursive SHA-256 mode, this
@@ -1508,18 +1509,24 @@ StorePath LocalStore::addTextToStore(
/* Create a temporary directory in the store that won't be
- garbage-collected. */
-Path LocalStore::createTempDirInStore()
+ garbage-collected until the returned FD is closed. */
+std::pair<Path, AutoCloseFD> LocalStore::createTempDirInStore()
{
- Path tmpDir;
+ Path tmpDirFn;
+ AutoCloseFD tmpDirFd;
+ bool lockedByUs = false;
do {
/* There is a slight possibility that `tmpDir' gets deleted by
- the GC between createTempDir() and addTempRoot(), so repeat
- until `tmpDir' exists. */
- tmpDir = createTempDir(realStoreDir);
- addTempRoot(parseStorePath(tmpDir));
- } while (!pathExists(tmpDir));
- return tmpDir;
+ the GC between createTempDir() and when we acquire a lock on it.
+ We'll repeat until 'tmpDir' exists and we've locked it. */
+ tmpDirFn = createTempDir(realStoreDir, "tmp");
+ tmpDirFd = open(tmpDirFn.c_str(), O_RDONLY | O_DIRECTORY);
+ if (tmpDirFd.get() < 0) {
+ continue;
+ }
+ lockedByUs = lockFile(tmpDirFd.get(), ltWrite, true);
+ } while (!pathExists(tmpDirFn) || !lockedByUs);
+ return {tmpDirFn, std::move(tmpDirFd)};
}
@@ -1942,8 +1949,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log)
writeFile(tmpFile, compress("bzip2", log));
- if (rename(tmpFile.c_str(), logPath.c_str()) != 0)
- throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath);
+ renameFile(tmpFile, logPath);
}
std::optional<std::string> LocalStore::getVersion()
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 70d225be3..bd0ce1fe6 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -256,7 +256,7 @@ private:
void findRuntimeRoots(Roots & roots, bool censor);
- Path createTempDirInStore();
+ std::pair<Path, AutoCloseFD> createTempDirInStore();
void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 72d41cc94..398147fc3 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -75,6 +75,9 @@ struct NarAccessor : public FSAccessor
createMember(path, {FSAccessor::Type::tRegular, false, 0, 0});
}
+ void closeRegularFile() override
+ { }
+
void isExecutable() override
{
parents.top()->isExecutable = true;
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 8af9b1dde..4d2781180 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -229,7 +229,9 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
}
/* Atomically replace the old file with the new hard link. */
- if (rename(tempLink.c_str(), path.c_str()) == -1) {
+ try {
+ renameFile(tempLink, path);
+ } catch (SysError & e) {
if (unlink(tempLink.c_str()) == -1)
printError("unable to unlink '%1%'", tempLink);
if (errno == EMLINK) {
@@ -240,7 +242,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
debug("'%s' has reached maximum number of links", linkPath);
return;
}
- throw SysError("cannot rename '%1%' to '%2%'", tempLink, path);
+ throw;
}
stats.filesLinked++;
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index bc36aef5d..96a29155c 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -580,7 +580,6 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
try {
conn->to.written = 0;
- conn->to.warn = true;
connections->incCapacity();
{
Finally cleanup([&]() { connections->decCapacity(); });
@@ -591,7 +590,6 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
dumpString(contents, conn->to);
}
}
- conn->to.warn = false;
conn.processStderr();
} catch (SysError & e) {
/* Daemon closed while we were sending the path. Probably OOM
@@ -674,6 +672,23 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
void RemoteStore::addMultipleToStore(
+ PathsSource & pathsToCopy,
+ Activity & act,
+ RepairFlag repair,
+ CheckSigsFlag checkSigs)
+{
+ auto source = sinkToSource([&](Sink & sink) {
+ sink << pathsToCopy.size();
+ for (auto & [pathInfo, pathSource] : pathsToCopy) {
+ pathInfo.write(sink, *this, 16);
+ pathSource->drainInto(sink);
+ }
+ });
+
+ addMultipleToStore(*source, repair, checkSigs);
+}
+
+void RemoteStore::addMultipleToStore(
Source & source,
RepairFlag repair,
CheckSigsFlag checkSigs)
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 8493be6fc..11d089cd2 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -88,6 +88,12 @@ public:
RepairFlag repair,
CheckSigsFlag checkSigs) override;
+ void addMultipleToStore(
+ PathsSource & pathsToCopy,
+ Activity & act,
+ RepairFlag repair,
+ CheckSigsFlag checkSigs) override;
+
StorePath addTextToStore(
std::string_view name,
std::string_view s,
diff --git a/src/libstore/sandbox-network.sb b/src/libstore/sandbox-network.sb
index 56beec761..19e9eea9a 100644
--- a/src/libstore/sandbox-network.sb
+++ b/src/libstore/sandbox-network.sb
@@ -14,3 +14,7 @@
; Allow DNS lookups.
(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder")))
+
+; Allow access to trustd.
+(allow mach-lookup (global-name "com.apple.trustd"))
+(allow mach-lookup (global-name "com.apple.trustd.agent"))
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 91dbd991e..86b12257a 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -258,6 +258,84 @@ StorePath Store::addToStore(
return addToStoreFromDump(*source, name, method, hashAlgo, repair, references);
}
+void Store::addMultipleToStore(
+ PathsSource & pathsToCopy,
+ Activity & act,
+ RepairFlag repair,
+ CheckSigsFlag checkSigs)
+{
+ std::atomic<size_t> nrDone{0};
+ std::atomic<size_t> nrFailed{0};
+ std::atomic<uint64_t> bytesExpected{0};
+ std::atomic<uint64_t> nrRunning{0};
+
+ using PathWithInfo = std::pair<ValidPathInfo, std::unique_ptr<Source>>;
+
+ std::map<StorePath, PathWithInfo *> infosMap;
+ StorePathSet storePathsToAdd;
+ for (auto & thingToAdd : pathsToCopy) {
+ infosMap.insert_or_assign(thingToAdd.first.path, &thingToAdd);
+ storePathsToAdd.insert(thingToAdd.first.path);
+ }
+
+ auto showProgress = [&]() {
+ act.progress(nrDone, pathsToCopy.size(), nrRunning, nrFailed);
+ };
+
+ ThreadPool pool;
+
+ processGraph<StorePath>(pool,
+ storePathsToAdd,
+
+ [&](const StorePath & path) {
+
+ auto & [info, _] = *infosMap.at(path);
+
+ if (isValidPath(info.path)) {
+ nrDone++;
+ showProgress();
+ return StorePathSet();
+ }
+
+ bytesExpected += info.narSize;
+ act.setExpected(actCopyPath, bytesExpected);
+
+ return info.references;
+ },
+
+ [&](const StorePath & path) {
+ checkInterrupt();
+
+ auto & [info_, source_] = *infosMap.at(path);
+ auto info = info_;
+ info.ultimate = false;
+
+ /* Make sure that the Source object is destroyed when
+ we're done. In particular, a SinkToSource object must
+ be destroyed to ensure that the destructors on its
+ stack frame are run; this includes
+ LegacySSHStore::narFromPath()'s connection lock. */
+ auto source = std::move(source_);
+
+ if (!isValidPath(info.path)) {
+ MaintainCount<decltype(nrRunning)> mc(nrRunning);
+ showProgress();
+ try {
+ addToStore(info, *source, repair, checkSigs);
+ } catch (Error & e) {
+ nrFailed++;
+ if (!settings.keepGoing)
+ throw e;
+ printMsg(lvlError, "could not copy %s: %s", printStorePath(path), e.what());
+ showProgress();
+ return;
+ }
+ }
+
+ nrDone++;
+ showProgress();
+ });
+}
void Store::addMultipleToStore(
Source & source,
@@ -992,113 +1070,61 @@ std::map<StorePath, StorePath> copyPaths(
for (auto & path : storePaths)
if (!valid.count(path)) missing.insert(path);
+ Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
+
+ // In the general case, `addMultipleToStore` requires a sorted list of
+ // store paths to add, so sort them right now
+ auto sortedMissing = srcStore.topoSortPaths(missing);
+ std::reverse(sortedMissing.begin(), sortedMissing.end());
+
std::map<StorePath, StorePath> pathsMap;
for (auto & path : storePaths)
pathsMap.insert_or_assign(path, path);
- Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
+ Store::PathsSource pathsToCopy;
+
+ auto computeStorePathForDst = [&](const ValidPathInfo & currentPathInfo) -> StorePath {
+ auto storePathForSrc = currentPathInfo.path;
+ auto storePathForDst = storePathForSrc;
+ if (currentPathInfo.ca && currentPathInfo.references.empty()) {
+ storePathForDst = dstStore.makeFixedOutputPathFromCA(storePathForSrc.name(), *currentPathInfo.ca);
+ if (dstStore.storeDir == srcStore.storeDir)
+ assert(storePathForDst == storePathForSrc);
+ if (storePathForDst != storePathForSrc)
+ debug("replaced path '%s' to '%s' for substituter '%s'",
+ srcStore.printStorePath(storePathForSrc),
+ dstStore.printStorePath(storePathForDst),
+ dstStore.getUri());
+ }
+ return storePathForDst;
+ };
- auto sorted = srcStore.topoSortPaths(missing);
- std::reverse(sorted.begin(), sorted.end());
+ for (auto & missingPath : sortedMissing) {
+ auto info = srcStore.queryPathInfo(missingPath);
- auto source = sinkToSource([&](Sink & sink) {
- sink << sorted.size();
- for (auto & storePath : sorted) {
+ auto storePathForDst = computeStorePathForDst(*info);
+ pathsMap.insert_or_assign(missingPath, storePathForDst);
+
+ ValidPathInfo infoForDst = *info;
+ infoForDst.path = storePathForDst;
+
+ auto source = sinkToSource([&](Sink & sink) {
+ // We can reasonably assume that the copy will happen whenever we
+ // read the path, so log something about that at that point
auto srcUri = srcStore.getUri();
auto dstUri = dstStore.getUri();
- auto storePathS = srcStore.printStorePath(storePath);
+ auto storePathS = srcStore.printStorePath(missingPath);
Activity act(*logger, lvlInfo, actCopyPath,
makeCopyPathMessage(srcUri, dstUri, storePathS),
{storePathS, srcUri, dstUri});
PushActivity pact(act.id);
- auto info = srcStore.queryPathInfo(storePath);
- info->write(sink, srcStore, 16);
- srcStore.narFromPath(storePath, sink);
- }
- });
-
- dstStore.addMultipleToStore(*source, repair, checkSigs);
-
- #if 0
- std::atomic<size_t> nrDone{0};
- std::atomic<size_t> nrFailed{0};
- std::atomic<uint64_t> bytesExpected{0};
- std::atomic<uint64_t> nrRunning{0};
-
- auto showProgress = [&]() {
- act.progress(nrDone, missing.size(), nrRunning, nrFailed);
- };
-
- ThreadPool pool;
-
- processGraph<StorePath>(pool,
- StorePathSet(missing.begin(), missing.end()),
-
- [&](const StorePath & storePath) {
- auto info = srcStore.queryPathInfo(storePath);
- auto storePathForDst = storePath;
- if (info->ca && info->references.empty()) {
- storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca);
- if (dstStore.storeDir == srcStore.storeDir)
- assert(storePathForDst == storePath);
- if (storePathForDst != storePath)
- debug("replaced path '%s' to '%s' for substituter '%s'",
- srcStore.printStorePath(storePath),
- dstStore.printStorePath(storePathForDst),
- dstStore.getUri());
- }
- pathsMap.insert_or_assign(storePath, storePathForDst);
-
- if (dstStore.isValidPath(storePath)) {
- nrDone++;
- showProgress();
- return StorePathSet();
- }
-
- bytesExpected += info->narSize;
- act.setExpected(actCopyPath, bytesExpected);
-
- return info->references;
- },
-
- [&](const StorePath & storePath) {
- checkInterrupt();
-
- auto info = srcStore.queryPathInfo(storePath);
-
- auto storePathForDst = storePath;
- if (info->ca && info->references.empty()) {
- storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca);
- if (dstStore.storeDir == srcStore.storeDir)
- assert(storePathForDst == storePath);
- if (storePathForDst != storePath)
- debug("replaced path '%s' to '%s' for substituter '%s'",
- srcStore.printStorePath(storePath),
- dstStore.printStorePath(storePathForDst),
- dstStore.getUri());
- }
- pathsMap.insert_or_assign(storePath, storePathForDst);
-
- if (!dstStore.isValidPath(storePathForDst)) {
- MaintainCount<decltype(nrRunning)> mc(nrRunning);
- showProgress();
- try {
- copyStorePath(srcStore, dstStore, storePath, repair, checkSigs);
- } catch (Error &e) {
- nrFailed++;
- if (!settings.keepGoing)
- throw e;
- printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what());
- showProgress();
- return;
- }
- }
-
- nrDone++;
- showProgress();
+ srcStore.narFromPath(missingPath, sink);
});
- #endif
+ pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
+ }
+
+ dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
return pathsMap;
}
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 0c8a4db56..c8a667c6d 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -1,5 +1,6 @@
#pragma once
+#include "nar-info.hh"
#include "realisation.hh"
#include "path.hh"
#include "derived-path.hh"
@@ -359,12 +360,22 @@ public:
virtual void addToStore(const ValidPathInfo & info, Source & narSource,
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) = 0;
+ // A list of paths infos along with a source providing the content of the
+ // associated store path
+ using PathsSource = std::vector<std::pair<ValidPathInfo, std::unique_ptr<Source>>>;
+
/* Import multiple paths into the store. */
virtual void addMultipleToStore(
Source & source,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs);
+ virtual void addMultipleToStore(
+ PathsSource & pathsToCopy,
+ Activity & act,
+ RepairFlag repair = NoRepair,
+ CheckSigsFlag checkSigs = CheckSigs);
+
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
The function object `filter' can be used to exclude files (see
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 03:41:53 +0000