aboutsummaryrefslogtreecommitdiff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/binary-cache-store.cc8
-rw-r--r--src/libstore/build/derivation-goal.cc10
-rw-r--r--src/libstore/build/entry-points.cc2
-rw-r--r--src/libstore/build/local-derivation-goal.cc8
-rw-r--r--src/libstore/daemon.cc5
-rw-r--r--src/libstore/derivations.cc2
-rw-r--r--src/libstore/derived-path.cc2
-rw-r--r--src/libstore/gc.cc759
-rw-r--r--src/libstore/globals.cc15
-rw-r--r--src/libstore/globals.hh16
-rw-r--r--src/libstore/local-store.cc30
-rw-r--r--src/libstore/local-store.hh31
-rw-r--r--src/libstore/misc.cc5
-rw-r--r--src/libstore/pathlocks.cc13
-rw-r--r--src/libstore/pathlocks.hh14
-rw-r--r--src/libstore/remote-store.cc9
-rw-r--r--src/libstore/remote-store.hh2
-rw-r--r--src/libstore/store-api.cc32
-rw-r--r--src/libstore/store-api.hh21
-rw-r--r--src/libstore/uds-remote-store.cc9
20 files changed, 473 insertions, 520 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 8fce94264..943132754 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo");
- std::string hashPart(narInfo->path.hashPart());
-
{
auto state_(state.lock());
- state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
+ state_->pathInfoCache.upsert(
+ std::string(narInfo->path.to_string()),
+ PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) });
}
if (diskCache)
- diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
+ diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo));
}
AutoCloseFD openFile(const Path & path)
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 0907120db..b924d23b2 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -204,7 +204,7 @@ void DerivationGoal::haveDerivation()
trace("have derivation");
if (drv->type() == DerivationType::CAFloating)
- settings.requireExperimentalFeature("ca-derivations");
+ settings.requireExperimentalFeature(Xp::CaDerivations);
retrySubstitution = false;
@@ -453,7 +453,7 @@ void DerivationGoal::inputsRealised()
if (useDerivation) {
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
- if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
+ if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) &&
((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
|| fullDrv.type() == DerivationType::DeferredInputAddressed)) {
/* We are be able to resolve this derivation based on the
@@ -616,7 +616,9 @@ void DerivationGoal::tryToBuild()
/* Don't do a remote build if the derivation has the attribute
`preferLocalBuild' set. Also, check and repair modes are only
supported for local builds. */
- bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
+ bool buildLocally =
+ (buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store))
+ && settings.maxBuildJobs.get() != 0;
if (!buildLocally) {
switch (tryBuildHook()) {
@@ -1273,7 +1275,7 @@ void DerivationGoal::checkPathValidity()
: PathStatus::Corrupt,
};
}
- if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+ if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first};
if (auto real = worker.store.queryRealisation(drvOutput)) {
info.known = {
diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc
index b1e5d8504..9b4cfd835 100644
--- a/src/libstore/build/entry-points.cc
+++ b/src/libstore/build/entry-points.cc
@@ -73,7 +73,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
outputId,
Realisation{ outputId, *staticOutput.second}
);
- if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+ if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
auto realisation = this->queryRealisation(outputId);
if (realisation)
result.builtOutputs.insert_or_assign(
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 8d245f84a..2182f0bb4 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1259,7 +1259,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
for (auto & [outputName, outputPath] : outputs)
if (wantOutput(outputName, bfd.outputs)) {
newPaths.insert(outputPath);
- if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+ if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
auto thisRealisation = next->queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName}
);
@@ -1320,7 +1320,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
void LocalDerivationGoal::startDaemon()
{
- settings.requireExperimentalFeature("recursive-nix");
+ settings.requireExperimentalFeature(Xp::RecursiveNix);
Store::Params params;
params["path-info-cache-size"] = "0";
@@ -1353,7 +1353,7 @@ void LocalDerivationGoal::startDaemon()
AutoCloseFD remote = accept(daemonSocket.get(),
(struct sockaddr *) &remoteAddr, &remoteAddrLen);
if (!remote) {
- if (errno == EINTR) continue;
+ if (errno == EINTR || errno == EAGAIN) continue;
if (errno == EINVAL) break;
throw SysError("accepting connection");
}
@@ -2561,7 +2561,7 @@ void LocalDerivationGoal::registerOutputs()
that for floating CA derivations, which otherwise couldn't be cached,
but it's fine to do in all cases. */
- if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+ if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
for (auto& [outputName, newInfo] : infos) {
auto thisRealisation = Realisation{
.id = DrvOutput{initialOutputs.at(outputName).outputHash,
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 164a9b2be..9f6c3da3d 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -230,11 +230,12 @@ struct ClientSettings
else if (name == settings.experimentalFeatures.name) {
// We don’t want to forward the experimental features to
// the daemon, as that could cause some pretty weird stuff
- if (tokenizeString<Strings>(value) != settings.experimentalFeatures.get())
+ if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get())
debug("Ignoring the client-specified experimental features");
}
else if (trusted
|| name == settings.buildTimeout.name
+ || name == settings.buildRepeat.name
|| name == "connect-timeout"
|| (name == "builders" && value == ""))
settings.set(name, value);
@@ -624,9 +625,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
+ // Obsolete.
case wopSyncWithGC: {
logger->startWork();
- store->syncWithGC();
logger->stopWork();
to << 1;
break;
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index ef8765841..b926bb711 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -187,7 +187,7 @@ static DerivationOutput parseDerivationOutput(const Store & store,
},
};
} else {
- settings.requireExperimentalFeature("ca-derivations");
+ settings.requireExperimentalFeature(Xp::CaDerivations);
assert(pathS == "");
return DerivationOutput {
.output = DerivationOutputCAFloating {
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index e55af21e9..3d188e981 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -100,7 +100,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
staticOutputHashes(store, store.readDerivation(p.drvPath));
for (auto& [outputName, outputPath] : p.outputs) {
if (settings.isExperimentalFeatureEnabled(
- "ca-derivations")) {
+ Xp::CaDerivations)) {
auto thisRealisation = store.queryRealisation(
DrvOutput{drvHashes.at(outputName), outputName});
assert(thisRealisation); // We’ve built it, so we must h
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 5a62c6529..8030c84f5 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -10,48 +10,22 @@
#include <regex>
#include <random>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/statvfs.h>
+#include <climits>
#include <errno.h>
#include <fcntl.h>
+#include <poll.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/statvfs.h>
+#include <sys/types.h>
+#include <sys/un.h>
#include <unistd.h>
-#include <climits>
namespace nix {
-static string gcLockName = "gc.lock";
-static string gcRootsDir = "gcroots";
-
-
-/* Acquire the global GC lock. This is used to prevent new Nix
- processes from starting after the temporary root files have been
- read. To be precise: when they try to create a new temporary root
- file, they will block until the garbage collector has finished /
- yielded the GC lock. */
-AutoCloseFD LocalStore::openGCLock(LockType lockType)
-{
- Path fnGCLock = (format("%1%/%2%")
- % stateDir % gcLockName).str();
-
- debug(format("acquiring global GC lock '%1%'") % fnGCLock);
-
- AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
- if (!fdGCLock)
- throw SysError("opening global GC lock '%1%'", fnGCLock);
-
- if (!lockFile(fdGCLock.get(), lockType, false)) {
- printInfo("waiting for the big garbage collector lock...");
- lockFile(fdGCLock.get(), lockType, true);
- }
-
- /* !!! Restrict read permission on the GC root. Otherwise any
- process that can open the file for reading can DoS the
- collector. */
-
- return fdGCLock;
-}
+static std::string gcSocketPath = "/gc-socket/socket";
+static std::string gcRootsDir = "gcroots";
static void makeSymlink(const Path & link, const Path & target)
@@ -71,12 +45,6 @@ static void makeSymlink(const Path & link, const Path & target)
}
-void LocalStore::syncWithGC()
-{
- AutoCloseFD fdGCLock = openGCLock(ltRead);
-}
-
-
void LocalStore::addIndirectRoot(const Path & path)
{
string hash = hashString(htSHA1, path).to_string(Base32, false);
@@ -95,6 +63,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
"creating a garbage collector root (%1%) in the Nix store is forbidden "
"(are you running nix-build inside the store?)", gcRoot);
+ /* Register this root with the garbage collector, if it's
+ running. This should be superfluous since the caller should
+ have registered this root yet, but let's be on the safe
+ side. */
+ addTempRoot(storePath);
+
/* Don't clobber the link if it already exists and doesn't
point to the Nix store. */
if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot))))
@@ -102,11 +76,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot
makeSymlink(gcRoot, printStorePath(storePath));
addIndirectRoot(gcRoot);
- /* Grab the global GC root, causing us to block while a GC is in
- progress. This prevents the set of permanent roots from
- increasing while a GC is in progress. */
- syncWithGC();
-
return gcRoot;
}
@@ -119,8 +88,6 @@ void LocalStore::addTempRoot(const StorePath & path)
if (!state->fdTempRoots) {
while (1) {
- AutoCloseFD fdGCLock = openGCLock(ltRead);
-
if (pathExists(fnTempRoots))
/* It *must* be stale, since there can be no two
processes with the same pid. */
@@ -128,10 +95,8 @@ void LocalStore::addTempRoot(const StorePath & path)
state->fdTempRoots = openLockFile(fnTempRoots, true);
- fdGCLock = -1;
-
- debug(format("acquiring read lock on '%1%'") % fnTempRoots);
- lockFile(state->fdTempRoots.get(), ltRead, true);
+ debug("acquiring write lock on '%s'", fnTempRoots);
+ lockFile(state->fdTempRoots.get(), ltWrite, true);
/* Check whether the garbage collector didn't get in our
way. */
@@ -147,24 +112,55 @@ void LocalStore::addTempRoot(const StorePath & path)
}
- /* Upgrade the lock to a write lock. This will cause us to block
- if the garbage collector is holding our lock. */
- debug(format("acquiring write lock on '%1%'") % fnTempRoots);
- lockFile(state->fdTempRoots.get(), ltWrite, true);
+ if (!state->fdGCLock)
+ state->fdGCLock = openGCLock();
+
+ restart:
+ FdLock gcLock(state->fdGCLock.get(), ltRead, false, "");
+
+ if (!gcLock.acquired) {
+ /* We couldn't get a shared global GC lock, so the garbage
+ collector is running. So we have to connect to the garbage
+ collector and inform it about our root. */
+ if (!state->fdRootsSocket) {
+ auto socketPath = stateDir.get() + gcSocketPath;
+ debug("connecting to '%s'", socketPath);
+ state->fdRootsSocket = createUnixDomainSocket();
+ nix::connect(state->fdRootsSocket.get(), socketPath);
+ }
+
+ try {
+ debug("sending GC root '%s'", printStorePath(path));
+ writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false);
+ char c;
+ readFull(state->fdRootsSocket.get(), &c, 1);
+ assert(c == '1');
+ debug("got ack for GC root '%s'", printStorePath(path));
+ } catch (SysError & e) {
+ /* The garbage collector may have exited, so we need to
+ restart. */
+ if (e.errNo == EPIPE) {
+ debug("GC socket disconnected");
+ state->fdRootsSocket.close();
+ goto restart;
+ }
+ } catch (EndOfFile & e) {
+ debug("GC socket disconnected");
+ state->fdRootsSocket.close();
+ goto restart;
+ }
+ }
+ /* Append the store path to the temporary roots file. */
string s = printStorePath(path) + '\0';
writeFull(state->fdTempRoots.get(), s);
-
- /* Downgrade to a read lock. */
- debug(format("downgrading to read lock on '%1%'") % fnTempRoots);
- lockFile(state->fdTempRoots.get(), ltRead, true);
}
static std::string censored = "{censored}";
-void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
+void LocalStore::findTempRoots(Roots & tempRoots, bool censor)
{
/* Read the `temproots' directory for per-process temporary root
files. */
@@ -179,35 +175,25 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
pid_t pid = std::stoi(i.name);
debug(format("reading temporary root file '%1%'") % path);
- FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)));
- if (!*fd) {
+ AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666));
+ if (!fd) {
/* It's okay if the file has disappeared. */
if (errno == ENOENT) continue;
throw SysError("opening temporary roots file '%1%'", path);
}
- /* This should work, but doesn't, for some reason. */
- //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
- //if (*fd == -1) continue;
-
/* Try to acquire a write lock without blocking. This can
only succeed if the owning process has died. In that case
we don't care about its temporary roots. */
- if (lockFile(fd->get(), ltWrite, false)) {
+ if (lockFile(fd.get(), ltWrite, false)) {
printInfo("removing stale temporary roots file '%1%'", path);
unlink(path.c_str());
- writeFull(fd->get(), "d");
+ writeFull(fd.get(), "d");
continue;
}
- /* Acquire a read lock. This will prevent the owning process
- from upgrading to a write lock, therefore it will block in
- addTempRoot(). */
- debug(format("waiting for read lock on '%1%'") % path);
- lockFile(fd->get(), ltRead, true);
-
/* Read the entire file. */
- string contents = readFile(fd->get());
+ string contents = readFile(fd.get());
/* Extract the roots. */
string::size_type pos = 0, end;
@@ -218,8 +204,6 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid));
pos = end + 1;
}
-
- fds.push_back(fd); /* keep open */
}
}
@@ -304,8 +288,7 @@ Roots LocalStore::findRoots(bool censor)
Roots roots;
findRootsNoTemp(roots, censor);
- FDs fds;
- findTempRoots(fds, roots, censor);
+ findTempRoots(roots, censor);
return roots;
}
@@ -455,391 +438,397 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
struct GCLimitReached { };
-struct LocalStore::GCState
-{
- const GCOptions & options;
- GCResults & results;
- StorePathSet roots;
- StorePathSet tempRoots;
- StorePathSet dead;
- StorePathSet alive;
- bool gcKeepOutputs;
- bool gcKeepDerivations;
- uint64_t bytesInvalidated;
- bool moveToTrash = true;
- bool shouldDelete;
- GCState(const GCOptions & options, GCResults & results)
- : options(options), results(results), bytesInvalidated(0) { }
-};
-
-
-bool LocalStore::isActiveTempFile(const GCState & state,
- const Path & path, const string & suffix)
+void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
{
- return hasSuffix(path, suffix)
- && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size())));
-}
+ bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
+ bool gcKeepOutputs = settings.gcKeepOutputs;
+ bool gcKeepDerivations = settings.gcKeepDerivations;
+ StorePathSet roots, dead, alive;
-void LocalStore::deleteGarbage(GCState & state, const Path & path)
-{
- uint64_t bytesFreed;
- deletePath(path, bytesFreed);
- state.results.bytesFreed += bytesFreed;
-}
+ struct Shared
+ {
+ // The temp roots only store the hash part to make it easier to
+ // ignore suffixes like '.lock', '.chroot' and '.check'.
+ std::unordered_set<std::string> tempRoots;
+ // Hash part of the store path currently being deleted, if
+ // any.
+ std::optional<std::string> pending;
+ };
-void LocalStore::deletePathRecursive(GCState & state, const Path & path)
-{
- checkInterrupt();
-
- uint64_t size = 0;
-
- auto storePath = maybeParseStorePath(path);
- if (storePath && isValidPath(*storePath)) {
- StorePathSet referrers;
- queryReferrers(*storePath, referrers);
- for (auto & i : referrers)
- if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i));
- size = queryPathInfo(*storePath)->narSize;
- invalidatePathChecked(*storePath);
- }
+ Sync<Shared> _shared;
- Path realPath = realStoreDir + "/" + std::string(baseNameOf(path));
+ std::condition_variable wakeup;
- struct stat st;
- if (lstat(realPath.c_str(), &st)) {
- if (errno == ENOENT) return;
- throw SysError("getting status of %1%", realPath);
+ /* Using `--ignore-liveness' with `--delete' can have unintended
+ consequences if `keep-outputs' or `keep-derivations' are true
+ (the garbage collector will recurse into deleting the outputs
+ or derivers, respectively). So disable them. */
+ if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
+ gcKeepOutputs = false;
+ gcKeepDerivations = false;
}
- printInfo(format("deleting '%1%'") % path);
-
- state.results.paths.insert(path);
+ if (shouldDelete)
+ deletePath(reservedPath);
- /* If the path is not a regular file or symlink, move it to the
- trash directory. The move is to ensure that later (when we're
- not holding the global GC lock) we can delete the path without
- being afraid that the path has become alive again. Otherwise
- delete it right away. */
- if (state.moveToTrash && S_ISDIR(st.st_mode)) {
- // Estimate the amount freed using the narSize field. FIXME:
- // if the path was not valid, need to determine the actual
- // size.
- try {
- if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1)
- throw SysError("making '%1%' writable", realPath);
- Path tmp = trashDir + "/" + std::string(baseNameOf(path));
- if (rename(realPath.c_str(), tmp.c_str()))
- throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp);
- state.bytesInvalidated += size;
- } catch (SysError & e) {
- if (e.errNo == ENOSPC) {
- printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg());
- deleteGarbage(state, realPath);
+ /* Acquire the global GC root. Note: we don't use fdGCLock
+ here because then in auto-gc mode, another thread could
+ downgrade our exclusive lock. */
+ auto fdGCLock = openGCLock();
+ FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock...");
+
+ /* Start the server for receiving new roots. */
+ auto socketPath = stateDir.get() + gcSocketPath;
+ createDirs(dirOf(socketPath));
+ auto fdServer = createUnixDomainSocket(socketPath, 0666);
+
+ if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1)
+ throw SysError("making socket '%1%' non-blocking", socketPath);
+
+ Pipe shutdownPipe;
+ shutdownPipe.create();
+
+ std::thread serverThread([&]() {
+ Sync<std::map<int, std::thread>> connections;
+
+ Finally cleanup([&]() {
+ debug("GC roots server shutting down");
+ while (true) {
+ auto item = remove_begin(*connections.lock());
+ if (!item) break;
+ auto & [fd, thread] = *item;
+ shutdown(fd, SHUT_RDWR);
+ thread.join();
}
- }
- } else
- deleteGarbage(state, realPath);
-
- if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) {
- printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed);
- throw GCLimitReached();
- }
-}
-
-
-bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path)
-{
- if (visited.count(path)) return false;
-
- if (state.alive.count(path)) return true;
-
- if (state.dead.count(path)) return false;
-
- if (state.roots.count(path)) {
- debug("cannot delete '%1%' because it's a root", printStorePath(path));
- state.alive.insert(path);
- return true;
- }
-
- visited.insert(path);
-
- if (!isValidPath(path)) return false;
-
- StorePathSet incoming;
-
- /* Don't delete this path if any of its referrers are alive. */
- queryReferrers(path, incoming);
-
- /* If keep-derivations is set and this is a derivation, then
- don't delete the derivation if any of the outputs are alive. */
- if (state.gcKeepDerivations && path.isDerivation()) {
- for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path))
- if (maybeOutPath &&
- isValidPath(*maybeOutPath) &&
- queryPathInfo(*maybeOutPath)->deriver == path
- )
- incoming.insert(*maybeOutPath);
- }
-
- /* If keep-outputs is set, then don't delete this path if there
- are derivers of this path that are not garbage. */
- if (state.gcKeepOutputs) {
- auto derivers = queryValidDerivers(path);
- for (auto & i : derivers)
- incoming.insert(i);
- }
+ });
+
+ while (true) {
+ std::vector<struct pollfd> fds;
+ fds.push_back({.fd = shutdownPipe.readSide.get(), .events = POLLIN});
+ fds.push_back({.fd = fdServer.get(), .events = POLLIN});
+ auto count = poll(fds.data(), fds.size(), -1);
+ assert(count != -1);
+
+ if (fds[0].revents)
+ /* Parent is asking us to quit. */
+ break;
+
+ if (fds[1].revents) {
+ /* Accept a new connection. */
+ assert(fds[1].revents & POLLIN);
+ AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr);
+ if (!fdClient) continue;
+
+ /* Process the connection in a separate thread. */
+ auto fdClient_ = fdClient.get();
+ std::thread clientThread([&, fdClient = std::move(fdClient)]() {
+ Finally cleanup([&]() {
+ auto conn(connections.lock());
+ auto i = conn->find(fdClient.get());
+ if (i != conn->end()) {
+ i->second.detach();
+ conn->erase(i);
+ }
+ });
+
+ while (true) {
+ try {
+ auto path = readLine(fdClient.get());
+ auto storePath = maybeParseStorePath(path);
+ if (storePath) {
+ debug("got new GC root '%s'", path);
+ auto hashPart = std::string(storePath->hashPart());
+ auto shared(_shared.lock());
+ shared->tempRoots.insert(hashPart);
+ /* If this path is currently being
+ deleted, then we have to wait until
+ deletion is finished to ensure that
+ the client doesn't start
+ re-creating it before we're
+ done. FIXME: ideally we would use a
+ FD for this so we don't block the
+ poll loop. */
+ while (shared->pending == hashPart) {
+ debug("synchronising with deletion of path '%s'", path);
+ shared.wait(wakeup);
+ }
+ } else
+ printError("received garbage instead of a root from client");
+ writeFull(fdClient.get(), "1", false);
+ } catch (Error &) { break; }
+ }
+ });
- for (auto & i : incoming)
- if (i != path)
- if (canReachRoot(state, visited, i)) {
- state.alive.insert(path);
- return true;
+ connections.lock()->insert({fdClient_, std::move(clientThread)});
}
+ }
+ });
- return false;
-}
-
-
-void LocalStore::tryToDelete(GCState & state, const Path & path)
-{
- checkInterrupt();
-
- auto realPath = realStoreDir + "/" + std::string(baseNameOf(path));
- if (realPath == linksDir || realPath == trashDir) return;
-
- //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path);
-
- auto storePath = maybeParseStorePath(path);
-
- if (!storePath || !isValidPath(*storePath)) {
- /* A lock file belonging to a path that we're building right
- now isn't garbage. */
- if (isActiveTempFile(state, path, ".lock")) return;
+ Finally stopServer([&]() {
+ writeFull(shutdownPipe.writeSide.get(), "x", false);
+ wakeup.notify_all();
+ if (serverThread.joinable()) serverThread.join();
+ });
- /* Don't delete .chroot directories for derivations that are
- currently being built. */
- if (isActiveTempFile(state, path, ".chroot")) return;
+ /* Find the roots. Since we've grabbed the GC lock, the set of
+ permanent roots cannot increase now. */
+ printInfo("finding garbage collector roots...");
+ Roots rootMap;
+ if (!options.ignoreLiveness)
+ findRootsNoTemp(rootMap, true);
- /* Don't delete .check directories for derivations that are
- currently being built, because we may need to run
- diff-hook. */
- if (isActiveTempFile(state, path, ".check")) return;
- }
+ for (auto & i : rootMap) roots.insert(i.first);
- StorePathSet visited;
-
- if (storePath && canReachRoot(state, visited, *storePath)) {
- debug("cannot delete '%s' because it's still reachable", path);
- } else {
- /* No path we visited was a root, so everything is garbage.
- But we only delete ‘path’ and its referrers here so that
- ‘nix-store --delete’ doesn't have the unexpected effect of
- recursing into derivations and outputs. */
- for (auto & i : visited)
- state.dead.insert(i);
- if (state.shouldDelete)
- deletePathRecursive(state, path);
+ /* Read the temporary roots created before we acquired the global
+ GC root. Any new roots will be sent to our socket. */
+ Roots tempRoots;
+ findTempRoots(tempRoots, true);
+ for (auto & root : tempRoots) {
+ _shared.lock()->tempRoots.insert(std::string(root.first.hashPart()));
+ roots.insert(root.first);
}
-}
+ /* Helper function that deletes a path from the store and throws
+ GCLimitReached if we've deleted enough garbage. */
+ auto deleteFromStore = [&](std::string_view baseName)
+ {
+ Path path = storeDir + "/" + std::string(baseName);
+ Path realPath = realStoreDir + "/" + std::string(baseName);
-/* Unlink all files in /nix/store/.links that have a link count of 1,
- which indicates that there are no other links and so they can be
- safely deleted. FIXME: race condition with optimisePath(): we
- might see a link count of 1 just before optimisePath() increases
- the link count. */
-void LocalStore::removeUnusedLinks(const GCState & state)
-{
- AutoCloseDir dir(opendir(linksDir.c_str()));
- if (!dir) throw SysError("opening directory '%1%'", linksDir);
-
- int64_t actualSize = 0, unsharedSize = 0;
+ printInfo("deleting '%1%'", path);
- struct dirent * dirent;
- while (errno = 0, dirent = readdir(dir.get())) {
- checkInterrupt();
- string name = dirent->d_name;
- if (name == "." || name == "..") continue;
- Path path = linksDir + "/" + name;
+ results.paths.insert(path);
- auto st = lstat(path);
+ uint64_t bytesFreed;
+ deletePath(realPath, bytesFreed);
+ results.bytesFreed += bytesFreed;
- if (st.st_nlink != 1) {
- actualSize += st.st_size;
- unsharedSize += (st.st_nlink - 1) * st.st_size;
- continue;
+ if (results.bytesFreed > options.maxFreed) {
+ printInfo("deleted more than %d bytes; stopping", options.maxFreed);
+ throw GCLimitReached();
}
+ };
- printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
-
- if (unlink(path.c_str()) == -1)
- throw SysError("deleting '%1%'", path);
+ std::map<StorePath, StorePathSet> referrersCache;
- state.results.bytesFreed += st.st_size;
- }
+ /* Helper function that visits all paths reachable from `start`
+ via the referrers edges and optionally derivers and derivation
+ output edges. If none of those paths are roots, then all
+ visited paths are garbage and are deleted. */
+ auto deleteReferrersClosure = [&](const StorePath & start) {
+ StorePathSet visited;
+ std::queue<StorePath> todo;
- struct stat st;
- if (stat(linksDir.c_str(), &st) == -1)
- throw SysError("statting '%1%'", linksDir);
- int64_t overhead = st.st_blocks * 512ULL;
+ /* Wake up any GC client waiting for deletion of the paths in
+ 'visited' to finish. */
+ Finally releasePending([&]() {
+ auto shared(_shared.lock());
+ shared->pending.reset();
+ wakeup.notify_all();
+ });
- printInfo("note: currently hard linking saves %.2f MiB",
- ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
-}
+ auto enqueue = [&](const StorePath & path) {
+ if (visited.insert(path).second)
+ todo.push(path);
+ };
+ enqueue(start);
-void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
-{
- GCState state(options, results);
- state.gcKeepOutputs = settings.gcKeepOutputs;
- state.gcKeepDerivations = settings.gcKeepDerivations;
+ while (auto path = pop(todo)) {
+ checkInterrupt();
- /* Using `--ignore-liveness' with `--delete' can have unintended
- consequences if `keep-outputs' or `keep-derivations' are true
- (the garbage collector will recurse into deleting the outputs
- or derivers, respectively). So disable them. */
- if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) {
- state.gcKeepOutputs = false;
- state.gcKeepDerivations = false;
- }
+ /* Bail out if we've previously discovered that this path
+ is alive. */
+ if (alive.count(*path)) {
+ alive.insert(start);
+ return;
+ }
- state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific;
+ /* If we've previously deleted this path, we don't have to
+ handle it again. */
+ if (dead.count(*path)) continue;
- if (state.shouldDelete)
- deletePath(reservedPath);
+ auto markAlive = [&]()
+ {
+ alive.insert(*path);
+ alive.insert(start);
+ try {
+ StorePathSet closure;
+ computeFSClosure(*path, closure);
+ for (auto & p : closure)
+ alive.insert(p);
+ } catch (InvalidPath &) { }
+ };
+
+ /* If this is a root, bail out. */
+ if (roots.count(*path)) {
+ debug("cannot delete '%s' because it's a root", printStorePath(*path));
+ return markAlive();
+ }
- /* Acquire the global GC root. This prevents
- a) New roots from being added.
- b) Processes from creating new temporary root files. */
- AutoCloseFD fdGCLock = openGCLock(ltWrite);
+ if (options.action == GCOptions::gcDeleteSpecific
+ && !options.pathsToDelete.count(*path))
+ return;
- /* Find the roots. Since we've grabbed the GC lock, the set of
- permanent roots cannot increase now. */
- printInfo("finding garbage collector roots...");
- Roots rootMap;
- if (!options.ignoreLiveness)
- findRootsNoTemp(rootMap, true);
+ {
+ auto hashPart = std::string(path->hashPart());
+ auto shared(_shared.lock());
+ if (shared->tempRoots.count(hashPart)) {
+ debug("cannot delete '%s' because it's a temporary root", printStorePath(*path));
+ return markAlive();
+ }
+ shared->pending = hashPart;
+ }
- for (auto & i : rootMap) state.roots.insert(i.first);
+ if (isValidPath(*path)) {
- /* Read the temporary roots. This acquires read locks on all
- per-process temporary root files. So after this point no paths
- can be added to the set of temporary roots. */
- FDs fds;
- Roots tempRoots;
- findTempRoots(fds, tempRoots, true);
- for (auto & root : tempRoots) {
- state.tempRoots.insert(root.first);
- state.roots.insert(root.first);
- }
+ /* Visit the referrers of this path. */
+ auto i = referrersCache.find(*path);
+ if (i == referrersCache.end()) {
+ StorePathSet referrers;
+ queryReferrers(*path, referrers);
+ referrersCache.emplace(*path, std::move(referrers));
+ i = referrersCache.find(*path);
+ }
+ for (auto & p : i->second)
+ enqueue(p);
+
+ /* If keep-derivations is set and this is a
+ derivation, then visit the derivation outputs. */
+ if (gcKeepDerivations && path->isDerivation()) {
+ for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path))
+ if (maybeOutPath &&
+ isValidPath(*maybeOutPath) &&
+ queryPathInfo(*maybeOutPath)->deriver == *path)
+ enqueue(*maybeOutPath);
+ }
- /* After this point the set of roots or temporary roots cannot
- increase, since we hold locks on everything. So everything
- that is not reachable from `roots' is garbage. */
+ /* If keep-outputs is set, then visit the derivers. */
+ if (gcKeepOutputs) {
+ auto derivers = queryValidDerivers(*path);
+ for (auto & i : derivers)
+ enqueue(i);
+ }
+ }
+ }
- if (state.shouldDelete) {
- if (pathExists(trashDir)) deleteGarbage(state, trashDir);
- try {
- createDirs(trashDir);
- } catch (SysError & e) {
- if (e.errNo == ENOSPC) {
- printInfo("note: can't create trash directory: %s", e.msg());
- state.moveToTrash = false;
+ for (auto & path : topoSortPaths(visited)) {
+ if (!dead.insert(path).second) continue;
+ if (shouldDelete) {
+ invalidatePathChecked(path);
+ deleteFromStore(path.to_string());
+ referrersCache.erase(path);
}
}
- }
+ };
- /* Now either delete all garbage paths, or just the specified
- paths (for gcDeleteSpecific). */
+ /* Synchronisation point for testing, see tests/gc-concurrent.sh. */
+ if (auto p = getEnv("_NIX_TEST_GC_SYNC"))
+ readFile(*p);
+ /* Either delete all garbage paths, or just the specified
+ paths (for gcDeleteSpecific). */
if (options.action == GCOptions::gcDeleteSpecific) {
for (auto & i : options.pathsToDelete) {
- tryToDelete(state, printStorePath(i));
- if (state.dead.find(i) == state.dead.end())
+ deleteReferrersClosure(i);
+ if (!dead.count(i))
throw Error(
- "cannot delete path '%1%' since it is still alive. "
- "To find out why use: "
+ "Cannot delete path '%1%' since it is still alive. "
+ "To find out why, use: "
"nix-store --query --roots",
printStorePath(i));
}
} else if (options.maxFreed > 0) {
- if (state.shouldDelete)
+ if (shouldDelete)
printInfo("deleting garbage...");
else
printInfo("determining live/dead paths...");
try {
-
AutoCloseDir dir(opendir(realStoreDir.get().c_str()));
if (!dir) throw SysError("opening directory '%1%'", realStoreDir);
- /* Read the store and immediately delete all paths that
- aren't valid. When using --max-freed etc., deleting
- invalid paths is preferred over deleting unreachable
- paths, since unreachable paths could become reachable
- again. We don't use readDirectory() here so that GCing
- can start faster. */
+ /* Read the store and delete all paths that are invalid or
+ unreachable. We don't use readDirectory() here so that
+ GCing can start faster. */
+ auto linksName = baseNameOf(linksDir);
Paths entries;
struct dirent * dirent;
while (errno = 0, dirent = readdir(dir.get())) {
checkInterrupt();
string name = dirent->d_name;
- if (name == "." || name == "..") continue;
- Path path = storeDir + "/" + name;
- auto storePath = maybeParseStorePath(path);
- if (storePath && isValidPath(*storePath))
- entries.push_back(path);
- else
- tryToDelete(state, path);
- }
-
- dir.reset();
-
- /* Now delete the unreachable valid paths. Randomise the
- order in which we delete entries to make the collector
- less biased towards deleting paths that come
- alphabetically first (e.g. /nix/store/000...). This
- matters when using --max-freed etc. */
- vector<Path> entries_(entries.begin(), entries.end());
- std::mt19937 gen(1);
- std::shuffle(entries_.begin(), entries_.end(), gen);
+ if (name == "." || name == ".." || name == linksName) continue;
- for (auto & i : entries_)
- tryToDelete(state, i);
+ if (auto storePath = maybeParseStorePath(storeDir + "/" + name))
+ deleteReferrersClosure(*storePath);
+ else
+ deleteFromStore(name);
+ }
} catch (GCLimitReached & e) {
}
}
- if (state.options.action == GCOptions::gcReturnLive) {
- for (auto & i : state.alive)
- state.results.paths.insert(printStorePath(i));
+ if (options.action == GCOptions::gcReturnLive) {
+ for (auto & i : alive)
+ results.paths.insert(printStorePath(i));
return;
}
- if (state.options.action == GCOptions::gcReturnDead) {
- for (auto & i : state.dead)
- state.results.paths.insert(printStorePath(i));
+ if (options.action == GCOptions::gcReturnDead) {
+ for (auto & i : dead)
+ results.paths.insert(printStorePath(i));
return;
}
- /* Allow other processes to add to the store from here on. */
- fdGCLock = -1;
- fds.clear();
-
- /* Delete the trash directory. */
- printInfo(format("deleting '%1%'") % trashDir);
- deleteGarbage(state, trashDir);
-
- /* Clean up the links directory. */
+ /* Unlink all files in /nix/store/.links that have a link count of 1,
+ which indicates that there are no other links and so they can be
+ safely deleted. FIXME: race condition with optimisePath(): we
+ might see a link count of 1 just before optimisePath() increases
+ the link count. */
if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) {
printInfo("deleting unused links...");
- removeUnusedLinks(state);
+
+ AutoCloseDir dir(opendir(linksDir.c_str()));
+ if (!dir) throw SysError("opening directory '%1%'", linksDir);
+
+ int64_t actualSize = 0, unsharedSize = 0;
+
+ struct dirent * dirent;
+ while (errno = 0, dirent = readdir(dir.get())) {
+ checkInterrupt();
+ string name = dirent->d_name;
+ if (name == "." || name == "..") continue;
+ Path path = linksDir + "/" + name;
+
+ auto st = lstat(path);
+
+ if (st.st_nlink != 1) {
+ actualSize += st.st_size;
+ unsharedSize += (st.st_nlink - 1) * st.st_size;
+ continue;
+ }
+
+ printMsg(lvlTalkative, format("deleting unused link '%1%'") % path);
+
+ if (unlink(path.c_str()) == -1)
+ throw SysError("deleting '%1%'", path);
+
+ results.bytesFreed += st.st_size;
+ }
+
+ struct stat st;
+ if (stat(linksDir.c_str(), &st) == -1)
+ throw SysError("statting '%1%'", linksDir);
+ int64_t overhead = st.st_blocks * 512ULL;
+
+ printInfo("note: currently hard linking saves %.2f MiB",
+ ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
}
/* While we're at it, vacuum the database. */
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 4903d0922..9f1a88130 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -160,21 +160,16 @@ StringSet Settings::getDefaultExtraPlatforms()
return extraPlatforms;
}
-bool Settings::isExperimentalFeatureEnabled(const std::string & name)
+bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature)
{
auto & f = experimentalFeatures.get();
- return std::find(f.begin(), f.end(), name) != f.end();
+ return std::find(f.begin(), f.end(), feature) != f.end();
}
-MissingExperimentalFeature::MissingExperimentalFeature(std::string feature)
- : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", feature)
- , missingFeature(feature)
- {}
-
-void Settings::requireExperimentalFeature(const std::string & name)
+void Settings::requireExperimentalFeature(const ExperimentalFeature & feature)
{
- if (!isExperimentalFeatureEnabled(name))
- throw MissingExperimentalFeature(name);
+ if (!isExperimentalFeatureEnabled(feature))
+ throw MissingExperimentalFeature(feature);
}
bool Settings::isWSL1()
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 8784d5faf..165639261 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -3,6 +3,7 @@
#include "types.hh"
#include "config.hh"
#include "util.hh"
+#include "experimental-features.hh"
#include <map>
#include <limits>
@@ -45,15 +46,6 @@ struct PluginFilesSetting : public BaseSetting<Paths>
void set(const std::string & str, bool append = false) override;
};
-class MissingExperimentalFeature: public Error
-{
-public:
- std::string missingFeature;
-
- MissingExperimentalFeature(std::string feature);
- virtual const char* sname() const override { return "MissingExperimentalFeature"; }
-};
-
class Settings : public Config {
unsigned int getDefaultCores();
@@ -925,12 +917,12 @@ public:
value.
)"};
- Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
+ Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features",
"Experimental Nix features to enable."};
- bool isExperimentalFeatureEnabled(const std::string & name);
+ bool isExperimentalFeatureEnabled(const ExperimentalFeature &);
- void requireExperimentalFeature(const std::string & name);
+ void requireExperimentalFeature(const ExperimentalFeature &);
Setting<bool> allowDirty{this, true, "allow-dirty",
"Whether to allow dirty Git/Mercurial trees."};
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 5b2490472..225a19e1e 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -145,7 +145,6 @@ LocalStore::LocalStore(const Params & params)
, linksDir(realStoreDir + "/.links")
, reservedPath(dbDir + "/reserved")
, schemaPath(dbDir + "/schema")
- , trashDir(realStoreDir + "/trash")
, tempRootsDir(stateDir + "/temproots")
, fnTempRoots(fmt("%s/%d", tempRootsDir, getpid()))
, locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
@@ -309,7 +308,7 @@ LocalStore::LocalStore(const Params & params)
else openDB(*state, false);
- if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+ if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
}
@@ -339,7 +338,7 @@ LocalStore::LocalStore(const Params & params)
state->stmts->QueryPathFromHashPart.create(state->db,
"select path from ValidPaths where path >= ? limit 1;");
state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
- if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+ if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
state->stmts->RegisterRealisedOutput.create(state->db,
R"(
insert into Realisations (drvPath, outputName, outputPath, signatures)
@@ -386,6 +385,16 @@ LocalStore::LocalStore(const Params & params)
}
+AutoCloseFD LocalStore::openGCLock()
+{
+ Path fnGCLock = stateDir + "/gc.lock";
+ auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600);
+ if (!fdGCLock)
+ throw SysError("opening global GC lock '%1%'", fnGCLock);
+ return fdGCLock;
+}
+
+
LocalStore::~LocalStore()
{
std::shared_future<void> future;
@@ -708,7 +717,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
{
- settings.requireExperimentalFeature("ca-derivations");
+ settings.requireExperimentalFeature(Xp::CaDerivations);
if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info))
registerDrvOutput(info);
else
@@ -717,7 +726,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check
void LocalStore::registerDrvOutput(const Realisation & info)
{
- settings.requireExperimentalFeature("ca-derivations");
+ settings.requireExperimentalFeature(Xp::CaDerivations);
retrySQLite<void>([&]() {
auto state(_state.lock());
if (auto oldR = queryRealisation_(*state, info.id)) {
@@ -825,7 +834,7 @@ uint64_t LocalStore::addValidPath(State & state,
{
auto state_(Store::state.lock());
- state_->pathInfoCache.upsert(std::string(info.path.hashPart()),
+ state_->pathInfoCache.upsert(std::string(info.path.to_string()),
PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) });
}
@@ -1003,7 +1012,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
return outputs;
});
- if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
+ if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations))
return outputs;
auto drv = readInvalidDerivation(path);
@@ -1198,7 +1207,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
{
auto state_(Store::state.lock());
- state_->pathInfoCache.erase(std::string(path.hashPart()));
+ state_->pathInfoCache.erase(std::string(path.to_string()));
}
}
@@ -1505,7 +1514,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
/* Acquire the global GC lock to get a consistent snapshot of
existing and valid paths. */
- AutoCloseFD fdGCLock = openGCLock(ltWrite);
+ auto fdGCLock = openGCLock();
+ FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock...");
StringSet store;
for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
@@ -1516,8 +1526,6 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
StorePathSet validPaths;
PathSet done;
- fdGCLock = -1;
-
for (auto & i : queryAllValidPaths())
verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index a01d48c4b..301425eb1 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -58,9 +58,15 @@ private:
struct Stmts;
std::unique_ptr<Stmts> stmts;
+ /* The global GC lock */
+ AutoCloseFD fdGCLock;
+
/* The file to which we write our temporary roots. */
AutoCloseFD fdTempRoots;
+ /* Connection to the garbage collector. */
+ AutoCloseFD fdRootsSocket;
+
/* The last time we checked whether to do an auto-GC, or an
auto-GC finished. */
std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;
@@ -87,7 +93,6 @@ public:
const Path linksDir;
const Path reservedPath;
const Path schemaPath;
- const Path trashDir;
const Path tempRootsDir;
const Path fnTempRoots;
@@ -149,14 +154,11 @@ public:
void addIndirectRoot(const Path & path) override;
- void syncWithGC() override;
-
private:
- typedef std::shared_ptr<AutoCloseFD> FDPtr;
- typedef list<FDPtr> FDs;
+ void findTempRoots(Roots & roots, bool censor);
- void findTempRoots(FDs & fds, Roots & roots, bool censor);
+ AutoCloseFD openGCLock();
public:
@@ -236,29 +238,12 @@ private:
PathSet queryValidPathsOld();
ValidPathInfo queryPathInfoOld(const Path & path);
- struct GCState;
-
- void deleteGarbage(GCState & state, const Path & path);
-
- void tryToDelete(GCState & state, const Path & path);
-
- bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path);
-
- void deletePathRecursive(GCState & state, const Path & path);
-
- bool isActiveTempFile(const GCState & state,
- const Path & path, const string & suffix);
-
- AutoCloseFD openGCLock(LockType lockType);
-
void findRoots(const Path & path, unsigned char type, Roots & roots);
void findRootsNoTemp(Roots & roots, bool censor);
void findRuntimeRoots(Roots & roots, bool censor);
- void removeUnusedLinks(const GCState & state);
-
Path createTempDirInStore();
void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv);
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index f184dd857..32786e963 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -239,12 +239,11 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths)
{
return topoSort(paths,
{[&](const StorePath & path) {
- StorePathSet references;
try {
- references = queryPathInfo(path)->references;
+ return queryPathInfo(path)->references;
} catch (InvalidPath &) {
+ return StorePathSet();
}
- return references;
}},
{[&](const StorePath & path, const StorePath & parent) {
return BuildError(
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 926f4ea1e..2da74e262 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -176,4 +176,17 @@ void PathLocks::setDeletion(bool deletePaths)
}
+FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg)
+ : fd(fd)
+{
+ if (wait) {
+ if (!lockFile(fd, lockType, false)) {
+ printInfo("%s", waitMsg);
+ acquired = lockFile(fd, lockType, true);
+ }
+ } else
+ acquired = lockFile(fd, lockType, false);
+}
+
+
}
diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh
index 411da0222..919c8904c 100644
--- a/src/libstore/pathlocks.hh
+++ b/src/libstore/pathlocks.hh
@@ -35,4 +35,18 @@ public:
void setDeletion(bool deletePaths);
};
+struct FdLock
+{
+ int fd;
+ bool acquired = false;
+
+ FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg);
+
+ ~FdLock()
+ {
+ if (acquired)
+ lockFile(fd, ltNone, false);
+ }
+};
+
}
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index fa5ea8af7..7decc059c 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -797,15 +797,6 @@ void RemoteStore::addIndirectRoot(const Path & path)
}
-void RemoteStore::syncWithGC()
-{
- auto conn(getConnection());
- conn->to << wopSyncWithGC;
- conn.processStderr();
- readInt(conn->from);
-}
-
-
Roots RemoteStore::findRoots(bool censor)
{
auto conn(getConnection());
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index ac1eaa19e..a3036e6b0 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -101,8 +101,6 @@ public:
void addIndirectRoot(const Path & path) override;
- void syncWithGC() override;
-
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions & options, GCResults & results) override;
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index b5ff3dccf..b0d3688ce 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -355,7 +355,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
StringSet StoreConfig::getDefaultSystemFeatures()
{
auto res = settings.systemFeatures.get();
- if (settings.isExperimentalFeatureEnabled("ca-derivations"))
+ if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations))
res.insert("ca-derivations");
return res;
}
@@ -414,11 +414,9 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path)
bool Store::isValidPath(const StorePath & storePath)
{
- std::string hashPart(storePath.hashPart());
-
{
auto state_(state.lock());
- auto res = state_->pathInfoCache.get(hashPart);
+ auto res = state_->pathInfoCache.get(std::string(storePath.to_string()));
if (res && res->isKnownNow()) {
stats.narInfoReadAverted++;
return res->didExist();
@@ -426,11 +424,11 @@ bool Store::isValidPath(const StorePath & storePath)
}
if (diskCache) {
- auto res = diskCache->lookupNarInfo(getUri(), hashPart);
+ auto res = diskCache->lookupNarInfo(getUri(), std::string(storePath.hashPart()));
if (res.first != NarInfoDiskCache::oUnknown) {
stats.narInfoReadAverted++;
auto state_(state.lock());
- state_->pathInfoCache.upsert(hashPart,
+ state_->pathInfoCache.upsert(std::string(storePath.to_string()),
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second });
return res.first == NarInfoDiskCache::oValid;
}
@@ -440,7 +438,7 @@ bool Store::isValidPath(const StorePath & storePath)
if (diskCache && !valid)
// FIXME: handle valid = true case.
- diskCache->upsertNarInfo(getUri(), hashPart, 0);
+ diskCache->upsertNarInfo(getUri(), std::string(storePath.hashPart()), 0);
return valid;
}
@@ -487,13 +485,11 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual)
void Store::queryPathInfo(const StorePath & storePath,
Callback<ref<const ValidPathInfo>> callback) noexcept
{
- std::string hashPart;
+ auto hashPart = std::string(storePath.hashPart());
try {
- hashPart = storePath.hashPart();
-
{
- auto res = state.lock()->pathInfoCache.get(hashPart);
+ auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string()));
if (res && res->isKnownNow()) {
stats.narInfoReadAverted++;
if (!res->didExist())
@@ -508,7 +504,7 @@ void Store::queryPathInfo(const StorePath & storePath,
stats.narInfoReadAverted++;
{
auto state_(state.lock());
- state_->pathInfoCache.upsert(hashPart,
+ state_->pathInfoCache.upsert(std::string(storePath.to_string()),
res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second });
if (res.first == NarInfoDiskCache::oInvalid ||
!goodStorePath(storePath, res.second->path))
@@ -523,7 +519,7 @@ void Store::queryPathInfo(const StorePath & storePath,
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
queryPathInfoUncached(storePath,
- {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
+ {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
try {
auto info = fut.get();
@@ -533,14 +529,12 @@ void Store::queryPathInfo(const StorePath & storePath,
{
auto state_(state.lock());
- state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info });
+ state_->pathInfoCache.upsert(std::string(storePath.to_string()), PathInfoCacheValue { .value = info });
}
- auto storePath = parseStorePath(storePathS);
-
if (!info || !goodStorePath(storePath, info->path)) {
stats.narInfoMissing++;
- throw InvalidPath("path '%s' is not valid", storePathS);
+ throw InvalidPath("path '%s' is not valid", printStorePath(storePath));
}
(*callbackPtr)(ref<const ValidPathInfo>(info));
@@ -860,7 +854,7 @@ std::map<StorePath, StorePath> copyPaths(
for (auto & path : paths) {
storePaths.insert(path.path());
if (auto realisation = std::get_if<Realisation>(&path.raw)) {
- settings.requireExperimentalFeature("ca-derivations");
+ settings.requireExperimentalFeature(Xp::CaDerivations);
toplevelRealisations.insert(*realisation);
}
}
@@ -892,7 +886,7 @@ std::map<StorePath, StorePath> copyPaths(
// Don't fail if the remote doesn't support CA derivations is it might
// not be within our control to change that, and we might still want
// to at least copy the output paths.
- if (e.missingFeature == "ca-derivations")
+ if (e.missingFeature == Xp::CaDerivations)
ignoreException();
else
throw;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 54471bdf2..7d02340df 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -232,7 +232,6 @@ protected:
struct State
{
- // FIXME: fix key
LRUCache<std::string, PathInfoCacheValue> pathInfoCache;
};
@@ -561,26 +560,6 @@ public:
virtual void addIndirectRoot(const Path & path)
{ unsupported("addIndirectRoot"); }
- /* Acquire the global GC lock, then immediately release it. This
- function must be called after registering a new permanent root,
- but before exiting. Otherwise, it is possible that a running
- garbage collector doesn't see the new root and deletes the
- stuff we've just built. By acquiring the lock briefly, we
- ensure that either:
-
- - The collector is already running, and so we block until the
- collector is finished. The collector will know about our
- *temporary* locks, which should include whatever it is we
- want to register as a permanent lock.
-
- - The collector isn't running, or it's just started but hasn't
- acquired the GC lock yet. In that case we get and release
- the lock right away, then exit. The collector scans the
- permanent root and sees ours.
-
- In either case the permanent root is seen by the collector. */
- virtual void syncWithGC() { };
-
/* Find the roots of the garbage collector. Each root is a pair
(link, storepath) where `link' is the path of the symlink
outside of the Nix store that point to `storePath'. If
diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc
index 02e81b022..5c38323cd 100644
--- a/src/libstore/uds-remote-store.cc
+++ b/src/libstore/uds-remote-store.cc
@@ -56,14 +56,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
auto conn = make_ref<Connection>();
/* Connect to a daemon that does the privileged work for us. */
- conn->fd = socket(PF_UNIX, SOCK_STREAM
- #ifdef SOCK_CLOEXEC
- | SOCK_CLOEXEC
- #endif
- , 0);
- if (!conn->fd)
- throw SysError("cannot create Unix domain socket");
- closeOnExec(conn->fd.get());
+ conn->fd = createUnixDomainSocket();
nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile);
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 20:39:42 +0000