diff options
Diffstat (limited to 'src/libstore')
76 files changed, 2366 insertions, 1472 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 319b06269..9c022f9e9 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -31,7 +31,7 @@ BinaryCacheStore::BinaryCacheStore(const Params & params) StringSink sink; sink << narVersionMagic1; - narMagic = *sink.s; + narMagic = sink.s; } void BinaryCacheStore::init() @@ -68,7 +68,7 @@ void BinaryCacheStore::upsertFile(const std::string & path, } void BinaryCacheStore::getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback) noexcept + Callback<std::optional<std::string>> callback) noexcept { try { callback(getFile(path)); @@ -77,9 +77,9 @@ void BinaryCacheStore::getFile(const std::string & path, void BinaryCacheStore::getFile(const std::string & path, Sink & sink) { - std::promise<std::shared_ptr<std::string>> promise; + std::promise<std::optional<std::string>> promise; getFile(path, - {[&](std::future<std::shared_ptr<std::string>> result) { + {[&](std::future<std::optional<std::string>> result) { try { promise.set_value(result.get()); } catch (...) { @@ -89,15 +89,15 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink) sink(*promise.get_future().get()); } -std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) +std::optional<std::string> BinaryCacheStore::getFile(const std::string & path) { StringSink sink; try { getFile(path, sink); } catch (NoSuchBinaryCacheFile &) { - return nullptr; + return std::nullopt; } - return sink.s; + return std::move(sink.s); } std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) @@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - std::string hashPart(narInfo->path.hashPart()); - { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) }); + state_->pathInfoCache.upsert( + std::string(narInfo->path.to_string()), + PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) }); } if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); + diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo)); } AutoCloseFD openFile(const Path & path) @@ -149,7 +149,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon( { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; - auto compressionSink = makeCompressionSink(compression, teeSinkCompressed); + auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel); TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; TeeSource teeSource { narSource, teeSinkUncompressed }; narAccessor = makeNarAccessor(teeSource); @@ -306,8 +306,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource }}); } -StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) +StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) unsupported("addToStoreFromDump"); @@ -315,13 +315,16 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam ValidPathInfo info { *this, { - .name = name, + .name = std::string { name }, .info = FixedOutputInfo { { .method = method, .hash = nar.first, }, - {}, + { + .references = references, + .hasSelfReference = false, + }, }, }, nar.first, @@ -375,11 +378,11 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); getFile(narInfoFile, - {[=](std::future<std::shared_ptr<std::string>> fut) { + {[=](std::future<std::optional<std::string>> fut) { try { auto data = fut.get(); - if (!data) return (*callbackPtr)(nullptr); + if (!data) return (*callbackPtr)({}); stats.narInfoRead++; @@ -393,8 +396,14 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, }}); } -StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) +StorePath BinaryCacheStore::addToStore( + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashType hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default @@ -415,13 +424,16 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath ValidPathInfo info { *this, { - .name = name, + .name = std::string { name }, .info = FixedOutputInfo { { .method = method, .hash = h, }, - {}, + { + .references = references, + .hasSelfReference = false, + }, }, }, nar.first, @@ -431,8 +443,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath })->path; } -StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) +StorePath BinaryCacheStore::addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) { auto textHash = hashString(htSHA256, s); auto path = makeTextPath(name, TextInfo { textHash, references }); @@ -442,12 +457,12 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s StringSink sink; dumpString(s, sink); - auto source = StringSource { *sink.s }; + StringSource source(sink.s); return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { *this, { - .name = name, + .name = std::string { name }, .info = TextInfo { { .hash = textHash }, references, @@ -456,45 +471,33 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s nar.first, }; info.narSize = nar.second; - info.ca = TextHash { textHash }; return info; })->path; } -std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id) +void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { - if (diskCache) { - auto [cacheOutcome, maybeCachedRealisation] = - diskCache->lookupRealisation(getUri(), id); - switch (cacheOutcome) { - case NarInfoDiskCache::oValid: - debug("Returning a cached realisation for %s", id.to_string()); - return *maybeCachedRealisation; - case NarInfoDiskCache::oInvalid: - debug("Returning a cached missing realisation for %s", id.to_string()); - return {}; - case NarInfoDiskCache::oUnknown: - break; - } - } - auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; - auto rawOutputInfo = getFile(outputInfoFilePath); - if (rawOutputInfo) { - auto realisation = Realisation::fromJSON( - nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath); + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + + Callback<std::optional<std::string>> newCallback = { + [=](std::future<std::optional<std::string>> fut) { + try { + auto data = fut.get(); + if (!data) return (*callbackPtr)({}); - if (diskCache) - diskCache->upsertRealisation( - getUri(), realisation); + auto realisation = Realisation::fromJSON( + nlohmann::json::parse(*data), outputInfoFilePath); + return (*callbackPtr)(std::make_shared<const Realisation>(realisation)); + } catch (...) { + callbackPtr->rethrow(); + } + } + }; - return {realisation}; - } else { - if (diskCache) - diskCache->upsertAbsentRealisation(getUri(), id); - return std::nullopt; - } + getFile(outputInfoFilePath, std::move(newCallback)); } void BinaryCacheStore::registerDrvOutput(const Realisation& info) { @@ -523,7 +526,7 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe writeNarInfo(narInfo); } -std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) +std::optional<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) { auto drvPath = path; @@ -531,10 +534,10 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat try { auto info = queryPathInfo(path); // FIXME: add a "Log" field to .narinfo - if (!info->deriver) return nullptr; + if (!info->deriver) return std::nullopt; drvPath = *info->deriver; } catch (InvalidPath &) { - return nullptr; + return std::nullopt; } } @@ -545,4 +548,14 @@ std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & pat return getFile(logPath); } +void BinaryCacheStore::addBuildLog(const StorePath & drvPath, std::string_view log) +{ + assert(drvPath.isDerivation()); + + upsertFile( + "log/" + std::string(drvPath.to_string()), + (std::string) log, // FIXME: don't copy + "text/plain; charset=utf-8"); +} + } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 657be2fcf..9603a8caa 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -15,13 +15,17 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting<std::string> compression{(StoreConfig*) this, "xz", "compression", "NAR compression method ('xz', 'bzip2', 'gzip', 'zstd', or 'none')"}; const Setting<bool> writeNARListing{(StoreConfig*) this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; const Setting<bool> writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", "whether to index DWARF debug info files by build ID"}; const Setting<Path> secretKeyFile{(StoreConfig*) this, "", "secret-key", "path to secret key used to sign the binary cache"}; const Setting<Path> localNarCache{(StoreConfig*) this, "", "local-nar-cache", "path to a local cache of NARs"}; const Setting<bool> parallelCompression{(StoreConfig*) this, false, "parallel-compression", - "enable multi-threading compression, available for xz only currently"}; + "enable multi-threading compression for NARs, available for xz and zstd only currently"}; + const Setting<int> compressionLevel{(StoreConfig*) this, -1, "compression-level", + "specify 'preset level' of compression to be used with NARs: " + "meaning and accepted range of values depends on compression method selected, " + "other than -1 which we reserve to indicate Nix defaults should be used"}; }; class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store @@ -47,6 +51,7 @@ public: const std::string & mimeType) = 0; void upsertFile(const std::string & path, + // FIXME: use std::string_view std::string && data, const std::string & mimeType); @@ -58,10 +63,11 @@ public: /* Fetch the specified file and call the specified callback with the result. A subclass may implement this asynchronously. */ - virtual void getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback) noexcept; + virtual void getFile( + const std::string & path, + Callback<std::optional<std::string>> callback) noexcept; - std::shared_ptr<std::string> getFile(const std::string & path); + std::optional<std::string> getFile(const std::string & path); public: @@ -92,19 +98,28 @@ public: void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + StorePath addToStoreFromDump(Source & dump, std::string_view name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; + StorePath addToStore( + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashType hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override; - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) override; + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override; void registerDrvOutput(const Realisation & info) override; - std::optional<const Realisation> queryRealisation(const DrvOutput &) override; + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; void narFromPath(const StorePath & path, Sink & sink) override; @@ -112,7 +127,9 @@ public: void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - std::shared_ptr<std::string> getBuildLog(const StorePath & path) override; + std::optional<std::string> getBuildLog(const StorePath & path) override; + + void addBuildLog(const StorePath & drvPath, std::string_view log) override; }; diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh new file mode 100644 index 000000000..cb6d19b8e --- /dev/null +++ b/src/libstore/build-result.hh @@ -0,0 +1,89 @@ +#pragma once + +#include "realisation.hh" + +#include <string> +#include <chrono> + + +namespace nix { + +struct BuildResult +{ + /* Note: don't remove status codes, and only add new status codes + at the end of the list, to prevent client/server + incompatibilities in the nix-store --serve protocol. */ + enum Status { + Built = 0, + Substituted, + AlreadyValid, + PermanentFailure, + InputRejected, + OutputRejected, + TransientFailure, // possibly transient + CachedFailure, // no longer used + TimedOut, + MiscFailure, + DependencyFailed, + LogLimitExceeded, + NotDeterministic, + ResolvesToAlreadyValid, + NoSubstituters, + } status = MiscFailure; + std::string errorMsg; + + std::string toString() const { + auto strStatus = [&]() { + switch (status) { + case Built: return "Built"; + case Substituted: return "Substituted"; + case AlreadyValid: return "AlreadyValid"; + case PermanentFailure: return "PermanentFailure"; + case InputRejected: return "InputRejected"; + case OutputRejected: return "OutputRejected"; + case TransientFailure: return "TransientFailure"; + case CachedFailure: return "CachedFailure"; + case TimedOut: return "TimedOut"; + case MiscFailure: return "MiscFailure"; + case DependencyFailed: return "DependencyFailed"; + case LogLimitExceeded: return "LogLimitExceeded"; + case NotDeterministic: return "NotDeterministic"; + case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + default: return "Unknown"; + }; + }(); + return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); + } + + /* How many times this build was performed. */ + unsigned int timesBuilt = 0; + + /* If timesBuilt > 1, whether some builds did not produce the same + result. (Note that 'isNonDeterministic = false' does not mean + the build is deterministic, just that we don't have evidence of + non-determinism.) */ + bool isNonDeterministic = false; + + /* The derivation we built or the store path we substituted. */ + DerivedPath path; + + /* For derivations, a mapping from the names of the wanted outputs + to actual paths. */ + DrvOutputs builtOutputs; + + /* The start/stop times of the build (or one of the rounds, if it + was repeated). */ + time_t startTime = 0, stopTime = 0; + + bool success() + { + return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; + } + + void rethrow() + { + throw Error("%s", errorMsg); + } +}; + +} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 0907120db..afed9bf16 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -17,6 +17,7 @@ #include <regex> #include <queue> +#include <fstream> #include <sys/types.h> #include <sys/socket.h> #include <sys/un.h> @@ -65,7 +66,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) + : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -84,7 +85,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker) + : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -115,7 +116,7 @@ DerivationGoal::~DerivationGoal() } -string DerivationGoal::key() +std::string DerivationGoal::key() { /* Ensure that derivations get built in order of their name, i.e. a derivation named "aardvark" always comes before @@ -134,7 +135,7 @@ void DerivationGoal::killChild() void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, ex); + done(BuildResult::TimedOut, {}, ex); } @@ -181,7 +182,7 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); + done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); return; } @@ -193,7 +194,7 @@ void DerivationGoal::loadDerivation() assert(worker.evalStore.isValidPath(drvPath)); /* Get the derivation. */ - drv = std::make_unique<Derivation>(worker.evalStore.derivationFromPath(drvPath)); + drv = std::make_unique<Derivation>(worker.evalStore.readDerivation(drvPath)); haveDerivation(); } @@ -204,7 +205,7 @@ void DerivationGoal::haveDerivation() trace("have derivation"); if (drv->type() == DerivationType::CAFloating) - settings.requireExperimentalFeature("ca-derivations"); + settings.requireExperimentalFeature(Xp::CaDerivations); retrySubstitution = false; @@ -214,28 +215,20 @@ void DerivationGoal::haveDerivation() auto outputHashes = staticOutputHashes(worker.evalStore, *drv); for (auto & [outputName, outputHash] : outputHashes) - initialOutputs.insert({ + initialOutputs.insert({ outputName, - InitialOutput{ + InitialOutput { .wanted = true, // Will be refined later .outputHash = outputHash } - }); + }); /* Check what outputs paths are not already valid. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } + auto [allValid, validOutputs] = checkPathValidity(); /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid); + done(BuildResult::AlreadyValid, std::move(validOutputs)); return; } @@ -276,8 +269,8 @@ void DerivationGoal::outputsSubstitutionTried() trace("all outputs substituted (maybe)"); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, - fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", + done(BuildResult::TransientFailure, {}, + Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", worker.store.printStorePath(drvPath))); return; } @@ -300,23 +293,17 @@ void DerivationGoal::outputsSubstitutionTried() return; } - checkPathValidity(); - size_t nrInvalid = 0; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) - nrInvalid++; - } + auto [allValid, validOutputs] = checkPathValidity(); - if (buildMode == bmNormal && nrInvalid == 0) { - done(BuildResult::Substituted); + if (buildMode == bmNormal && allValid) { + done(BuildResult::Substituted, std::move(validOutputs)); return; } - if (buildMode == bmRepair && nrInvalid == 0) { + if (buildMode == bmRepair && allValid) { repairClosure(); return; } - if (buildMode == bmCheck && nrInvalid > 0) + if (buildMode == bmCheck && !allValid) throw Error("some outputs of '%s' are not valid, so checking is not possible", worker.store.printStorePath(drvPath)); @@ -408,7 +395,7 @@ void DerivationGoal::repairClosure() } if (waitees.empty()) { - done(BuildResult::AlreadyValid); + done(BuildResult::AlreadyValid, assertPathValidity()); return; } @@ -422,7 +409,7 @@ void DerivationGoal::closureRepaired() if (nrFailed > 0) throw Error("some paths in the output closure of derivation '%s' could not be repaired", worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid); + done(BuildResult::AlreadyValid, assertPathValidity()); } @@ -433,7 +420,7 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, Error( + done(BuildResult::DependencyFailed, {}, Error( "%s dependencies of derivation '%s' failed to build", nrFailed, worker.store.printStorePath(drvPath))); return; @@ -453,7 +440,7 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast<Derivation *>(drv.get()); - if (settings.isExperimentalFeatureEnabled("ca-derivations") && + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) || fullDrv.type() == DerivationType::DeferredInputAddressed)) { /* We are be able to resolve this derivation based on the @@ -464,7 +451,6 @@ void DerivationGoal::inputsRealised() Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); - resolvedDrv = drvResolved; auto msg = fmt("Resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), @@ -475,9 +461,9 @@ void DerivationGoal::inputsRealised() worker.store.printStorePath(pathResolved), }); - auto resolvedGoal = worker.makeDerivationGoal( + resolvedDrvGoal = worker.makeDerivationGoal( pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); + addWaitee(resolvedDrvGoal); state = &DerivationGoal::resolvedFinished; return; @@ -523,10 +509,11 @@ void DerivationGoal::inputsRealised() state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); - result = BuildResult(); + buildResult = BuildResult { .path = buildResult.path }; } -void DerivationGoal::started() { +void DerivationGoal::started() +{ auto msg = fmt( buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmCheck ? "checking outputs of '%s'" : @@ -588,19 +575,12 @@ void DerivationGoal::tryToBuild() omitted, but that would be less efficient.) Note that since we now hold the locks on the output paths, no other process can build this derivation, so no further checks are necessary. */ - checkPathValidity(); - bool allValid = true; - for (auto & [_, status] : initialOutputs) { - if (!status.wanted) continue; - if (!status.known || !status.known->isValid()) { - allValid = false; - break; - } - } + auto [allValid, validOutputs] = checkPathValidity(); + if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid); + done(BuildResult::AlreadyValid, std::move(validOutputs)); return; } @@ -616,7 +596,9 @@ void DerivationGoal::tryToBuild() /* Don't do a remote build if the derivation has the attribute `preferLocalBuild' set. Also, check and repair modes are only supported for local builds. */ - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); + bool buildLocally = + (buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store)) + && settings.maxBuildJobs.get() != 0; if (!buildLocally) { switch (tryBuildHook()) { @@ -624,7 +606,7 @@ void DerivationGoal::tryToBuild() /* Yes, it has started doing so. Wait until we get EOF from the hook. */ actLock.reset(); - result.startTime = time(0); // inexact + buildResult.startTime = time(0); // inexact state = &DerivationGoal::buildDone; started(); return; @@ -653,7 +635,7 @@ void DerivationGoal::tryLocalBuild() { throw Error( "unable to build with a primary store that isn't a local store; " "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); } @@ -828,8 +810,8 @@ void DerivationGoal::buildDone() debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); - result.timesBuilt++; - result.stopTime = time(0); + buildResult.timesBuilt++; + buildResult.stopTime = time(0); /* So the child is gone now. */ worker.childTerminated(this); @@ -874,11 +856,11 @@ void DerivationGoal::buildDone() /* Compute the FS closure of the outputs and register them as being valid. */ - registerOutputs(); + auto builtOutputs = registerOutputs(); StorePathSet outputPaths; - for (auto & [_, path] : finalOutputs) - outputPaths.insert(path); + for (auto & [_, output] : buildResult.builtOutputs) + outputPaths.insert(output.outPath); runPostBuildHook( worker.store, *logger, @@ -888,7 +870,7 @@ void DerivationGoal::buildDone() if (buildMode == bmCheck) { cleanupPostOutputsRegisteredModeCheck(); - done(BuildResult::Built); + done(BuildResult::Built, std::move(builtOutputs)); return; } @@ -909,6 +891,8 @@ void DerivationGoal::buildDone() outputLocks.setDeletion(true); outputLocks.unlock(); + done(BuildResult::Built, std::move(builtOutputs)); + } catch (BuildError & e) { outputLocks.unlock(); @@ -928,30 +912,32 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, e); + done(st, {}, e); return; } - - done(BuildResult::Built); } -void DerivationGoal::resolvedFinished() { - assert(resolvedDrv); +void DerivationGoal::resolvedFinished() +{ + assert(resolvedDrvGoal); + auto resolvedDrv = *resolvedDrvGoal->drv; - auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); + auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); StorePathSet outputPaths; // `wantedOutputs` might be empty, which means “all the outputs” auto realWantedOutputs = wantedOutputs; if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv->outputNames(); + realWantedOutputs = resolvedDrv.outputNames(); + + DrvOutputs builtOutputs; for (auto & wantedOutput : realWantedOutputs) { assert(initialOutputs.count(wantedOutput) != 0); assert(resolvedHashes.count(wantedOutput) != 0); auto realisation = worker.store.queryRealisation( - DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} + DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} ); // We've just built it, but maybe the build failed, in which case the // realisation won't be there @@ -963,10 +949,11 @@ void DerivationGoal::resolvedFinished() { signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); outputPaths.insert(realisation->outPath); + builtOutputs.emplace(realisation->id, *realisation); } else { // If we don't have a realisation, then it must mean that something // failed when building the resolved drv - assert(!result.success()); + assert(!buildResult.success()); } } @@ -977,9 +964,17 @@ void DerivationGoal::resolvedFinished() { outputPaths ); - // This is potentially a bit fishy in terms of error reporting. Not sure - // how to do it in a cleaner way - amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); + auto status = [&]() { + auto & resolvedResult = resolvedDrvGoal->buildResult; + switch (resolvedResult.status) { + case BuildResult::AlreadyValid: + return BuildResult::ResolvesToAlreadyValid; + default: + return resolvedResult.status; + } + }(); + + done(status, std::move(builtOutputs)); } HookReply DerivationGoal::tryBuildHook() @@ -1002,7 +997,7 @@ HookReply DerivationGoal::tryBuildHook() /* Read the first line of input, which should be a word indicating whether the hook wishes to perform the build. */ - string reply; + std::string reply; while (true) { auto s = [&]() { try { @@ -1014,8 +1009,8 @@ HookReply DerivationGoal::tryBuildHook() }(); if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) ; - else if (string(s, 0, 2) == "# ") { - reply = string(s, 2); + else if (s.substr(0, 2) == "# ") { + reply = s.substr(2); break; } else { @@ -1080,7 +1075,7 @@ HookReply DerivationGoal::tryBuildHook() /* Create the log file and pipe. */ Path logFile = openLogFile(); - set<int> fds; + std::set<int> fds; fds.insert(hook->fromHook.readSide.get()); fds.insert(hook->builderOut.readSide.get()); worker.childStarted(shared_from_this(), fds, false, false); @@ -1089,7 +1084,7 @@ HookReply DerivationGoal::tryBuildHook() } -void DerivationGoal::registerOutputs() +DrvOutputs DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -1098,21 +1093,7 @@ void DerivationGoal::registerOutputs() We can only early return when the outputs are known a priori. For floating content-addressed derivations this isn't the case. */ - for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) { - if (!wantOutput(outputName, wantedOutputs)) - continue; - if (!optOutputPath) - throw BuildError( - "output '%s' from derivation '%s' does not have a known output path", - outputName, worker.store.printStorePath(drvPath)); - auto & outputPath = *optOutputPath; - if (!worker.store.isValidPath(outputPath)) - throw BuildError( - "output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid", - outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath)); - - finalOutputs.insert_or_assign(outputName, outputPath); - } + return assertPathValidity(); } Path DerivationGoal::openLogFile() @@ -1129,10 +1110,10 @@ Path DerivationGoal::openLogFile() logDir = localStore->logDir; else logDir = settings.nixLogDir; - Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2)); + Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2)); createDirs(dir); - Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), + Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), settings.compressLog ? ".bz2" : ""); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); @@ -1164,16 +1145,17 @@ bool DerivationGoal::isReadDesc(int fd) return fd == hook->builderOut.readSide.get(); } - -void DerivationGoal::handleChildOutput(int fd, const string & data) +void DerivationGoal::handleChildOutput(int fd, std::string_view data) { - if (isReadDesc(fd)) + // local & `ssh://`-builds are dealt with here. + auto isWrittenToLog = isReadDesc(fd); + if (isWrittenToLog) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { killChild(); done( - BuildResult::LogLimitExceeded, + BuildResult::LogLimitExceeded, {}, Error("%s killed after writing more than %d bytes of log output", getName(), settings.maxLogSize)); return; @@ -1196,7 +1178,16 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) if (hook && fd == hook->fromHook.readSide.get()) { for (auto c : data) if (c == '\n') { - handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); + auto json = parseJSONMessage(currentHookLine); + if (json) { + auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); + // ensure that logs from a builder using `ssh-ng://` as protocol + // are also available to `nix log`. + if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) { + auto f = (*json)["fields"]; + (*logSink)((f.size() > 0 ? f.at(0).get<std::string>() : "") + "\n"); + } + } currentHookLine.clear(); } else currentHookLine += c; @@ -1253,10 +1244,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() } -void DerivationGoal::checkPathValidity() +std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() { bool checkHash = buildMode == bmRepair; auto wantedOutputsLeft = wantedOutputs; + DrvOutputs validOutputs; + for (auto & i : queryPartialDerivationOutputMap()) { InitialOutput & info = initialOutputs.at(i.first); info.wanted = wantOutput(i.first, wantedOutputs); @@ -1273,26 +1266,28 @@ void DerivationGoal::checkPathValidity() : PathStatus::Corrupt, }; } - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first}; + auto drvOutput = DrvOutput{initialOutputs.at(i.first).outputHash, i.first}; + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { if (auto real = worker.store.queryRealisation(drvOutput)) { info.known = { .path = real->outPath, .status = PathStatus::Valid, }; - } else if (info.known && info.known->status == PathStatus::Valid) { - // We know the output because it' a static output of the + } else if (info.known && info.known->isValid()) { + // We know the output because it's a static output of the // derivation, and the output path is valid, but we don't have // its realisation stored (probably because it has been built - // without the `ca-derivations` experimental flag) + // without the `ca-derivations` experimental flag). worker.store.registerDrvOutput( - Realisation{ + Realisation { drvOutput, info.known->path, } ); } } + if (info.wanted && info.known && info.known->isValid()) + validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); } // If we requested all the outputs via the empty set, we are always fine. // If we requested specific elements, the loop above removes all the valid @@ -1301,24 +1296,50 @@ void DerivationGoal::checkPathValidity() throw Error("derivation '%s' does not have wanted outputs %s", worker.store.printStorePath(drvPath), concatStringsSep(", ", quoteStrings(wantedOutputsLeft))); + + bool allValid = true; + for (auto & [_, status] : initialOutputs) { + if (!status.wanted) continue; + if (!status.known || !status.known->isValid()) { + allValid = false; + break; + } + } + + return { allValid, validOutputs }; } -void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex) +DrvOutputs DerivationGoal::assertPathValidity() { - result.status = status; + auto [allValid, validOutputs] = checkPathValidity(); + if (!allValid) + throw Error("some outputs are unexpectedly invalid"); + return validOutputs; +} + + +void DerivationGoal::done( + BuildResult::Status status, + DrvOutputs builtOutputs, + std::optional<Error> ex) +{ + buildResult.status = status; if (ex) - result.errorMsg = ex->what(); - amDone(result.success() ? ecSuccess : ecFailed, ex); - if (result.status == BuildResult::TimedOut) + // FIXME: strip: "error: " + buildResult.errorMsg = ex->what(); + amDone(buildResult.success() ? ecSuccess : ecFailed, ex); + if (buildResult.status == BuildResult::TimedOut) worker.timedOut = true; - if (result.status == BuildResult::PermanentFailure) + if (buildResult.status == BuildResult::PermanentFailure) worker.permanentFailure = true; mcExpectedBuilds.reset(); mcRunningBuilds.reset(); - if (result.success()) { + if (buildResult.success()) { + assert(!builtOutputs.empty()); + buildResult.builtOutputs = std::move(builtOutputs); if (status == BuildResult::Built) worker.doneBuilds++; } else { @@ -1327,6 +1348,13 @@ void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex) } worker.updateProgress(); + + auto traceBuiltOutputsFile = getEnv("_NIX_TRACE_BUILT_OUTPUTS").value_or(""); + if (traceBuiltOutputsFile != "") { + std::fstream fs; + fs.open(traceBuiltOutputsFile, std::fstream::out); + fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; + } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 704b77caf..ea2db89b2 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -50,8 +50,8 @@ struct DerivationGoal : public Goal /* The path of the derivation. */ StorePath drvPath; - /* The path of the corresponding resolved derivation */ - std::optional<BasicDerivation> resolvedDrv; + /* The goal for the corresponding resolved derivation */ + std::shared_ptr<DerivationGoal> resolvedDrvGoal; /* The specific outputs that we need to build. Empty means all of them. */ @@ -104,20 +104,8 @@ struct DerivationGoal : public Goal typedef void (DerivationGoal::*GoalState)(); GoalState state; - /* The final output paths of the build. - - - For input-addressed derivations, always the precomputed paths - - - For content-addressed derivations, calcuated from whatever the hash - ends up being. (Note that fixed outputs derivations that produce the - "wrong" output still install that data under its true content-address.) - */ - OutputPathMap finalOutputs; - BuildMode buildMode; - BuildResult result; - /* The current round, if we're building multiple times. */ size_t curRound = 1; @@ -145,15 +133,13 @@ struct DerivationGoal : public Goal void timedOut(Error && ex) override; - string key() override; + std::string key() override; void work() override; /* Add wanted outputs to an already existing derivation goal. */ void addWantedOutputs(const StringSet & outputs); - BuildResult getResult() { return result; } - /* The states. */ void getDerivation(); void loadDerivation(); @@ -175,7 +161,7 @@ struct DerivationGoal : public Goal /* Check that the derivation outputs all exist and register them as valid. */ - virtual void registerOutputs(); + virtual DrvOutputs registerOutputs(); /* Open a log file and a pipe to it. */ Path openLogFile(); @@ -200,7 +186,7 @@ struct DerivationGoal : public Goal virtual bool isReadDesc(int fd); /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; + void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; void flushLine(); @@ -210,8 +196,17 @@ struct DerivationGoal : public Goal std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(); OutputPathMap queryDerivationOutputMap(); - /* Return the set of (in)valid paths. */ - void checkPathValidity(); + /* Update 'initialOutputs' to determine the current status of the + outputs of the derivation. Also returns a Boolean denoting + whether all outputs are valid and non-corrupt, and a + 'DrvOutputs' structure containing the valid and wanted + outputs. */ + std::pair<bool, DrvOutputs> checkPathValidity(); + + /* Aborts if any output is not valid or corrupt, and otherwise + returns a 'DrvOutputs' structure containing the wanted + outputs. */ + DrvOutputs assertPathValidity(); /* Forcibly kill the child process, if any. */ virtual void killChild(); @@ -222,6 +217,7 @@ struct DerivationGoal : public Goal void done( BuildResult::Status status, + DrvOutputs builtOutputs = {}, std::optional<Error> ex = {}); StorePathSet exportReferences(const StorePathSet & storePaths); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index be270d079..e50292c1e 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -1,11 +1,17 @@ #include "drv-output-substitution-goal.hh" +#include "finally.hh" #include "worker.hh" #include "substitution-goal.hh" +#include "callback.hh" namespace nix { -DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca) - : Goal(worker) +DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( + const DrvOutput & id, + Worker & worker, + RepairFlag repair, + std::optional<ContentAddress> ca) + : Goal(worker, DerivedPath::Opaque { StorePath::dummy }) , id(id) { state = &DrvOutputSubstitutionGoal::init; @@ -30,7 +36,7 @@ void DrvOutputSubstitutionGoal::init() void DrvOutputSubstitutionGoal::tryNext() { - trace("Trying next substituter"); + trace("trying next substituter"); if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal @@ -50,14 +56,42 @@ void DrvOutputSubstitutionGoal::tryNext() return; } - auto sub = subs.front(); + sub = subs.front(); subs.pop_front(); // FIXME: Make async - outputInfo = sub->queryRealisation(id); + // outputInfo = sub->queryRealisation(id); + outPipe.create(); + promise = decltype(promise)(); + + sub->queryRealisation( + id, { [&](std::future<std::shared_ptr<const Realisation>> res) { + try { + Finally updateStats([this]() { outPipe.writeSide.close(); }); + promise.set_value(res.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + } }); + + worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + + state = &DrvOutputSubstitutionGoal::realisationFetched; +} + +void DrvOutputSubstitutionGoal::realisationFetched() +{ + worker.childTerminated(this); + + try { + outputInfo = promise.get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + substituterFailed = true; + } + if (!outputInfo) { - tryNext(); - return; + return tryNext(); } for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { @@ -89,7 +123,7 @@ void DrvOutputSubstitutionGoal::tryNext() void DrvOutputSubstitutionGoal::outPathValid() { assert(outputInfo); - trace("Output path substituted"); + trace("output path substituted"); if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); @@ -107,7 +141,7 @@ void DrvOutputSubstitutionGoal::finished() amDone(ecSuccess); } -string DrvOutputSubstitutionGoal::key() +std::string DrvOutputSubstitutionGoal::key() { /* "a$" ensures substitution goals happen before derivation goals. */ @@ -119,4 +153,10 @@ void DrvOutputSubstitutionGoal::work() (this->*state)(); } +void DrvOutputSubstitutionGoal::handleEOF(int fd) +{ + if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); +} + + } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 63ab53d89..948dbda8f 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -3,6 +3,8 @@ #include "store-api.hh" #include "goal.hh" #include "realisation.hh" +#include <thread> +#include <future> namespace nix { @@ -20,11 +22,18 @@ private: // The realisation corresponding to the given output id. // Will be filled once we can get it. - std::optional<Realisation> outputInfo; + std::shared_ptr<const Realisation> outputInfo; /* The remaining substituters. */ std::list<ref<Store>> subs; + /* The current substituter. */ + std::shared_ptr<Store> sub; + + Pipe outPipe; + std::thread thr; + std::promise<std::shared_ptr<const Realisation>> promise; + /* Whether a substituter failed. */ bool substituterFailed = false; @@ -36,15 +45,16 @@ public: void init(); void tryNext(); + void realisationFetched(); void outPathValid(); void finished(); void timedOut(Error && ex) override { abort(); }; - string key() override; + std::string key() override; void work() override; - + void handleEOF(int fd) override; }; } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 2b77e4354..bea7363db 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,4 +1,3 @@ -#include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" #include "derivation-goal.hh" @@ -48,43 +47,51 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod } } +std::vector<BuildResult> Store::buildPathsWithResults( + const std::vector<DerivedPath> & reqs, + BuildMode buildMode, + std::shared_ptr<Store> evalStore) +{ + Worker worker(*this, evalStore ? *evalStore : *this); + + Goals goals; + for (const auto & br : reqs) { + std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode)); + }, + [&](const DerivedPath::Opaque & bo) { + goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair)); + }, + }, br.raw()); + } + + worker.run(goals); + + std::vector<BuildResult> results; + + for (auto & i : goals) + results.push_back(i->buildResult); + + return results; +} + BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { Worker worker(*this, *this); auto goal = worker.makeBasicDerivationGoal(drvPath, drv, {}, buildMode); - BuildResult result; - try { worker.run(Goals{goal}); - result = goal->getResult(); + return goal->buildResult; } catch (Error & e) { - result.status = BuildResult::MiscFailure; - result.errorMsg = e.msg(); - } - // XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's - // extended to return the full realisation for each output - auto staticDrvOutputs = drv.outputsAndOptPaths(*this); - auto outputHashes = staticOutputHashes(*this, drv); - for (auto & [outputName, staticOutput] : staticDrvOutputs) { - auto outputId = DrvOutput{outputHashes.at(outputName), outputName}; - if (staticOutput.second) - result.builtOutputs.insert_or_assign( - outputId, - Realisation{ outputId, *staticOutput.second} - ); - if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) { - auto realisation = this->queryRealisation(outputId); - if (realisation) - result.builtOutputs.insert_or_assign( - outputId, - *realisation - ); - } - } - - return result; + return BuildResult { + .status = BuildResult::MiscFailure, + .errorMsg = e.msg(), + .path = DerivedPath::Built { .drvPath = drvPath }, + }; + }; } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 7c985128b..d2420b107 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -5,8 +5,8 @@ namespace nix { bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - string s1 = a->key(); - string s2 = b->key(); + std::string s1 = a->key(); + std::string s2 = b->key(); return s1 < s2; } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 192e416d2..07c752bb9 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -2,6 +2,7 @@ #include "types.hh" #include "store-api.hh" +#include "build-result.hh" namespace nix { @@ -18,8 +19,8 @@ struct CompareGoalPtrs { }; /* Set of goals. */ -typedef set<GoalPtr, CompareGoalPtrs> Goals; -typedef set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals; +typedef std::set<GoalPtr, CompareGoalPtrs> Goals; +typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals; /* A map of paths to goals (and the other way around). */ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; @@ -50,15 +51,20 @@ struct Goal : public std::enable_shared_from_this<Goal> unsigned int nrIncompleteClosure; /* Name of this goal for debugging purposes. */ - string name; + std::string name; /* Whether the goal is finished. */ ExitCode exitCode; + /* Build result. */ + BuildResult buildResult; + /* Exception containing an error message, if any. */ std::optional<Error> ex; - Goal(Worker & worker) : worker(worker) + Goal(Worker & worker, DerivedPath path) + : worker(worker) + , buildResult { .path = std::move(path) } { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; exitCode = ecBusy; @@ -75,7 +81,7 @@ struct Goal : public std::enable_shared_from_this<Goal> virtual void waiteeDone(GoalPtr waitee, ExitCode result); - virtual void handleChildOutput(int fd, const string & data) + virtual void handleChildOutput(int fd, std::string_view data) { abort(); } @@ -87,7 +93,7 @@ struct Goal : public std::enable_shared_from_this<Goal> void trace(const FormatOrString & fs); - string getName() + std::string getName() { return name; } @@ -97,7 +103,7 @@ struct Goal : public std::enable_shared_from_this<Goal> by the worker (important!), etc. */ virtual void timedOut(Error && ex) = 0; - virtual string key() = 0; + virtual std::string key() = 0; void amDone(ExitCode result, std::optional<Error> ex = {}); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index e739f4d9d..14b1582f1 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,4 +1,5 @@ #include "local-derivation-goal.hh" +#include "gc-store.hh" #include "hook-instance.hh" #include "worker.hh" #include "builtins.hh" @@ -193,7 +194,7 @@ void LocalDerivationGoal::tryLocalBuild() { outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, e); + done(BuildResult::InputRejected, {}, e); return; } @@ -260,6 +261,7 @@ void LocalDerivationGoal::cleanupHookFinally() void LocalDerivationGoal::cleanupPreChildKill() { sandboxMountNamespace = -1; + sandboxUserNamespace = -1; } @@ -342,7 +344,7 @@ int childEntry(void * arg) return 1; } - +#if __linux__ static void linkOrCopy(const Path & from, const Path & to) { if (link(from.c_str(), to.c_str()) == -1) { @@ -358,6 +360,7 @@ static void linkOrCopy(const Path & from, const Path & to) copyPath(from, to); } } +#endif void LocalDerivationGoal::startBuilder() @@ -479,12 +482,12 @@ void LocalDerivationGoal::startBuilder() temporary build directory. The text files have the format used by `nix-store --register-validity'. However, the deriver fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph").value_or(""); + auto s = get(drv->env, "exportReferencesGraph").value_or(""); Strings ss = tokenizeString<Strings>(s); if (ss.size() % 2 != 0) throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); for (Strings::iterator i = ss.begin(); i != ss.end(); ) { - string fileName = *i++; + auto fileName = *i++; static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); if (!std::regex_match(fileName, regex)) throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); @@ -515,10 +518,10 @@ void LocalDerivationGoal::startBuilder() i.pop_back(); } size_t p = i.find('='); - if (p == string::npos) + if (p == std::string::npos) dirsInChroot[i] = {i, optional}; else - dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; + dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } dirsInChroot[tmpDirInSandbox] = tmpDir; @@ -669,9 +672,10 @@ void LocalDerivationGoal::startBuilder() auto state = stBegin; auto lines = runProgram(settings.preBuildHook, false, args); auto lastPos = std::string::size_type{0}; - for (auto nlPos = lines.find('\n'); nlPos != string::npos; - nlPos = lines.find('\n', lastPos)) { - auto line = std::string{lines, lastPos, nlPos - lastPos}; + for (auto nlPos = lines.find('\n'); nlPos != std::string::npos; + nlPos = lines.find('\n', lastPos)) + { + auto line = lines.substr(lastPos, nlPos - lastPos); lastPos = nlPos + 1; if (state == stBegin) { if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { @@ -684,10 +688,10 @@ void LocalDerivationGoal::startBuilder() state = stBegin; } else { auto p = line.find('='); - if (p == string::npos) + if (p == std::string::npos) dirsInChroot[line] = line; else - dirsInChroot[string(line, 0, p)] = string(line, p + 1); + dirsInChroot[line.substr(0, p)] = line.substr(p + 1); } } } @@ -752,7 +756,7 @@ void LocalDerivationGoal::startBuilder() if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) throw SysError("putting pseudoterminal into raw mode"); - result.startTime = time(0); + buildResult.startTime = time(0); /* Fork a child to build the package. */ @@ -905,19 +909,27 @@ void LocalDerivationGoal::startBuilder() "nobody:x:65534:65534:Nobody:/:/noshell\n", sandboxUid(), sandboxGid(), settings.sandboxBuildDir)); - /* Save the mount namespace of the child. We have to do this + /* Save the mount- and user namespace of the child. We have to do this *before* the child does a chroot. */ sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY); if (sandboxMountNamespace.get() == -1) throw SysError("getting sandbox mount namespace"); + if (usingUserNamespace) { + sandboxUserNamespace = open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY); + if (sandboxUserNamespace.get() == -1) + throw SysError("getting sandbox user namespace"); + } + /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); } else #endif { +#if __linux__ fallback: +#endif pid = startProcess([&]() { runChild(); }); @@ -931,7 +943,7 @@ void LocalDerivationGoal::startBuilder() /* Check if setting up the build environment failed. */ std::vector<std::string> msgs; while (true) { - string msg = [&]() { + std::string msg = [&]() { try { return readLine(builderOut.readSide.get()); } catch (Error & e) { @@ -943,12 +955,12 @@ void LocalDerivationGoal::startBuilder() throw; } }(); - if (string(msg, 0, 1) == "\2") break; - if (string(msg, 0, 1) == "\1") { + if (msg.substr(0, 1) == "\2") break; + if (msg.substr(0, 1) == "\1") { FdSource source(builderOut.readSide.get()); auto ex = readError(source); ex.addTrace({}, "while setting up the build environment"); - throw; + throw ex; } debug("sandbox setup: " + msg); msgs.push_back(std::move(msg)); @@ -980,7 +992,7 @@ void LocalDerivationGoal::initTmpDir() { env[i.first] = i.second; } else { auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); + std::string fn = ".attr-" + hash.to_string(Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -1071,7 +1083,7 @@ void LocalDerivationGoal::writeStructuredAttrs() for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) { /* The placeholder must have a rewrite, so we use it to cover both the cases where we know or don't know the output path ahead of time. */ - rewritten[i] = rewriteStrings(v, inputRewrites); + rewritten[i] = rewriteStrings((std::string) v, inputRewrites); } json["outputs"] = rewritten; @@ -1116,7 +1128,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore { ref<LocalStore> next; @@ -1177,9 +1189,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override + StorePath addToStore( + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashType hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1189,18 +1206,26 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo goal.addDependency(info.path); } - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) override + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair = NoRepair) override { auto path = next->addTextToStore(name, s, references, repair); goal.addDependency(path); return path; } - StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashType hashAlgo, + RepairFlag repair, + const StorePathSet & references) override { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); goal.addDependency(path); return path; } @@ -1224,17 +1249,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // corresponds to an allowed derivation { throw Error("registerDrvOutput"); } - std::optional<const Realisation> queryRealisation(const DrvOutput & id) override + void queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override // XXX: This should probably be allowed if the realisation corresponds to // an allowed derivation { if (!goal.isAllowed(id)) - throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string()); - return next->queryRealisation(id); + callback(nullptr); + next->queryRealisation(id, std::move(callback)); } void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override { + for (auto & result : buildPathsWithResults(paths, buildMode, evalStore)) + if (!result.success()) + result.rethrow(); + } + + std::vector<BuildResult> buildPathsWithResults( + const std::vector<DerivedPath> & paths, + BuildMode buildMode = bmNormal, + std::shared_ptr<Store> evalStore = nullptr) override + { assert(!evalStore); if (buildMode != bmNormal) throw Error("unsupported build mode"); @@ -1247,26 +1283,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next)); } - next->buildPaths(paths, buildMode); - - for (auto & path : paths) { - auto p = std::get_if<DerivedPath::Built>(&path); - if (!p) continue; - auto & bfd = *p; - auto drv = readDerivation(bfd.drvPath); - auto drvHashes = staticOutputHashes(*this, drv); - auto outputs = next->queryDerivationOutputMap(bfd.drvPath); - for (auto & [outputName, outputPath] : outputs) - if (wantOutput(outputName, bfd.outputs)) { - newPaths.insert(outputPath); - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - auto thisRealisation = next->queryRealisation( - DrvOutput{drvHashes.at(outputName), outputName} - ); - assert(thisRealisation); - newRealisations.insert(*thisRealisation); - } - } + auto results = next->buildPathsWithResults(paths, buildMode); + + for (auto & result : results) { + for (auto & [outputName, output] : result.builtOutputs) { + newPaths.insert(output.outPath); + newRealisations.insert(output); + } } StorePathSet closure; @@ -1275,6 +1298,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo goal.addDependency(path); for (auto & real : Realisation::closure(*next, newRealisations)) goal.addedDrvOutputs.insert(real.id); + + return results; } BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, @@ -1320,7 +1345,7 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void LocalDerivationGoal::startDaemon() { - settings.requireExperimentalFeature("recursive-nix"); + settings.requireExperimentalFeature(Xp::RecursiveNix); Store::Params params; params["path-info-cache-size"] = "0"; @@ -1353,7 +1378,7 @@ void LocalDerivationGoal::startDaemon() AutoCloseFD remote = accept(daemonSocket.get(), (struct sockaddr *) &remoteAddr, &remoteAddrLen); if (!remote) { - if (errno == EINTR) continue; + if (errno == EINTR || errno == EAGAIN) continue; if (errno == EINVAL) break; throw SysError("accepting connection"); } @@ -1432,6 +1457,9 @@ void LocalDerivationGoal::addDependency(const StorePath & path) child process.*/ Pid child(startProcess([&]() { + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + throw SysError("entering sandbox user namespace"); + if (setns(sandboxMountNamespace.get(), 0) == -1) throw SysError("entering sandbox mount namespace"); @@ -1773,11 +1801,14 @@ void LocalDerivationGoal::runChild() i686-linux build on an x86_64-linux machine. */ struct utsname utsbuf; uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { + if ((drv->platform == "i686-linux" + && (settings.thisSystem == "x86_64-linux" + || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) + || drv->platform == "armv7l-linux" + || drv->platform == "armv6l-linux") + { if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); + throw SysError("cannot set 32-bit personality"); } /* Impersonate a Linux 2.6 machine to get some determinism in @@ -1902,7 +1933,7 @@ void LocalDerivationGoal::runChild() "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", i.first, i.second.source); - string path = i.first; + std::string path = i.first; struct stat st; if (lstat(path.c_str(), &st)) { if (i.second.optional && errno == ENOENT) @@ -1954,7 +1985,7 @@ void LocalDerivationGoal::runChild() args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); if (allowLocalNetworking) { args.push_back("-D"); - args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); + args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1")); } args.push_back(drv->builder); } else { @@ -1973,7 +2004,7 @@ void LocalDerivationGoal::runChild() args.push_back(rewriteStrings(i, inputRewrites)); /* Indicate that we managed to set up the build environment. */ - writeFull(STDERR_FILENO, string("\2\n")); + writeFull(STDERR_FILENO, std::string("\2\n")); /* Execute the program. This should not return. */ if (drv->isBuiltin()) { @@ -1991,7 +2022,7 @@ void LocalDerivationGoal::runChild() else if (drv->builder == "builtin:unpack-channel") builtinUnpackChannel(drv2); else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); + throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); @@ -2037,7 +2068,7 @@ void LocalDerivationGoal::runChild() } -void LocalDerivationGoal::registerOutputs() +DrvOutputs LocalDerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -2046,10 +2077,8 @@ void LocalDerivationGoal::registerOutputs() We can only early return when the outputs are known a priori. For floating content-addressed derivations this isn't the case. */ - if (hook) { - DerivationGoal::registerOutputs(); - return; - } + if (hook) + return DerivationGoal::registerOutputs(); std::map<std::string, ValidPathInfo> infos; @@ -2172,6 +2201,8 @@ void LocalDerivationGoal::registerOutputs() std::reverse(sortedOutputNames.begin(), sortedOutputNames.end()); + OutputPathMap finalOutputs; + for (auto & outputName : sortedOutputNames) { auto output = drv->outputs.at(outputName); auto & scratchPath = scratchOutputs.at(outputName); @@ -2210,8 +2241,8 @@ void LocalDerivationGoal::registerOutputs() StringSink sink; dumpPath(actualPath, sink); deletePath(actualPath); - sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites)); - StringSource source(*sink.s); + sink.s = rewriteStrings(sink.s, outputRewrites); + StringSource source(sink.s); restorePath(actualPath, source); } }; @@ -2290,7 +2321,7 @@ void LocalDerivationGoal::registerOutputs() StringSink sink; dumpPath(actualPath, sink); RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - rsink2(*sink.s); + rsink2(sink.s); rsink2.flush(); }); Path tmpPath = actualPath + ".tmp"; @@ -2308,6 +2339,7 @@ void LocalDerivationGoal::registerOutputs() }; ValidPathInfo newInfo = std::visit(overloaded { + [&](const DerivationOutputInputAddressed & output) { /* input-addressed case */ auto requiredFinalPath = output.path; @@ -2324,6 +2356,7 @@ void LocalDerivationGoal::registerOutputs() static_cast<PathReferences<StorePath> &>(newInfo0) = rewriteRefs(); return newInfo0; }, + [&](const DerivationOutputCAFixed & dof) { auto wanted = getContentAddressHash(dof.ca); @@ -2352,18 +2385,17 @@ void LocalDerivationGoal::registerOutputs() return newInfo0; }, - [&](DerivationOutputCAFloating dof) { + + [&](DerivationOutputCAFloating & dof) { return newInfoFromCA(dof); }, - [&](DerivationOutputDeferred) { + + [&](DerivationOutputDeferred) -> ValidPathInfo { // No derivation should reach that point without having been // rewritten first assert(false); - // Ugly, but the compiler insists on having this return a value - // of type `ValidPathInfo` despite the `assert(false)`, so - // let's provide it - return *(ValidPathInfo*)0; }, + }, output.output); /* FIXME: set proper permissions in restorePath() so @@ -2454,7 +2486,7 @@ void LocalDerivationGoal::registerOutputs() } if (curRound == nrRounds) { - localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences() + localStore.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() worker.markContentsGood(newInfo.path); } @@ -2474,11 +2506,12 @@ void LocalDerivationGoal::registerOutputs() } if (buildMode == bmCheck) { - // In case of FOD mismatches on `--check` an error must be thrown as this is also - // a source for non-determinism. + /* In case of fixed-output derivations, if there are + mismatches on `--check` an error must be thrown as this is + also a source for non-determinism. */ if (delayedException) std::rethrow_exception(delayedException); - return; + return assertPathValidity(); } /* Apply output checks. */ @@ -2490,7 +2523,7 @@ void LocalDerivationGoal::registerOutputs() assert(prevInfos.size() == infos.size()); for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { - result.isNonDeterministic = true; + buildResult.isNonDeterministic = true; Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); hintformat hint = prevExists @@ -2528,7 +2561,7 @@ void LocalDerivationGoal::registerOutputs() if (curRound < nrRounds) { prevInfos = std::move(infos); - return; + return {}; } /* Remove the .check directories if we're done. FIXME: keep them @@ -2563,17 +2596,24 @@ void LocalDerivationGoal::registerOutputs() means it's safe to link the derivation to the output hash. We must do that for floating CA derivations, which otherwise couldn't be cached, but it's fine to do in all cases. */ + DrvOutputs builtOutputs; - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { - for (auto& [outputName, newInfo] : infos) { - auto thisRealisation = Realisation{ - .id = DrvOutput{initialOutputs.at(outputName).outputHash, - outputName}, - .outPath = newInfo.path}; + for (auto & [outputName, newInfo] : infos) { + auto thisRealisation = Realisation { + .id = DrvOutput { + initialOutputs.at(outputName).outputHash, + outputName + }, + .outPath = newInfo.path + }; + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } + builtOutputs.emplace(thisRealisation.id, thisRealisation); } + + return builtOutputs; } void LocalDerivationGoal::signRealisation(Realisation & realisation) @@ -2582,7 +2622,7 @@ void LocalDerivationGoal::signRealisation(Realisation & realisation) } -void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) +void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo> & outputs) { std::map<Path, const ValidPathInfo &> outputsByPath; for (auto & output : outputs) @@ -2654,8 +2694,8 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out for (auto & i : *value) { if (worker.store.isStorePath(i)) spec.insert(worker.store.parseStorePath(i)); - else if (finalOutputs.count(i)) - spec.insert(finalOutputs.at(i)); + else if (outputs.count(i)) + spec.insert(outputs.at(i).path); else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } @@ -2678,7 +2718,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out } if (!badPaths.empty()) { - string badPathsStr; + std::string badPathsStr; for (auto & i : badPaths) { badPathsStr += "\n "; badPathsStr += worker.store.printStorePath(i); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 088a57209..d456e9cae 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -27,9 +27,10 @@ struct LocalDerivationGoal : public DerivationGoal /* Pipe for synchronising updates to the builder namespaces. */ Pipe userNamespaceSync; - /* The mount namespace of the builder, used to add additional + /* The mount namespace and user namespace of the builder, used to add additional paths to the sandbox as a result of recursive Nix calls. */ AutoCloseFD sandboxMountNamespace; + AutoCloseFD sandboxUserNamespace; /* On Linux, whether we're doing the build in its own user namespace. */ @@ -57,11 +58,11 @@ struct LocalDerivationGoal : public DerivationGoal typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path DirsInChroot dirsInChroot; - typedef map<string, string> Environment; + typedef map<std::string, std::string> Environment; Environment env; #if __APPLE__ - typedef string SandboxProfile; + typedef std::string SandboxProfile; SandboxProfile additionalSandboxProfile; #endif @@ -168,7 +169,7 @@ struct LocalDerivationGoal : public DerivationGoal /* Check that the derivation outputs all exist and register them as valid. */ - void registerOutputs() override; + DrvOutputs registerOutputs() override; void signRealisation(Realisation &) override; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index cd00e41f9..2aaa89a57 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -6,7 +6,7 @@ namespace nix { PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca) - : Goal(worker) + : Goal(worker, DerivedPath::Opaque { storePath }) , storePath(storePath) , repair(repair) , ca(ca) @@ -24,6 +24,13 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } +void PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status) +{ + buildResult.status = status; + amDone(result); +} + + void PathSubstitutionGoal::work() { (this->*state)(); @@ -38,7 +45,7 @@ void PathSubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); + done(ecSuccess, BuildResult::AlreadyValid); return; } @@ -65,7 +72,7 @@ void PathSubstitutionGoal::tryNext() /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); + done(substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters); if (substituterFailed) { worker.failedSubstitutions++; @@ -141,8 +148,8 @@ void PathSubstitutionGoal::tryNext() only after we've downloaded the path. */ if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) { - warn("substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)); + warn("the substitute for '%s' from '%s' is not signed by any of the keys in 'trusted-public-keys'", + worker.store.printStorePath(storePath), sub->getUri()); tryNext(); return; } @@ -165,7 +172,9 @@ void PathSubstitutionGoal::referencesValid() if (nrFailed > 0) { debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); + done( + nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, + BuildResult::DependencyFailed); return; } @@ -269,11 +278,11 @@ void PathSubstitutionGoal::finished() worker.updateProgress(); - amDone(ecSuccess); + done(ecSuccess, BuildResult::Substituted); } -void PathSubstitutionGoal::handleChildOutput(int fd, const string & data) +void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data) { } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 70c806d23..946f13841 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -53,13 +53,15 @@ struct PathSubstitutionGoal : public Goal /* Content address for recomputing store path */ std::optional<ContentAddress> ca; + void done(ExitCode result, BuildResult::Status status); + public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); ~PathSubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; - string key() override + std::string key() override { /* "a$" ensures substitution goals happen before derivation goals. */ @@ -77,7 +79,7 @@ public: void finished(); /* Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, const string & data) override; + void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; void cleanup() override; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 55afb5cca..f72c1cc9c 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -161,7 +161,7 @@ unsigned Worker::getNrLocalBuilds() } -void Worker::childStarted(GoalPtr goal, const set<int> & fds, +void Worker::childStarted(GoalPtr goal, const std::set<int> & fds, bool inBuildSlot, bool respectTimeouts) { Child child; @@ -281,11 +281,11 @@ void Worker::run(const Goals & _topGoals) if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); else throw Error("unable to start any build; remote machines may not have " "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); } assert(!awake.empty()); @@ -377,7 +377,7 @@ void Worker::waitForInput() GoalPtr goal = j->goal.lock(); assert(goal); - set<int> fds2(j->fds); + std::set<int> fds2(j->fds); std::vector<unsigned char> buffer(4096); for (auto & k : fds2) { if (pollStatus.at(fdToPollStatus.at(k)).revents) { @@ -394,7 +394,7 @@ void Worker::waitForInput() } else { printMsg(lvlVomit, "%1%: read %2% bytes", goal->getName(), rd); - string data((char *) buffer.data(), rd); + std::string data((char *) buffer.data(), rd); j->lastOutput = after; goal->handleChildOutput(k, data); } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6a3b99c02..a1e036a96 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -38,7 +38,7 @@ struct Child { WeakGoalPtr goal; Goal * goal2; // ugly hackery - set<int> fds; + std::set<int> fds; bool respectTimeouts; bool inBuildSlot; steady_time_point lastOutput; /* time we last got output on stdout/stderr */ @@ -167,7 +167,7 @@ public: /* Registers a running child process. `inBuildSlot' means that the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const set<int> & fds, + void childStarted(GoalPtr goal, const std::set<int> & fds, bool inBuildSlot, bool respectTimeouts); /* Unregisters a running child process. `wakeSleepers' should be diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index e88fc687a..25d015cb9 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -123,7 +123,7 @@ void buildProfile(const Path & out, Packages && pkgs) createLinks(state, pkgDir, out, priority); try { - for (const auto & p : tokenizeString<std::vector<string>>( + for (const auto & p : tokenizeString<std::vector<std::string>>( readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) if (!done.count(p)) postponed.insert(p); @@ -161,7 +161,7 @@ void buildProfile(const Path & out, Packages && pkgs) void builtinBuildenv(const BasicDerivation & drv) { - auto getAttr = [&](const string & name) { + auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); if (i == drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 4fb5d8a06..af3dfc409 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -16,7 +16,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) writeFile(settings.netrcFile, netrcData, 0600); } - auto getAttr = [&](const string & name) { + auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); if (i == drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index d18e3ddaf..426d58a53 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -5,7 +5,7 @@ namespace nix { void builtinUnpackChannel(const BasicDerivation & drv) { - auto getAttr = [&](const string & name) { + auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); if (i == drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 08af0cc1f..64cc97fde 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -19,3 +19,8 @@ create table if not exists RealisationsRefs ( foreign key (referrer) references Realisations(id) on delete cascade, foreign key (realisationReference) references Realisations(id) on delete restrict ); + +-- used by QueryRealisationReferences +create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); +-- used by cascade deletion when ValidPaths is deleted +create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index e8c6b94be..f8e4e260b 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -144,8 +144,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) { std::pair<ContentAddressMethod, HashType> parseContentAddressMethod(std::string_view caMethod) { - std::string_view asPrefix {std::string{caMethod} + ":"}; - return parseContentAddressMethodPrefix(asPrefix); + std::string asPrefix = std::string{caMethod} + ":"; + // parseContentAddressMethodPrefix takes its argument by reference + std::string_view asPrefixView = asPrefix; + return parseContentAddressMethodPrefix(asPrefixView); } std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 74dd11cbd..4b5a57ff1 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,10 +1,11 @@ #include "daemon.hh" #include "monitor-fd.hh" #include "worker-protocol.hh" +#include "build-result.hh" #include "store-api.hh" +#include "gc-store.hh" #include "path-with-outputs.hh" #include "finally.hh" -#include "affinity.hh" #include "archive.hh" #include "derivations.hh" #include "args.hh" @@ -70,7 +71,7 @@ struct TunnelLogger : public Logger StringSink buf; buf << STDERR_NEXT << (fs.s + "\n"); - enqueueMsg(*buf.s); + enqueueMsg(buf.s); } void logEI(const ErrorInfo & ei) override @@ -82,7 +83,7 @@ struct TunnelLogger : public Logger StringSink buf; buf << STDERR_NEXT << oss.str(); - enqueueMsg(*buf.s); + enqueueMsg(buf.s); } /* startWork() means that we're starting an operation for which we @@ -130,7 +131,7 @@ struct TunnelLogger : public Logger StringSink buf; buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; - enqueueMsg(*buf.s); + enqueueMsg(buf.s); } void stopActivity(ActivityId act) override @@ -138,7 +139,7 @@ struct TunnelLogger : public Logger if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; StringSink buf; buf << STDERR_STOP_ACTIVITY << act; - enqueueMsg(*buf.s); + enqueueMsg(buf.s); } void result(ActivityId act, ResultType type, const Fields & fields) override @@ -146,7 +147,7 @@ struct TunnelLogger : public Logger if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; StringSink buf; buf << STDERR_RESULT << act << type << fields; - enqueueMsg(*buf.s); + enqueueMsg(buf.s); } }; @@ -230,11 +231,12 @@ struct ClientSettings else if (name == settings.experimentalFeatures.name) { // We don’t want to forward the experimental features to // the daemon, as that could cause some pretty weird stuff - if (tokenizeString<Strings>(value) != settings.experimentalFeatures.get()) + if (parseFeatures(tokenizeString<StringSet>(value)) != settings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); } else if (trusted || name == settings.buildTimeout.name + || name == settings.buildRepeat.name || name == "connect-timeout" || (name == "builders" && value == "")) settings.set(name, value); @@ -406,9 +408,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, return store->queryPathInfo(path); }, [&](FileIngestionMethod & fim) { - if (!refs.empty()) - throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); - auto path = store->addToStoreFromDump(source, name, fim, hashType, repair); + auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs); return store->queryPathInfo(path); }, }, contentAddressMethod); @@ -436,25 +436,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store, hashAlgo = parseHashType(hashAlgoRaw); } - StringSink saved; - TeeSource savedNARSource(from, saved); - RetrieveRegularNARSink savedRegular { saved }; - - if (method == FileIngestionMethod::Recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNARSource); - } else - parseDump(savedRegular, from); - + auto dumpSource = sinkToSource([&](Sink & saved) { + if (method == FileIngestionMethod::Recursive) { + /* We parse the NAR dump through into `saved` unmodified, + so why all this extra work? We still parse the NAR so + that we aren't sending arbitrary data to `saved` + unwittingly`, and we know when the NAR ends so we don't + consume the rest of `from` and can't parse another + command. (We don't trust `addToStoreFromDump` to not + eagerly consume the entire stream it's given, past the + length of the Nar. */ + TeeSource savedNARSource(from, saved); + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNARSource); + } else { + /* Incrementally parse the NAR file, stripping the + metadata, and streaming the sole file we expect into + `saved`. */ + RetrieveRegularNARSink savedRegular { saved }; + parseDump(savedRegular, from); + if (!savedRegular.regular) throw Error("regular file expected"); + } + }); logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - - // FIXME: try to stream directly from `from`. - StringSource dumpSource { *saved.s }; - auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); + auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -469,17 +474,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store, dontCheckSigs = false; logger->startWork(); - FramedSource source(from); - store->addMultipleToStore(source, - RepairFlag{repair}, - dontCheckSigs ? NoCheckSigs : CheckSigs); + { + FramedSource source(from); + store->addMultipleToStore(source, + RepairFlag{repair}, + dontCheckSigs ? NoCheckSigs : CheckSigs); + } logger->stopWork(); break; } case wopAddTextToStore: { - string suffix = readString(from); - string s = readString(from); + std::string suffix = readString(from); + std::string s = readString(from); auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {}); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); @@ -529,6 +536,25 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } + case wopBuildPathsWithResults: { + auto drvs = readDerivedPaths(*store, clientVersion, from); + BuildMode mode = bmNormal; + mode = (BuildMode) readInt(from); + + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ + if (mode == bmRepair && !trusted) + throw Error("repairing is not allowed because you are not in 'trusted-users'"); + + logger->startWork(); + auto results = store->buildPathsWithResults(drvs, mode); + logger->stopWork(); + + worker_proto::write(*store, to, results); + + break; + } + case wopBuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; @@ -621,16 +647,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopAddIndirectRoot: { Path path = absPath(readString(from)); + logger->startWork(); - store->addIndirectRoot(path); + auto & gcStore = requireGcStore(*store); + gcStore.addIndirectRoot(path); logger->stopWork(); + to << 1; break; } + // Obsolete. case wopSyncWithGC: { logger->startWork(); - store->syncWithGC(); logger->stopWork(); to << 1; break; @@ -638,7 +667,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopFindRoots: { logger->startWork(); - Roots roots = store->findRoots(!trusted); + auto & gcStore = requireGcStore(*store); + Roots roots = gcStore.findRoots(!trusted); logger->stopWork(); size_t size = 0; @@ -669,7 +699,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); if (options.ignoreLiveness) throw Error("you are not allowed to ignore liveness"); - store->collectGarbage(options, results); + auto & gcStore = requireGcStore(*store); + gcStore.collectGarbage(options, results); logger->stopWork(); to << results.paths << results.bytesFreed << 0 /* obsolete */; @@ -697,8 +728,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { unsigned int n = readInt(from); for (unsigned int i = 0; i < n; i++) { - string name = readString(from); - string value = readString(from); + auto name = readString(from); + auto value = readString(from); clientSettings.overrides.emplace(name, value); } } @@ -853,14 +884,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store, else { std::unique_ptr<Source> source; + StringSink saved; if (GET_PROTOCOL_MINOR(clientVersion) >= 21) source = std::make_unique<TunnelSource>(from, to); else { - StringSink saved; TeeSource tee { from, saved }; ParseSink ether; parseDump(ether, tee); - source = std::make_unique<StringSource>(std::move(*saved.s)); + source = std::make_unique<StringSource>(saved.s); } logger->startWork(); @@ -921,6 +952,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } + case wopAddBuildLog: { + StorePath path{readString(from)}; + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to add logs"); + { + FramedSource source(from); + StringSink sink; + source.drainInto(sink); + store->addBuildLog(path, sink.s); + } + logger->stopWork(); + to << 1; + break; + } + default: throw Error("invalid operation %1%", op); } @@ -956,15 +1003,19 @@ void processConnection( Finally finally([&]() { _isInterrupted = false; - prevLogger->log(lvlDebug, fmt("%d operations", opCount)); + printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { - auto affinity = readInt(from); - setAffinityTo(affinity); + // Obsolete CPU affinity. + readInt(from); } - readInt(from); // obsolete reserveSpace + if (GET_PROTOCOL_MINOR(clientVersion) >= 11) + readInt(from); // obsolete reserveSpace + + if (GET_PROTOCOL_MINOR(clientVersion) >= 33) + to << nixVersion; /* Send startup error messages to the client. */ tunnelLogger->startWork(); @@ -989,6 +1040,8 @@ void processConnection( break; } + printMsgUsing(prevLogger, lvlDebug, "received daemon op %d", op); + opCount++; try { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 7868c7f94..f6f41361b 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -5,6 +5,7 @@ #include "split.hh" #include "worker-protocol.hh" #include "fs-accessor.hh" +#include <boost/container/small_vector.hpp> namespace nix { @@ -83,7 +84,7 @@ bool derivationIsImpure(DerivationType dt) { bool BasicDerivation::isBuiltin() const { - return string(builder, 0, 8) == "builtin:"; + return builder.substr(0, 8) == "builtin:"; } @@ -105,19 +106,19 @@ StorePath writeDerivation(Store & store, /* Read string `s' from stream `str'. */ -static void expect(std::istream & str, const string & s) +static void expect(std::istream & str, std::string_view s) { char s2[s.size()]; str.read(s2, s.size()); - if (string(s2, s.size()) != s) + if (std::string(s2, s.size()) != s) throw FormatError("expected string '%1%'", s); } /* Read a C-style string from stream `str'. */ -static string parseString(std::istream & str) +static std::string parseString(std::istream & str) { - string res; + std::string res; expect(str, "\""); int c; while ((c = str.get()) != '"') @@ -185,7 +186,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, }, }; } else { - settings.requireExperimentalFeature("ca-derivations"); + settings.requireExperimentalFeature(Xp::CaDerivations); assert(pathS == ""); return DerivationOutput { .output = DerivationOutputCAFloating { @@ -257,8 +258,8 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi /* Parse the environment variables. */ expect(str, ",["); while (!endOfList(str)) { - expect(str, "("); string name = parseString(str); - expect(str, ","); string value = parseString(str); + expect(str, "("); auto name = parseString(str); + expect(str, ","); auto value = parseString(str); expect(str, ")"); drv.env[name] = value; } @@ -268,9 +269,11 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi } -static void printString(string & res, std::string_view s) +static void printString(std::string & res, std::string_view s) { - char buf[s.size() * 2 + 2]; + boost::container::small_vector<char, 64 * 1024> buffer; + buffer.reserve(s.size() * 2 + 2); + char * buf = buffer.data(); char * p = buf; *p++ = '"'; for (auto c : s) @@ -284,7 +287,7 @@ static void printString(string & res, std::string_view s) } -static void printUnquotedString(string & res, std::string_view s) +static void printUnquotedString(std::string & res, std::string_view s) { res += '"'; res.append(s); @@ -293,7 +296,7 @@ static void printUnquotedString(string & res, std::string_view s) template<class ForwardIterator> -static void printStrings(string & res, ForwardIterator i, ForwardIterator j) +static void printStrings(std::string & res, ForwardIterator i, ForwardIterator j) { res += '['; bool first = true; @@ -306,7 +309,7 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j) template<class ForwardIterator> -static void printUnquotedStrings(string & res, ForwardIterator i, ForwardIterator j) +static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIterator j) { res += '['; bool first = true; @@ -318,10 +321,10 @@ static void printUnquotedStrings(string & res, ForwardIterator i, ForwardIterato } -string Derivation::unparse(const Store & store, bool maskOutputs, +std::string Derivation::unparse(const Store & store, bool maskOutputs, std::map<std::string, StringSet> * actualInputs) const { - string s; + std::string s; s.reserve(65536); s += "Derive(["; @@ -396,7 +399,7 @@ string Derivation::unparse(const Store & store, bool maskOutputs, // FIXME: remove -bool isDerivation(const string & fileName) +bool isDerivation(const std::string & fileName) { return hasSuffix(fileName, drvExtension); } @@ -588,7 +591,7 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & } -bool wantOutput(const string & output, const std::set<string> & wanted) +bool wantOutput(const std::string & output, const std::set<std::string> & wanted) { return wanted.empty() || wanted.find(output) != wanted.end(); } @@ -694,10 +697,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr } -std::string hashPlaceholder(const std::string & outputName) +std::string hashPlaceholder(const std::string_view outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); + return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); } std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index caf6062cd..f59887465 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -59,21 +59,19 @@ struct DerivationOutput std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const; }; -typedef std::map<string, DerivationOutput> DerivationOutputs; +typedef std::map<std::string, DerivationOutput> DerivationOutputs; /* These are analogues to the previous DerivationOutputs data type, but they also contains, for each output, the (optional) store path in which it would be written. To calculate values of these types, see the corresponding functions in BasicDerivation */ -typedef std::map<string, std::pair<DerivationOutput, std::optional<StorePath>>> +typedef std::map<std::string, std::pair<DerivationOutput, std::optional<StorePath>>> DerivationOutputsAndOptPaths; /* For inputs that are sub-derivations, we specify exactly which output IDs we are interested in. */ typedef std::map<StorePath, StringSet> DerivationInputs; -typedef std::map<string, string> StringPairs; - enum struct DerivationType : uint8_t { InputAddressed, DeferredInputAddressed, @@ -103,7 +101,7 @@ struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ StorePathSet inputSrcs; /* inputs that are sources */ - string platform; + std::string platform; Path builder; Strings args; StringPairs env; @@ -164,7 +162,7 @@ StorePath writeDerivation(Store & store, Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); // FIXME: remove -bool isDerivation(const string & fileName); +bool isDerivation(const std::string & fileName); /* Calculate the name that will be used for the store path for this output. @@ -222,7 +220,7 @@ typedef std::map<StorePath, DrvHashModulo> DrvHashes; // FIXME: global, though at least thread-safe. extern Sync<DrvHashes> drvHashes; -bool wantOutput(const string & output, const std::set<string> & wanted); +bool wantOutput(const std::string & output, const std::set<std::string> & wanted); struct Source; struct Sink; @@ -236,7 +234,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr It is used as a placeholder to allow derivations to refer to their own outputs without needing to use the hash of a derivation in itself, making the hash near-impossible to calculate. */ -std::string hashPlaceholder(const std::string & outputName); +std::string hashPlaceholder(const std::string_view outputName); /* This creates an opaque and almost certainly unique string deterministically from a derivation path and output name. diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index e55af21e9..194489580 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -75,9 +75,9 @@ DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_vi assert(n != s.npos); auto drvPath = store.parseStorePath(s.substr(0, n)); auto outputsS = s.substr(n + 1); - std::set<string> outputs; + std::set<std::string> outputs; if (outputsS != "*") - outputs = tokenizeString<std::set<string>>(outputsS, ","); + outputs = tokenizeString<std::set<std::string>>(outputsS, ","); return {drvPath, outputs}; } @@ -100,7 +100,7 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const staticOutputHashes(store, store.readDerivation(p.drvPath)); for (auto& [outputName, outputPath] : p.outputs) { if (settings.isExperimentalFeatureEnabled( - "ca-derivations")) { + Xp::CaDerivations)) { auto thisRealisation = store.queryRealisation( DrvOutput{drvHashes.at(outputName), outputName}); assert(thisRealisation); // We’ve built it, so we must h diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 36c6e725c..b4fbe0b70 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -21,7 +21,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store , Store(params) { } - string getUri() override + std::string getUri() override { return *uriSchemes().begin(); } @@ -43,15 +43,19 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store RepairFlag repair, CheckSigsFlag checkSigs) override { unsupported("addToStore"); } - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) override + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override { unsupported("addTextToStore"); } void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } - std::optional<const Realisation> queryRealisation(const DrvOutput&) override - { unsupported("queryRealisation"); } + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override + { callback(nullptr); } }; static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore; diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 55585e977..4adf51573 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -75,20 +75,20 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {}); auto deriver = readString(source); - auto narHash = hashString(htSHA256, *saved.s); + auto narHash = hashString(htSHA256, saved.s); ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); info.setReferencesPossiblyToSelf(std::move(references)); - info.narSize = saved.s->size(); + info.narSize = saved.s.size(); // Ignore optional legacy signature. if (readInt(source) == 1) readString(source); // Can't use underlying source, which would have been exhausted - auto source = StringSource { *saved.s }; + auto source = StringSource(saved.s); addToStore(info, source, NoRepair, checkSigs); res.push_back(info.path); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 2cf35ec83..c46262299 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -33,12 +33,12 @@ FileTransferSettings fileTransferSettings; static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); -std::string resolveUri(const std::string & uri) +std::string resolveUri(std::string_view uri) { if (uri.compare(0, 8, "channel:") == 0) - return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz"; + return "https://nixos.org/channels/" + std::string(uri.substr(8)) + "/nixexprs.tar.xz"; else - return uri; + return std::string(uri); } struct curlFileTransfer : public FileTransfer @@ -106,7 +106,7 @@ struct curlFileTransfer : public FileTransfer this->request.dataCallback(data); } } else - this->result.data->append(data); + this->result.data.append(data); }) { if (!request.expectedETag.empty()) @@ -128,7 +128,7 @@ struct curlFileTransfer : public FileTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri)); + fail(FileTransferError(Interrupted, {}, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -195,17 +195,17 @@ struct curlFileTransfer : public FileTransfer std::smatch match; if (std::regex_match(line, match, statusLine)) { result.etag = ""; - result.data = std::make_shared<std::string>(); + result.data.clear(); result.bodySize = 0; - statusMsg = trim(match[1]); + statusMsg = trim(match.str(1)); acceptRanges = false; encoding = ""; } else { auto i = line.find(':'); - if (i != string::npos) { - string name = toLower(trim(string(line, 0, i))); + if (i != std::string::npos) { + std::string name = toLower(trim(line.substr(0, i))); if (name == "etag") { - result.etag = trim(string(line, i + 1)); + result.etag = trim(line.substr(i + 1)); /* Hack to work around a GitHub bug: it sends ETags, but ignores If-None-Match. So if we get the expected ETag on a 200 response, then shut @@ -218,8 +218,8 @@ struct curlFileTransfer : public FileTransfer return 0; } } else if (name == "content-encoding") - encoding = trim(string(line, i + 1)); - else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes") + encoding = trim(line.substr(i + 1)); + else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes") acceptRanges = true; } } @@ -340,7 +340,7 @@ struct curlFileTransfer : public FileTransfer if (writtenToSink) curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); - result.data = std::make_shared<std::string>(); + result.data.clear(); result.bodySize = 0; } @@ -434,21 +434,21 @@ struct curlFileTransfer : public FileTransfer attempt++; - std::shared_ptr<std::string> response; + std::optional<std::string> response; if (errorSink) - response = errorSink->s; + response = std::move(errorSink->s); auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri) + ? FileTransferError(Interrupted, std::move(response), "%s of '%s' was interrupted", request.verb(), request.uri) : httpStatus != 0 ? FileTransferError(err, - response, + std::move(response), fmt("unable to %s '%s': HTTP error %d ('%s')", request.verb(), request.uri, httpStatus, statusMsg) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) : FileTransferError(err, - response, + std::move(response), fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -544,6 +544,8 @@ struct curlFileTransfer : public FileTransfer stopWorkerThread(); }); + unshareFilesystem(); + std::map<CURL *, std::shared_ptr<TransferItem>> items; bool quit = false; @@ -702,8 +704,8 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri); - res.data = s3Res.data; + throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri); + res.data = std::move(*s3Res.data); callback(std::move(res)); #else throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri); @@ -716,15 +718,24 @@ struct curlFileTransfer : public FileTransfer } }; +ref<curlFileTransfer> makeCurlFileTransfer() +{ + return make_ref<curlFileTransfer>(); +} + ref<FileTransfer> getFileTransfer() { - static ref<FileTransfer> fileTransfer = makeFileTransfer(); + static ref<curlFileTransfer> fileTransfer = makeCurlFileTransfer(); + + if (fileTransfer->state_.lock()->quit) + fileTransfer = makeCurlFileTransfer(); + return fileTransfer; } ref<FileTransfer> makeFileTransfer() { - return make_ref<curlFileTransfer>(); + return makeCurlFileTransfer(); } std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) @@ -848,25 +859,25 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } template<typename... Args> -FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args) +FileTransferError::FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args) : Error(args...), error(error), response(response) { const auto hf = hintfmt(args...); // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how // to print different messages for different verbosity levels. For now // we add some heuristics for detecting when we want to show the response. - if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) + if (response && (response->size() < 1024 || response->find("<html>") != std::string::npos)) err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response)); else err.msg = hf; } -bool isUri(const string & s) +bool isUri(std::string_view s) { if (s.compare(0, 8, "channel:") == 0) return true; size_t pos = s.find("://"); - if (pos == string::npos) return false; - string scheme(s, 0, pos); + if (pos == std::string::npos) return false; + std::string scheme(s, 0, pos); return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 45d9ccf89..ca61e3937 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -59,7 +59,7 @@ struct FileTransferRequest unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; - std::shared_ptr<std::string> data; + std::optional<std::string> data; std::string mimeType; std::function<void(std::string_view data)> dataCallback; @@ -77,7 +77,7 @@ struct FileTransferResult bool cached = false; std::string etag; std::string effectiveUri; - std::shared_ptr<std::string> data; + std::string data; uint64_t bodySize = 0; }; @@ -119,17 +119,17 @@ class FileTransferError : public Error { public: FileTransfer::Error error; - std::shared_ptr<string> response; // intentionally optional + std::optional<std::string> response; // intentionally optional template<typename... Args> - FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args); + FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args); virtual const char* sname() const override { return "FileTransferError"; } }; -bool isUri(const string & s); +bool isUri(std::string_view s); /* Resolve deprecated 'channel:<foo>' URLs. */ -std::string resolveUri(const std::string & uri); +std::string resolveUri(std::string_view uri); } diff --git a/src/libstore/gc-store.cc b/src/libstore/gc-store.cc new file mode 100644 index 000000000..3dbdec53b --- /dev/null +++ b/src/libstore/gc-store.cc @@ -0,0 +1,13 @@ +#include "gc-store.hh" + +namespace nix { + +GcStore & requireGcStore(Store & store) +{ + auto * gcStore = dynamic_cast<GcStore *>(&store); + if (!gcStore) + throw UsageError("Garbage collection not supported by this store"); + return *gcStore; +} + +} diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh new file mode 100644 index 000000000..829f70dc4 --- /dev/null +++ b/src/libstore/gc-store.hh @@ -0,0 +1,84 @@ +#pragma once + +#include "store-api.hh" + + +namespace nix { + + +typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots; + + +struct GCOptions +{ + /* Garbage collector operation: + + - `gcReturnLive': return the set of paths reachable from + (i.e. in the closure of) the roots. + + - `gcReturnDead': return the set of paths not reachable from + the roots. + + - `gcDeleteDead': actually delete the latter set. + + - `gcDeleteSpecific': delete the paths listed in + `pathsToDelete', insofar as they are not reachable. + */ + typedef enum { + gcReturnLive, + gcReturnDead, + gcDeleteDead, + gcDeleteSpecific, + } GCAction; + + GCAction action{gcDeleteDead}; + + /* If `ignoreLiveness' is set, then reachability from the roots is + ignored (dangerous!). However, the paths must still be + unreferenced *within* the store (i.e., there can be no other + store paths that depend on them). */ + bool ignoreLiveness{false}; + + /* For `gcDeleteSpecific', the paths to delete. */ + StorePathSet pathsToDelete; + + /* Stop after at least `maxFreed' bytes have been freed. */ + uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; +}; + + +struct GCResults +{ + /* Depending on the action, the GC roots, or the paths that would + be or have been deleted. */ + PathSet paths; + + /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the + number of bytes that would be or was freed. */ + uint64_t bytesFreed = 0; +}; + + +struct GcStore : public virtual Store +{ + /* Add an indirect root, which is merely a symlink to `path' from + /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed + to be a symlink to a store path. The garbage collector will + automatically remove the indirect root when it finds that + `path' has disappeared. */ + virtual void addIndirectRoot(const Path & path) = 0; + + /* Find the roots of the garbage collector. Each root is a pair + (link, storepath) where `link' is the path of the symlink + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) = 0; + + /* Perform a garbage collection. */ + virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; +}; + +GcStore & requireGcStore(Store & store); + +} diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 5a62c6529..024da66c1 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -10,48 +10,22 @@ #include <regex> #include <random> -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/statvfs.h> +#include <climits> #include <errno.h> #include <fcntl.h> +#include <poll.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/types.h> +#include <sys/un.h> #include <unistd.h> -#include <climits> namespace nix { -static string gcLockName = "gc.lock"; -static string gcRootsDir = "gcroots"; - - -/* Acquire the global GC lock. This is used to prevent new Nix - processes from starting after the temporary root files have been - read. To be precise: when they try to create a new temporary root - file, they will block until the garbage collector has finished / - yielded the GC lock. */ -AutoCloseFD LocalStore::openGCLock(LockType lockType) -{ - Path fnGCLock = (format("%1%/%2%") - % stateDir % gcLockName).str(); - - debug(format("acquiring global GC lock '%1%'") % fnGCLock); - - AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fdGCLock) - throw SysError("opening global GC lock '%1%'", fnGCLock); - - if (!lockFile(fdGCLock.get(), lockType, false)) { - printInfo("waiting for the big garbage collector lock..."); - lockFile(fdGCLock.get(), lockType, true); - } - - /* !!! Restrict read permission on the GC root. Otherwise any - process that can open the file for reading can DoS the - collector. */ - - return fdGCLock; -} +static std::string gcSocketPath = "/gc-socket/socket"; +static std::string gcRootsDir = "gcroots"; static void makeSymlink(const Path & link, const Path & target) @@ -71,17 +45,10 @@ static void makeSymlink(const Path & link, const Path & target) } -void LocalStore::syncWithGC() -{ - AutoCloseFD fdGCLock = openGCLock(ltRead); -} - - void LocalStore::addIndirectRoot(const Path & path) { - string hash = hashString(htSHA1, path).to_string(Base32, false); - Path realRoot = canonPath((format("%1%/%2%/auto/%3%") - % stateDir % gcRootsDir % hash).str()); + std::string hash = hashString(htSHA1, path).to_string(Base32, false); + Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } @@ -95,6 +62,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot "creating a garbage collector root (%1%) in the Nix store is forbidden " "(are you running nix-build inside the store?)", gcRoot); + /* Register this root with the garbage collector, if it's + running. This should be superfluous since the caller should + have registered this root yet, but let's be on the safe + side. */ + addTempRoot(storePath); + /* Don't clobber the link if it already exists and doesn't point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) @@ -102,11 +75,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); - /* Grab the global GC root, causing us to block while a GC is in - progress. This prevents the set of permanent roots from - increasing while a GC is in progress. */ - syncWithGC(); - return gcRoot; } @@ -119,8 +87,6 @@ void LocalStore::addTempRoot(const StorePath & path) if (!state->fdTempRoots) { while (1) { - AutoCloseFD fdGCLock = openGCLock(ltRead); - if (pathExists(fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ @@ -128,10 +94,8 @@ void LocalStore::addTempRoot(const StorePath & path) state->fdTempRoots = openLockFile(fnTempRoots, true); - fdGCLock = -1; - - debug(format("acquiring read lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltRead, true); + debug("acquiring write lock on '%s'", fnTempRoots); + lockFile(state->fdTempRoots.get(), ltWrite, true); /* Check whether the garbage collector didn't get in our way. */ @@ -147,24 +111,65 @@ void LocalStore::addTempRoot(const StorePath & path) } - /* Upgrade the lock to a write lock. This will cause us to block - if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltWrite, true); + if (!state->fdGCLock) + state->fdGCLock = openGCLock(); - string s = printStorePath(path) + '\0'; - writeFull(state->fdTempRoots.get(), s); + restart: + FdLock gcLock(state->fdGCLock.get(), ltRead, false, ""); + + if (!gcLock.acquired) { + /* We couldn't get a shared global GC lock, so the garbage + collector is running. So we have to connect to the garbage + collector and inform it about our root. */ + if (!state->fdRootsSocket) { + auto socketPath = stateDir.get() + gcSocketPath; + debug("connecting to '%s'", socketPath); + state->fdRootsSocket = createUnixDomainSocket(); + try { + nix::connect(state->fdRootsSocket.get(), socketPath); + } catch (SysError & e) { + /* The garbage collector may have exited, so we need to + restart. */ + if (e.errNo == ECONNREFUSED) { + debug("GC socket connection refused"); + state->fdRootsSocket.close(); + goto restart; + } + } + } - /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltRead, true); + try { + debug("sending GC root '%s'", printStorePath(path)); + writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false); + char c; + readFull(state->fdRootsSocket.get(), &c, 1); + assert(c == '1'); + debug("got ack for GC root '%s'", printStorePath(path)); + } catch (SysError & e) { + /* The garbage collector may have exited, so we need to + restart. */ + if (e.errNo == EPIPE) { + debug("GC socket disconnected"); + state->fdRootsSocket.close(); + goto restart; + } + } catch (EndOfFile & e) { + debug("GC socket disconnected"); + state->fdRootsSocket.close(); + goto restart; + } + } + + /* Append the store path to the temporary roots file. */ + auto s = printStorePath(path) + '\0'; + writeFull(state->fdTempRoots.get(), s); } static std::string censored = "{censored}"; -void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) +void LocalStore::findTempRoots(Roots & tempRoots, bool censor) { /* Read the `temproots' directory for per-process temporary root files. */ @@ -179,47 +184,35 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) pid_t pid = std::stoi(i.name); debug(format("reading temporary root file '%1%'") % path); - FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666))); - if (!*fd) { + AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)); + if (!fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; throw SysError("opening temporary roots file '%1%'", path); } - /* This should work, but doesn't, for some reason. */ - //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); - //if (*fd == -1) continue; - /* Try to acquire a write lock without blocking. This can only succeed if the owning process has died. In that case we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { + if (lockFile(fd.get(), ltWrite, false)) { printInfo("removing stale temporary roots file '%1%'", path); unlink(path.c_str()); - writeFull(fd->get(), "d"); + writeFull(fd.get(), "d"); continue; } - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - /* Read the entire file. */ - string contents = readFile(fd->get()); + auto contents = readFile(fd.get()); /* Extract the roots. */ - string::size_type pos = 0, end; + std::string::size_type pos = 0, end; - while ((end = contents.find((char) 0, pos)) != string::npos) { + while ((end = contents.find((char) 0, pos)) != std::string::npos) { Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } - - fds.push_back(fd); /* keep open */ } } @@ -304,15 +297,14 @@ Roots LocalStore::findRoots(bool censor) Roots roots; findRootsNoTemp(roots, censor); - FDs fds; - findTempRoots(fds, roots, censor); + findTempRoots(roots, censor); return roots; } typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots; -static void readProcLink(const string & file, UncheckedRoots & roots) +static void readProcLink(const std::string & file, UncheckedRoots & roots) { /* 64 is the starting buffer size gnu readlink uses... */ auto bufsiz = ssize_t{64}; @@ -335,12 +327,13 @@ try_again: .emplace(file); } -static string quoteRegexChars(const string & raw) +static std::string quoteRegexChars(const std::string & raw) { static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])"); return std::regex_replace(raw, specialRegex, R"(\$&)"); } +#if __linux__ static void readFileRoots(const char * path, UncheckedRoots & roots) { try { @@ -350,6 +343,7 @@ static void readFileRoots(const char * path, UncheckedRoots & roots) throw; } } +#endif void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { @@ -388,7 +382,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) try { auto mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile), "\n"); + auto mapLines = tokenizeString<std::vector<std::string>>(readFile(mapFile), "\n"); for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) @@ -419,7 +413,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) try { std::regex lsofRegex(R"(^n(/.*)$)"); auto lsofLines = - tokenizeString<std::vector<string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + tokenizeString<std::vector<std::string>>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); for (const auto & line : lsofLines) { std::smatch match; if (std::regex_match(line, match, lsofRegex)) @@ -431,7 +425,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } #endif -#if defined(__linux__) +#if __linux__ readFileRoots("/proc/sys/kernel/modprobe", unchecked); readFileRoots("/proc/sys/kernel/fbsplash", unchecked); readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); @@ -455,391 +449,408 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) struct GCLimitReached { }; -struct LocalStore::GCState -{ - const GCOptions & options; - GCResults & results; - StorePathSet roots; - StorePathSet tempRoots; - StorePathSet dead; - StorePathSet alive; - bool gcKeepOutputs; - bool gcKeepDerivations; - uint64_t bytesInvalidated; - bool moveToTrash = true; - bool shouldDelete; - GCState(const GCOptions & options, GCResults & results) - : options(options), results(results), bytesInvalidated(0) { } -}; - - -bool LocalStore::isActiveTempFile(const GCState & state, - const Path & path, const string & suffix) +void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { - return hasSuffix(path, suffix) - && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size()))); -} + bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + bool gcKeepOutputs = settings.gcKeepOutputs; + bool gcKeepDerivations = settings.gcKeepDerivations; + StorePathSet roots, dead, alive; -void LocalStore::deleteGarbage(GCState & state, const Path & path) -{ - uint64_t bytesFreed; - deletePath(path, bytesFreed); - state.results.bytesFreed += bytesFreed; -} + struct Shared + { + // The temp roots only store the hash part to make it easier to + // ignore suffixes like '.lock', '.chroot' and '.check'. + std::unordered_set<std::string> tempRoots; + // Hash part of the store path currently being deleted, if + // any. + std::optional<std::string> pending; + }; -void LocalStore::deletePathRecursive(GCState & state, const Path & path) -{ - checkInterrupt(); - - uint64_t size = 0; - - auto storePath = maybeParseStorePath(path); - if (storePath && isValidPath(*storePath)) { - StorePathSet referrers; - queryReferrers(*storePath, referrers); - for (auto & i : referrers) - if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i)); - size = queryPathInfo(*storePath)->narSize; - invalidatePathChecked(*storePath); - } + Sync<Shared> _shared; - Path realPath = realStoreDir + "/" + std::string(baseNameOf(path)); + std::condition_variable wakeup; - struct stat st; - if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT) return; - throw SysError("getting status of %1%", realPath); + /* Using `--ignore-liveness' with `--delete' can have unintended + consequences if `keep-outputs' or `keep-derivations' are true + (the garbage collector will recurse into deleting the outputs + or derivers, respectively). So disable them. */ + if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + gcKeepOutputs = false; + gcKeepDerivations = false; } - printInfo(format("deleting '%1%'") % path); - - state.results.paths.insert(path); + if (shouldDelete) + deletePath(reservedPath); - /* If the path is not a regular file or symlink, move it to the - trash directory. The move is to ensure that later (when we're - not holding the global GC lock) we can delete the path without - being afraid that the path has become alive again. Otherwise - delete it right away. */ - if (state.moveToTrash && S_ISDIR(st.st_mode)) { - // Estimate the amount freed using the narSize field. FIXME: - // if the path was not valid, need to determine the actual - // size. - try { - if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError("making '%1%' writable", realPath); - Path tmp = trashDir + "/" + std::string(baseNameOf(path)); - if (rename(realPath.c_str(), tmp.c_str())) - throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp); - state.bytesInvalidated += size; - } catch (SysError & e) { - if (e.errNo == ENOSPC) { - printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg()); - deleteGarbage(state, realPath); + /* Acquire the global GC root. Note: we don't use fdGCLock + here because then in auto-gc mode, another thread could + downgrade our exclusive lock. */ + auto fdGCLock = openGCLock(); + FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock..."); + + /* Start the server for receiving new roots. */ + auto socketPath = stateDir.get() + gcSocketPath; + createDirs(dirOf(socketPath)); + auto fdServer = createUnixDomainSocket(socketPath, 0666); + + if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1) + throw SysError("making socket '%1%' non-blocking", socketPath); + + Pipe shutdownPipe; + shutdownPipe.create(); + + std::thread serverThread([&]() { + Sync<std::map<int, std::thread>> connections; + + Finally cleanup([&]() { + debug("GC roots server shutting down"); + while (true) { + auto item = remove_begin(*connections.lock()); + if (!item) break; + auto & [fd, thread] = *item; + shutdown(fd, SHUT_RDWR); + thread.join(); } - } - } else - deleteGarbage(state, realPath); - - if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { - printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); - throw GCLimitReached(); - } -} - - -bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path) -{ - if (visited.count(path)) return false; - - if (state.alive.count(path)) return true; - - if (state.dead.count(path)) return false; - - if (state.roots.count(path)) { - debug("cannot delete '%1%' because it's a root", printStorePath(path)); - state.alive.insert(path); - return true; - } - - visited.insert(path); - - if (!isValidPath(path)) return false; - - StorePathSet incoming; - - /* Don't delete this path if any of its referrers are alive. */ - queryReferrers(path, incoming); - - /* If keep-derivations is set and this is a derivation, then - don't delete the derivation if any of the outputs are alive. */ - if (state.gcKeepDerivations && path.isDerivation()) { - for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path)) - if (maybeOutPath && - isValidPath(*maybeOutPath) && - queryPathInfo(*maybeOutPath)->deriver == path - ) - incoming.insert(*maybeOutPath); - } - - /* If keep-outputs is set, then don't delete this path if there - are derivers of this path that are not garbage. */ - if (state.gcKeepOutputs) { - auto derivers = queryValidDerivers(path); - for (auto & i : derivers) - incoming.insert(i); - } + }); + + while (true) { + std::vector<struct pollfd> fds; + fds.push_back({.fd = shutdownPipe.readSide.get(), .events = POLLIN}); + fds.push_back({.fd = fdServer.get(), .events = POLLIN}); + auto count = poll(fds.data(), fds.size(), -1); + assert(count != -1); + + if (fds[0].revents) + /* Parent is asking us to quit. */ + break; + + if (fds[1].revents) { + /* Accept a new connection. */ + assert(fds[1].revents & POLLIN); + AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr); + if (!fdClient) continue; + + debug("GC roots server accepted new client"); + + /* Process the connection in a separate thread. */ + auto fdClient_ = fdClient.get(); + std::thread clientThread([&, fdClient = std::move(fdClient)]() { + Finally cleanup([&]() { + auto conn(connections.lock()); + auto i = conn->find(fdClient.get()); + if (i != conn->end()) { + i->second.detach(); + conn->erase(i); + } + }); + + /* On macOS, accepted sockets inherit the + non-blocking flag from the server socket, so + explicitly make it blocking. */ + if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1) + abort(); + + while (true) { + try { + auto path = readLine(fdClient.get()); + auto storePath = maybeParseStorePath(path); + if (storePath) { + debug("got new GC root '%s'", path); + auto hashPart = std::string(storePath->hashPart()); + auto shared(_shared.lock()); + shared->tempRoots.insert(hashPart); + /* If this path is currently being + deleted, then we have to wait until + deletion is finished to ensure that + the client doesn't start + re-creating it before we're + done. FIXME: ideally we would use a + FD for this so we don't block the + poll loop. */ + while (shared->pending == hashPart) { + debug("synchronising with deletion of path '%s'", path); + shared.wait(wakeup); + } + } else + printError("received garbage instead of a root from client"); + writeFull(fdClient.get(), "1", false); + } catch (Error & e) { + debug("reading GC root from client: %s", e.msg()); + break; + } + } + }); - for (auto & i : incoming) - if (i != path) - if (canReachRoot(state, visited, i)) { - state.alive.insert(path); - return true; + connections.lock()->insert({fdClient_, std::move(clientThread)}); } + } + }); - return false; -} - - -void LocalStore::tryToDelete(GCState & state, const Path & path) -{ - checkInterrupt(); - - auto realPath = realStoreDir + "/" + std::string(baseNameOf(path)); - if (realPath == linksDir || realPath == trashDir) return; - - //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); - - auto storePath = maybeParseStorePath(path); - - if (!storePath || !isValidPath(*storePath)) { - /* A lock file belonging to a path that we're building right - now isn't garbage. */ - if (isActiveTempFile(state, path, ".lock")) return; + Finally stopServer([&]() { + writeFull(shutdownPipe.writeSide.get(), "x", false); + wakeup.notify_all(); + if (serverThread.joinable()) serverThread.join(); + }); - /* Don't delete .chroot directories for derivations that are - currently being built. */ - if (isActiveTempFile(state, path, ".chroot")) return; + /* Find the roots. Since we've grabbed the GC lock, the set of + permanent roots cannot increase now. */ + printInfo("finding garbage collector roots..."); + Roots rootMap; + if (!options.ignoreLiveness) + findRootsNoTemp(rootMap, true); - /* Don't delete .check directories for derivations that are - currently being built, because we may need to run - diff-hook. */ - if (isActiveTempFile(state, path, ".check")) return; - } + for (auto & i : rootMap) roots.insert(i.first); - StorePathSet visited; - - if (storePath && canReachRoot(state, visited, *storePath)) { - debug("cannot delete '%s' because it's still reachable", path); - } else { - /* No path we visited was a root, so everything is garbage. - But we only delete ‘path’ and its referrers here so that - ‘nix-store --delete’ doesn't have the unexpected effect of - recursing into derivations and outputs. */ - for (auto & i : visited) - state.dead.insert(i); - if (state.shouldDelete) - deletePathRecursive(state, path); + /* Read the temporary roots created before we acquired the global + GC root. Any new roots will be sent to our socket. */ + Roots tempRoots; + findTempRoots(tempRoots, true); + for (auto & root : tempRoots) { + _shared.lock()->tempRoots.insert(std::string(root.first.hashPart())); + roots.insert(root.first); } -} - -/* Unlink all files in /nix/store/.links that have a link count of 1, - which indicates that there are no other links and so they can be - safely deleted. FIXME: race condition with optimisePath(): we - might see a link count of 1 just before optimisePath() increases - the link count. */ -void LocalStore::removeUnusedLinks(const GCState & state) -{ - AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError("opening directory '%1%'", linksDir); + /* Helper function that deletes a path from the store and throws + GCLimitReached if we've deleted enough garbage. */ + auto deleteFromStore = [&](std::string_view baseName) + { + Path path = storeDir + "/" + std::string(baseName); + Path realPath = realStoreDir + "/" + std::string(baseName); - int64_t actualSize = 0, unsharedSize = 0; + printInfo("deleting '%1%'", path); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = linksDir + "/" + name; + results.paths.insert(path); - auto st = lstat(path); + uint64_t bytesFreed; + deletePath(realPath, bytesFreed); + results.bytesFreed += bytesFreed; - if (st.st_nlink != 1) { - actualSize += st.st_size; - unsharedSize += (st.st_nlink - 1) * st.st_size; - continue; + if (results.bytesFreed > options.maxFreed) { + printInfo("deleted more than %d bytes; stopping", options.maxFreed); + throw GCLimitReached(); } + }; - printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); - - if (unlink(path.c_str()) == -1) - throw SysError("deleting '%1%'", path); + std::map<StorePath, StorePathSet> referrersCache; - state.results.bytesFreed += st.st_size; - } + /* Helper function that visits all paths reachable from `start` + via the referrers edges and optionally derivers and derivation + output edges. If none of those paths are roots, then all + visited paths are garbage and are deleted. */ + auto deleteReferrersClosure = [&](const StorePath & start) { + StorePathSet visited; + std::queue<StorePath> todo; - struct stat st; - if (stat(linksDir.c_str(), &st) == -1) - throw SysError("statting '%1%'", linksDir); - int64_t overhead = st.st_blocks * 512ULL; + /* Wake up any GC client waiting for deletion of the paths in + 'visited' to finish. */ + Finally releasePending([&]() { + auto shared(_shared.lock()); + shared->pending.reset(); + wakeup.notify_all(); + }); - printInfo("note: currently hard linking saves %.2f MiB", - ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); -} + auto enqueue = [&](const StorePath & path) { + if (visited.insert(path).second) + todo.push(path); + }; + enqueue(start); -void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) -{ - GCState state(options, results); - state.gcKeepOutputs = settings.gcKeepOutputs; - state.gcKeepDerivations = settings.gcKeepDerivations; + while (auto path = pop(todo)) { + checkInterrupt(); - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `keep-outputs' or `keep-derivations' are true - (the garbage collector will recurse into deleting the outputs - or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - state.gcKeepOutputs = false; - state.gcKeepDerivations = false; - } + /* Bail out if we've previously discovered that this path + is alive. */ + if (alive.count(*path)) { + alive.insert(start); + return; + } - state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + /* If we've previously deleted this path, we don't have to + handle it again. */ + if (dead.count(*path)) continue; - if (state.shouldDelete) - deletePath(reservedPath); + auto markAlive = [&]() + { + alive.insert(*path); + alive.insert(start); + try { + StorePathSet closure; + computeFSClosure(*path, closure); + for (auto & p : closure) + alive.insert(p); + } catch (InvalidPath &) { } + }; + + /* If this is a root, bail out. */ + if (roots.count(*path)) { + debug("cannot delete '%s' because it's a root", printStorePath(*path)); + return markAlive(); + } - /* Acquire the global GC root. This prevents - a) New roots from being added. - b) Processes from creating new temporary root files. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); + if (options.action == GCOptions::gcDeleteSpecific + && !options.pathsToDelete.count(*path)) + return; - /* Find the roots. Since we've grabbed the GC lock, the set of - permanent roots cannot increase now. */ - printInfo("finding garbage collector roots..."); - Roots rootMap; - if (!options.ignoreLiveness) - findRootsNoTemp(rootMap, true); + { + auto hashPart = std::string(path->hashPart()); + auto shared(_shared.lock()); + if (shared->tempRoots.count(hashPart)) { + debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); + return markAlive(); + } + shared->pending = hashPart; + } - for (auto & i : rootMap) state.roots.insert(i.first); + if (isValidPath(*path)) { - /* Read the temporary roots. This acquires read locks on all - per-process temporary root files. So after this point no paths - can be added to the set of temporary roots. */ - FDs fds; - Roots tempRoots; - findTempRoots(fds, tempRoots, true); - for (auto & root : tempRoots) { - state.tempRoots.insert(root.first); - state.roots.insert(root.first); - } + /* Visit the referrers of this path. */ + auto i = referrersCache.find(*path); + if (i == referrersCache.end()) { + StorePathSet referrers; + queryReferrers(*path, referrers); + referrersCache.emplace(*path, std::move(referrers)); + i = referrersCache.find(*path); + } + for (auto & p : i->second) + enqueue(p); + + /* If keep-derivations is set and this is a + derivation, then visit the derivation outputs. */ + if (gcKeepDerivations && path->isDerivation()) { + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) + if (maybeOutPath && + isValidPath(*maybeOutPath) && + queryPathInfo(*maybeOutPath)->deriver == *path) + enqueue(*maybeOutPath); + } - /* After this point the set of roots or temporary roots cannot - increase, since we hold locks on everything. So everything - that is not reachable from `roots' is garbage. */ + /* If keep-outputs is set, then visit the derivers. */ + if (gcKeepOutputs) { + auto derivers = queryValidDerivers(*path); + for (auto & i : derivers) + enqueue(i); + } + } + } - if (state.shouldDelete) { - if (pathExists(trashDir)) deleteGarbage(state, trashDir); - try { - createDirs(trashDir); - } catch (SysError & e) { - if (e.errNo == ENOSPC) { - printInfo("note: can't create trash directory: %s", e.msg()); - state.moveToTrash = false; + for (auto & path : topoSortPaths(visited)) { + if (!dead.insert(path).second) continue; + if (shouldDelete) { + invalidatePathChecked(path); + deleteFromStore(path.to_string()); + referrersCache.erase(path); } } - } + }; - /* Now either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ + /* Synchronisation point for testing, see tests/gc-concurrent.sh. */ + if (auto p = getEnv("_NIX_TEST_GC_SYNC")) + readFile(*p); + /* Either delete all garbage paths, or just the specified + paths (for gcDeleteSpecific). */ if (options.action == GCOptions::gcDeleteSpecific) { for (auto & i : options.pathsToDelete) { - tryToDelete(state, printStorePath(i)); - if (state.dead.find(i) == state.dead.end()) + deleteReferrersClosure(i); + if (!dead.count(i)) throw Error( - "cannot delete path '%1%' since it is still alive. " - "To find out why use: " + "Cannot delete path '%1%' since it is still alive. " + "To find out why, use: " "nix-store --query --roots", printStorePath(i)); } } else if (options.maxFreed > 0) { - if (state.shouldDelete) + if (shouldDelete) printInfo("deleting garbage..."); else printInfo("determining live/dead paths..."); try { - AutoCloseDir dir(opendir(realStoreDir.get().c_str())); if (!dir) throw SysError("opening directory '%1%'", realStoreDir); - /* Read the store and immediately delete all paths that - aren't valid. When using --max-freed etc., deleting - invalid paths is preferred over deleting unreachable - paths, since unreachable paths could become reachable - again. We don't use readDirectory() here so that GCing - can start faster. */ + /* Read the store and delete all paths that are invalid or + unreachable. We don't use readDirectory() here so that + GCing can start faster. */ + auto linksName = baseNameOf(linksDir); Paths entries; struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = storeDir + "/" + name; - auto storePath = maybeParseStorePath(path); - if (storePath && isValidPath(*storePath)) - entries.push_back(path); - else - tryToDelete(state, path); - } - - dir.reset(); - - /* Now delete the unreachable valid paths. Randomise the - order in which we delete entries to make the collector - less biased towards deleting paths that come - alphabetically first (e.g. /nix/store/000...). This - matters when using --max-freed etc. */ - vector<Path> entries_(entries.begin(), entries.end()); - std::mt19937 gen(1); - std::shuffle(entries_.begin(), entries_.end(), gen); + std::string name = dirent->d_name; + if (name == "." || name == ".." || name == linksName) continue; - for (auto & i : entries_) - tryToDelete(state, i); + if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) + deleteReferrersClosure(*storePath); + else + deleteFromStore(name); + } } catch (GCLimitReached & e) { } } - if (state.options.action == GCOptions::gcReturnLive) { - for (auto & i : state.alive) - state.results.paths.insert(printStorePath(i)); + if (options.action == GCOptions::gcReturnLive) { + for (auto & i : alive) + results.paths.insert(printStorePath(i)); return; } - if (state.options.action == GCOptions::gcReturnDead) { - for (auto & i : state.dead) - state.results.paths.insert(printStorePath(i)); + if (options.action == GCOptions::gcReturnDead) { + for (auto & i : dead) + results.paths.insert(printStorePath(i)); return; } - /* Allow other processes to add to the store from here on. */ - fdGCLock = -1; - fds.clear(); - - /* Delete the trash directory. */ - printInfo(format("deleting '%1%'") % trashDir); - deleteGarbage(state, trashDir); - - /* Clean up the links directory. */ + /* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { printInfo("deleting unused links..."); - removeUnusedLinks(state); + + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError("opening directory '%1%'", linksDir); + + int64_t actualSize = 0, unsharedSize = 0; + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + auto st = lstat(path); + + if (st.st_nlink != 1) { + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; + continue; + } + + printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError("deleting '%1%'", path); + + results.bytesFreed += st.st_size; + } + + struct stat st; + if (stat(linksDir.c_str(), &st) == -1) + throw SysError("statting '%1%'", linksDir); + int64_t overhead = st.st_blocks * 512ULL; + + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } /* While we're at it, vacuum the database. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 6934801e3..cc009a026 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -100,7 +100,7 @@ std::vector<Path> getUserConfigFiles() // Use the paths specified in NIX_USER_CONF_FILES if it has been defined auto nixConfFiles = getEnv("NIX_USER_CONF_FILES"); if (nixConfFiles.has_value()) { - return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":"); + return tokenizeString<std::vector<std::string>>(nixConfFiles.value(), ":"); } // Use the paths specified by the XDG spec @@ -122,7 +122,7 @@ StringSet Settings::getDefaultSystemFeatures() /* For backwards compatibility, accept some "features" that are used in Nixpkgs to route builds to certain machines but don't actually require anything special on the machines. */ - StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"}; + StringSet features{"nixos-test", "benchmark", "big-parallel"}; #if __linux__ if (access("/dev/kvm", R_OK | W_OK) == 0) @@ -148,7 +148,8 @@ StringSet Settings::getDefaultExtraPlatforms() // machines. Note that we can’t force processes from executing // x86_64 in aarch64 environments or vice versa since they can // always exec with their own binary preferences. - if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { + if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist") || + pathExists("/System/Library/LaunchDaemons/com.apple.oahd.plist")) { if (std::string{SYSTEM} == "x86_64-darwin") extraPlatforms.insert("aarch64-darwin"); else if (std::string{SYSTEM} == "aarch64-darwin") @@ -159,21 +160,16 @@ StringSet Settings::getDefaultExtraPlatforms() return extraPlatforms; } -bool Settings::isExperimentalFeatureEnabled(const std::string & name) +bool Settings::isExperimentalFeatureEnabled(const ExperimentalFeature & feature) { auto & f = experimentalFeatures.get(); - return std::find(f.begin(), f.end(), name) != f.end(); + return std::find(f.begin(), f.end(), feature) != f.end(); } -MissingExperimentalFeature::MissingExperimentalFeature(std::string feature) - : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", feature) - , missingFeature(feature) - {} - -void Settings::requireExperimentalFeature(const std::string & name) +void Settings::requireExperimentalFeature(const ExperimentalFeature & feature) { - if (!isExperimentalFeatureEnabled(name)) - throw MissingExperimentalFeature(name); + if (!isExperimentalFeatureEnabled(feature)) + throw MissingExperimentalFeature(feature); } bool Settings::isWSL1() @@ -185,7 +181,7 @@ bool Settings::isWSL1() return hasSuffix(utsbuf.release, "-Microsoft"); } -const string nixVersion = PACKAGE_VERSION; +const std::string nixVersion = PACKAGE_VERSION; NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, { {SandboxMode::smEnabled, true}, diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8784d5faf..feb6899cd 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "config.hh" #include "util.hh" +#include "experimental-features.hh" #include <map> #include <limits> @@ -20,7 +21,7 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int> const std::string & name, const std::string & description, const std::set<std::string> & aliases = {}) - : BaseSetting<unsigned int>(def, name, description, aliases) + : BaseSetting<unsigned int>(def, true, name, description, aliases) { options->addSetting(this); } @@ -37,7 +38,7 @@ struct PluginFilesSetting : public BaseSetting<Paths> const std::string & name, const std::string & description, const std::set<std::string> & aliases = {}) - : BaseSetting<Paths>(def, name, description, aliases) + : BaseSetting<Paths>(def, true, name, description, aliases) { options->addSetting(this); } @@ -45,15 +46,6 @@ struct PluginFilesSetting : public BaseSetting<Paths> void set(const std::string & str, bool append = false) override; }; -class MissingExperimentalFeature: public Error -{ -public: - std::string missingFeature; - - MissingExperimentalFeature(std::string feature); - virtual const char* sname() const override { return "MissingExperimentalFeature"; } -}; - class Settings : public Config { unsigned int getDefaultCores(); @@ -121,7 +113,7 @@ public: bool verboseBuild = true; Setting<size_t> logLines{this, 10, "log-lines", - "If `verbose-build` is false, the number of lines of the tail of " + "The number of lines of the tail of " "the log to show if a build fails."}; MaxBuildJobsSetting maxBuildJobs{ @@ -138,7 +130,9 @@ public: {"build-max-jobs"}}; Setting<unsigned int> buildCores{ - this, getDefaultCores(), "cores", + this, + getDefaultCores(), + "cores", R"( Sets the value of the `NIX_BUILD_CORES` environment variable in the invocation of builders. Builders can use this variable at their @@ -149,7 +143,7 @@ public: command line switch and defaults to `1`. The value `0` means that the builder should use all available CPU cores in the system. )", - {"build-cores"}}; + {"build-cores"}, false}; /* Read-only mode. Don't copy stuff to the store, don't change the database. */ @@ -591,10 +585,11 @@ public: platform and generate incompatible code, so you may wish to cross-check the results of using this option against proper natively-built versions of your derivations. - )"}; + )", {}, false}; Setting<StringSet> systemFeatures{ - this, getDefaultSystemFeatures(), + this, + getDefaultSystemFeatures(), "system-features", R"( A set of system “features” supported by this machine, e.g. `kvm`. @@ -610,7 +605,7 @@ public: This setting by default includes `kvm` if `/dev/kvm` is accessible, and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` that are used in Nixpkgs to route builds to specific machines. - )"}; + )", {}, false}; Setting<Strings> substituters{ this, @@ -805,6 +800,15 @@ public: may be useful in certain scenarios (e.g. to spin up containers or set up userspace network interfaces in tests). )"}; + + Setting<StringSet> ignoredAcls{ + this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls", + R"( + A list of ACLs that should be ignored, normally Nix attempts to + remove all ACLs from files and directories in the Nix store, but + some ACLs like `security.selinux` or `system.nfs4_acl` can't be + removed even by root. Therefore it's best to just ignore them. + )"}; #endif Setting<Strings> hashedMirrors{ @@ -876,74 +880,16 @@ public: are loaded as plugins (non-recursively). )"}; - Setting<StringMap> accessTokens{this, {}, "access-tokens", - R"( - Access tokens used to access protected GitHub, GitLab, or - other locations requiring token-based authentication. - - Access tokens are specified as a string made up of - space-separated `host=token` values. The specific token - used is selected by matching the `host` portion against the - "host" specification of the input. The actual use of the - `token` value is determined by the type of resource being - accessed: - - * Github: the token value is the OAUTH-TOKEN string obtained - as the Personal Access Token from the Github server (see - https://docs.github.com/en/developers/apps/authorizing-oath-apps). - - * Gitlab: the token value is either the OAuth2 token or the - Personal Access Token (these are different types tokens - for gitlab, see - https://docs.gitlab.com/12.10/ee/api/README.html#authentication). - The `token` value should be `type:tokenstring` where - `type` is either `OAuth2` or `PAT` to indicate which type - of token is being specified. - - Example `~/.config/nix/nix.conf`: - - ``` - access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk - ``` - - Example `~/code/flake.nix`: - - ```nix - input.foo = { - type = "gitlab"; - host = "gitlab.mycompany.com"; - owner = "mycompany"; - repo = "pro"; - }; - ``` - - This example specifies three tokens, one each for accessing - github.com, gitlab.mycompany.com, and sourceforge.net. - - The `input.foo` uses the "gitlab" fetcher, which might - requires specifying the token type along with the token - value. - )"}; - - Setting<Strings> experimentalFeatures{this, {}, "experimental-features", + Setting<std::set<ExperimentalFeature>> experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; - bool isExperimentalFeatureEnabled(const std::string & name); - - void requireExperimentalFeature(const std::string & name); + bool isExperimentalFeatureEnabled(const ExperimentalFeature &); - Setting<bool> allowDirty{this, true, "allow-dirty", - "Whether to allow dirty Git/Mercurial trees."}; - - Setting<bool> warnDirty{this, true, "warn-dirty", - "Whether to warn about dirty Git/Mercurial trees."}; + void requireExperimentalFeature(const ExperimentalFeature &); Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; - Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", - "Path or URI of the global flake registry."}; - Setting<bool> allowSymlinkedStore{ this, false, "allow-symlinked-store", R"( @@ -956,9 +902,6 @@ public: resolves to a different location from that of the build machine. You can enable this setting if you are sure you're not going to do that. )"}; - - Setting<bool> useRegistries{this, true, "use-registries", - "Whether to use flake registries to resolve flake references."}; }; @@ -974,6 +917,6 @@ void loadConfFile(); // Used by the Settings constructor std::vector<Path> getUserConfigFiles(); -extern const string nixVersion; +extern const std::string nixVersion; } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 605ec4b28..3cb5efdbf 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -126,7 +126,7 @@ protected: const std::string & mimeType) override { auto req = makeRequest(path); - req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain()); + req.data = StreamToSourceAdapter(istream).drain(); req.mimeType = mimeType; try { getFileTransfer()->upload(req); @@ -159,7 +159,7 @@ protected: } void getFile(const std::string & path, - Callback<std::shared_ptr<std::string>> callback) noexcept override + Callback<std::optional<std::string>> callback) noexcept override { checkEnabled(); @@ -170,10 +170,10 @@ protected: getFileTransfer()->enqueueFileTransfer(request, {[callbackPtr, this](std::future<FileTransferResult> result) { try { - (*callbackPtr)(result.get().data); + (*callbackPtr)(std::move(result.get().data)); } catch (FileTransferError & e) { if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) - return (*callbackPtr)(std::shared_ptr<std::string>()); + return (*callbackPtr)({}); maybeDisable(); callbackPtr->rethrow(); } catch (...) { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 770827b85..22d860667 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -2,6 +2,7 @@ #include "pool.hh" #include "remote-store.hh" #include "serve-protocol.hh" +#include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" #include "worker-protocol.hh" @@ -48,7 +49,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor static std::set<std::string> uriSchemes() { return {"ssh"}; } - LegacySSHStore(const string & scheme, const string & host, const Params & params) + LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) , LegacySSHStoreConfig(params) , Store(params) @@ -94,7 +95,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->sshConn->in.close(); auto msg = conn->from.drain(); throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(*saved.s + msg)); + host, chomp(saved.s + msg)); } conn->remoteVersion = readInt(conn->from); if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) @@ -107,7 +108,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor return conn; }; - string getUri() override + std::string getUri() override { return *uriSchemes().begin() + "://" + host; } @@ -224,13 +225,21 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } - StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override + StorePath addToStore( + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashType hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override { unsupported("addToStore"); } - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) override + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override { unsupported("addTextToStore"); } private: @@ -269,7 +278,7 @@ public: conn->to.flush(); - BuildResult status; + BuildResult status { .path = DerivedPath::Built { .drvPath = drvPath } }; status.status = (BuildResult::Status) readInt(conn->from); conn->from >> status.errorMsg; @@ -307,7 +316,7 @@ public: conn->to.flush(); - BuildResult result; + BuildResult result { .path = DerivedPath::Opaque { StorePath::dummy } }; result.status = (BuildResult::Status) readInt(conn->from); if (!result.success()) { @@ -366,7 +375,8 @@ public: return conn->remoteVersion; } - std::optional<const Realisation> queryRealisation(const DrvOutput&) override + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override // TODO: Implement { unsupported("queryRealisation"); } }; diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index f93111fce..f754770f9 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -96,6 +96,7 @@ void LocalBinaryCacheStore::init() createDirs(binaryCacheDir + "/" + realisationsPrefix); if (writeDebugInfo) createDirs(binaryCacheDir + "/debuginfo"); + createDirs(binaryCacheDir + "/log"); BinaryCacheStore::init(); } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 6de13c73a..c5ae7536f 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -85,36 +85,34 @@ void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); } -const string LocalFSStore::drvsLogDir = "drvs"; +const std::string LocalFSStore::drvsLogDir = "drvs"; - - -std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_) +std::optional<std::string> LocalFSStore::getBuildLog(const StorePath & path_) { auto path = path_; if (!path.isDerivation()) { try { auto info = queryPathInfo(path); - if (!info->deriver) return nullptr; + if (!info->deriver) return std::nullopt; path = *info->deriver; } catch (InvalidPath &) { - return nullptr; + return std::nullopt; } } - auto baseName = std::string(baseNameOf(printStorePath(path))); + auto baseName = path.to_string(); for (int j = 0; j < 2; j++) { Path logPath = j == 0 - ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2)) + ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) - return std::make_shared<std::string>(readFile(logPath)); + return readFile(logPath); else if (pathExists(logBz2Path)) { try { @@ -124,7 +122,7 @@ std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_) } - return nullptr; + return std::nullopt; } } diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index f8b19d00d..fbd49dc2c 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -1,6 +1,7 @@ #pragma once #include "store-api.hh" +#include "gc-store.hh" namespace nix { @@ -23,11 +24,11 @@ struct LocalFSStoreConfig : virtual StoreConfig "physical path to the Nix store"}; }; -class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store +class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store, virtual GcStore { public: - const static string drvsLogDir; + const static std::string drvsLogDir; LocalFSStore(const Params & params); @@ -45,7 +46,8 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } - std::shared_ptr<std::string> getBuildLog(const StorePath & path) override; + std::optional<std::string> getBuildLog(const StorePath & path) override; + }; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8fcedd2fc..52ca5a34a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -8,6 +8,8 @@ #include "references.hh" #include "callback.hh" #include "topo-sort.hh" +#include "finally.hh" +#include "compression.hh" #include <iostream> #include <algorithm> @@ -68,7 +70,7 @@ int getSchema(Path schemaPath) { int curSchema = 0; if (pathExists(schemaPath)) { - string s = readFile(schemaPath); + auto s = readFile(schemaPath); auto n = string2Int<int>(s); if (!n) throw Error("'%1%' is corrupt", schemaPath); @@ -79,7 +81,7 @@ int getSchema(Path schemaPath) void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) { - const int nixCASchemaVersion = 2; + const int nixCASchemaVersion = 3; int curCASchema = getSchema(schemaPath); if (curCASchema != nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) { @@ -130,6 +132,17 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) txn.commit(); } + if (curCASchema < 3) { + SQLiteTxn txn(db); + // Apply new indices added in this schema update. + db.exec(R"( + -- used by QueryRealisationReferences + create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); + -- used by cascade deletion when ValidPaths is deleted + create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); + )"); + txn.commit(); + } writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); lockFile(lockFd.get(), ltRead, true); } @@ -145,7 +158,6 @@ LocalStore::LocalStore(const Params & params) , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") - , trashDir(realStoreDir + "/trash") , tempRootsDir(stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) , locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or(""))) @@ -227,7 +239,7 @@ LocalStore::LocalStore(const Params & params) res = posix_fallocate(fd.get(), 0, settings.reservedSize); #endif if (res == -1) { - writeFull(fd.get(), string(settings.reservedSize, 'X')); + writeFull(fd.get(), std::string(settings.reservedSize, 'X')); [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); } } @@ -309,7 +321,7 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } @@ -339,7 +351,7 @@ LocalStore::LocalStore(const Params & params) state->stmts->QueryPathFromHashPart.create(state->db, "select path from ValidPaths where path >= ? limit 1;"); state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths"); - if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { state->stmts->RegisterRealisedOutput.create(state->db, R"( insert into Realisations (drvPath, outputName, outputPath, signatures) @@ -386,6 +398,16 @@ LocalStore::LocalStore(const Params & params) } +AutoCloseFD LocalStore::openGCLock() +{ + Path fnGCLock = stateDir + "/gc.lock"; + auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fdGCLock) + throw SysError("opening global GC lock '%1%'", fnGCLock); + return fdGCLock; +} + + LocalStore::~LocalStore() { std::shared_future<void> future; @@ -428,7 +450,7 @@ void LocalStore::openDB(State & state, bool create) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ - string dbPath = dbDir + "/db.sqlite"; + std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); state.db = SQLite(dbPath, create); @@ -449,19 +471,19 @@ void LocalStore::openDB(State & state, bool create) should be safe enough. If the user asks for it, don't sync at all. This can cause database corruption if the system crashes. */ - string syncMode = settings.fsyncMetadata ? "normal" : "off"; + std::string syncMode = settings.fsyncMetadata ? "normal" : "off"; db.exec("pragma synchronous = " + syncMode); /* Set the SQLite journal mode. WAL mode is fastest, so it's the default. */ - string mode = settings.useSQLiteWAL ? "wal" : "truncate"; - string prevMode; + std::string mode = settings.useSQLiteWAL ? "wal" : "truncate"; + std::string prevMode; { SQLiteStmt stmt; stmt.create(db, "pragma main.journal_mode;"); if (sqlite3_step(stmt) != SQLITE_ROW) throwSQLiteError(db, "querying journal mode"); - prevMode = string((const char *) sqlite3_column_text(stmt, 0)); + prevMode = std::string((const char *) sqlite3_column_text(stmt, 0)); } if (prevMode != mode && sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) @@ -495,9 +517,6 @@ void LocalStore::makeStoreWritable() throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) throw SysError("remounting %1% writable", realStoreDir); } @@ -583,9 +602,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe throw SysError("querying extended attributes of '%s'", path); for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - /* Ignore SELinux security labels since these cannot be - removed even by root. */ - if (eaName == "security.selinux") continue; + if (settings.ignoredAcls.get().count(eaName)) continue; if (lremovexattr(path.c_str(), eaName.c_str()) == -1) throw SysError("removing extended attribute '%s' from '%s'", eaName, path); } @@ -662,7 +679,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat { assert(drvPath.isDerivation()); std::string drvName(drvPath.name()); - drvName = string(drvName, 0, drvName.size() - drvExtension.size()); + drvName = drvName.substr(0, drvName.size() - drvExtension.size()); auto envHasRightPath = [&](const StorePath & actual, const std::string & varName) { @@ -708,7 +725,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { - settings.requireExperimentalFeature("ca-derivations"); + settings.requireExperimentalFeature(Xp::CaDerivations); if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info)) registerDrvOutput(info); else @@ -717,7 +734,7 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check void LocalStore::registerDrvOutput(const Realisation & info) { - settings.requireExperimentalFeature("ca-derivations"); + settings.requireExperimentalFeature(Xp::CaDerivations); retrySQLite<void>([&]() { auto state(_state.lock()); if (auto oldR = queryRealisation_(*state, info.id)) { @@ -769,7 +786,11 @@ void LocalStore::registerDrvOutput(const Realisation & info) }); } -void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output) +void LocalStore::cacheDrvOutputMapping( + State & state, + const uint64_t deriver, + const std::string & outputName, + const StorePath & output) { retrySQLite<void>([&]() { state.stmts->AddDerivationOutput.use() @@ -778,7 +799,6 @@ void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, co (printStorePath(output)) .exec(); }); - } @@ -825,7 +845,7 @@ uint64_t LocalStore::addValidPath(State & state, { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(std::string(info.path.hashPart()), + state_->pathInfoCache.upsert(std::string(info.path.to_string()), PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) }); } @@ -1004,7 +1024,7 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) return outputs; }); - if (!settings.isExperimentalFeatureEnabled("ca-derivations")) + if (!settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) return outputs; auto drv = readInvalidDerivation(path); @@ -1204,7 +1224,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(std::string(path.hashPart())); + state_->pathInfoCache.erase(std::string(path.to_string())); } } @@ -1297,7 +1317,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, canonicalisePathMetaData(realPath, -1); - optimisePath(realPath); // FIXME: combine with hashPath() + optimisePath(realPath, repair); // FIXME: combine with hashPath() registerValidPath(info); } @@ -1307,8 +1327,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) +StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { /* For computing the store path. */ auto hashSink = std::make_unique<HashSink>(hashAlgo); @@ -1333,13 +1353,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto want = std::min(chunkSize, settings.narBufferSize - oldSize); dump.resize(oldSize + want); auto got = 0; + Finally cleanup([&]() { + dump.resize(oldSize + got); + }); try { got = source.read(dump.data() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; } - dump.resize(oldSize + got); } std::unique_ptr<AutoDelete> delTempDir; @@ -1365,13 +1387,16 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); auto desc = StorePathDescriptor { - name, + std::string { name }, FixedOutputInfo { { .method = method, .hash = hash, }, - {}, + { + .references = references, + .hasSelfReference = false, + }, }, }; @@ -1418,7 +1443,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath - optimisePath(realPath); + optimisePath(realPath, repair); ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; @@ -1432,7 +1457,9 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, } -StorePath LocalStore::addTextToStore(const string & name, const string & s, +StorePath LocalStore::addTextToStore( + std::string_view name, + std::string_view s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); @@ -1461,12 +1488,12 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, StringSink sink; dumpString(s, sink); - auto narHash = hashString(htSHA256, *sink.s); + auto narHash = hashString(htSHA256, sink.s); - optimisePath(realPath); + optimisePath(realPath, repair); ValidPathInfo info { dstPath, narHash }; - info.narSize = sink.s->size(); + info.narSize = sink.s.size(); info.references = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); @@ -1524,7 +1551,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Acquire the global GC lock to get a consistent snapshot of existing and valid paths. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); + auto fdGCLock = openGCLock(); + FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); StringSet store; for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); @@ -1535,8 +1563,6 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) StorePathSet validPaths; PathSet done; - fdGCLock = -1; - for (auto & i : queryAllValidPaths()) verifyPath(printStorePath(i), store, done, validPaths, repair, errors); @@ -1548,7 +1574,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); + std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1849,13 +1875,24 @@ std::optional<const Realisation> LocalStore::queryRealisation_( return { res }; } -std::optional<const Realisation> -LocalStore::queryRealisation(const DrvOutput & id) +void LocalStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { - return retrySQLite<std::optional<const Realisation>>([&]() { - auto state(_state.lock()); - return queryRealisation_(*state, id); - }); + try { + auto maybeRealisation + = retrySQLite<std::optional<const Realisation>>([&]() { + auto state(_state.lock()); + return queryRealisation_(*state, id); + }); + if (maybeRealisation) + callback( + std::make_shared<const Realisation>(maybeRealisation.value())); + else + callback(nullptr); + + } catch (...) { + callback.rethrow(); + } } FixedOutputHash LocalStore::hashCAPath( @@ -1888,4 +1925,30 @@ FixedOutputHash LocalStore::hashCAPath( }; } +void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) +{ + assert(drvPath.isDerivation()); + + auto baseName = drvPath.to_string(); + + auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); + + if (pathExists(logPath)) return; + + createDirs(dirOf(logPath)); + + auto tmpFile = fmt("%s.tmp.%d", logPath, getpid()); + + writeFile(tmpFile, compress("bzip2", log)); + + if (rename(tmpFile.c_str(), logPath.c_str()) != 0) + throw SysError("renaming '%1%' to '%2%'", tmpFile, logPath); +} + +std::optional<std::string> LocalStore::getVersion() +{ + return nixVersion; +} + + } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a01d48c4b..70d225be3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -5,6 +5,7 @@ #include "pathlocks.hh" #include "store-api.hh" #include "local-fs-store.hh" +#include "gc-store.hh" #include "sync.hh" #include "util.hh" @@ -43,7 +44,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig }; -class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore +class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore { private: @@ -58,9 +59,15 @@ private: struct Stmts; std::unique_ptr<Stmts> stmts; + /* The global GC lock */ + AutoCloseFD fdGCLock; + /* The file to which we write our temporary roots. */ AutoCloseFD fdTempRoots; + /* Connection to the garbage collector. */ + AutoCloseFD fdRootsSocket; + /* The last time we checked whether to do an auto-GC, or an auto-GC finished. */ std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; @@ -87,7 +94,6 @@ public: const Path linksDir; const Path reservedPath; const Path schemaPath; - const Path trashDir; const Path tempRootsDir; const Path fnTempRoots; @@ -139,24 +145,24 @@ public: void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + StorePath addToStoreFromDump(Source & dump, std::string_view name, + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) override; + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override; void addTempRoot(const StorePath & path) override; void addIndirectRoot(const Path & path) override; - void syncWithGC() override; - private: - typedef std::shared_ptr<AutoCloseFD> FDPtr; - typedef list<FDPtr> FDs; + void findTempRoots(Roots & roots, bool censor); - void findTempRoots(FDs & fds, Roots & roots, bool censor); + AutoCloseFD openGCLock(); public: @@ -170,8 +176,9 @@ public: void optimiseStore() override; - /* Optimise a single store path. */ - void optimisePath(const Path & path); + /* Optimise a single store path. Optionally, test the encountered + symlinks for corruption. */ + void optimisePath(const Path & path, RepairFlag repair); bool verifyStore(bool checkContents, RepairFlag repair) override; @@ -201,11 +208,18 @@ public: derivation 'deriver'. */ void registerDrvOutput(const Realisation & info) override; void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; - void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); + void cacheDrvOutputMapping( + State & state, + const uint64_t deriver, + const std::string & outputName, + const StorePath & output); std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id); std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id); - std::optional<const Realisation> queryRealisation(const DrvOutput&) override; + void queryRealisationUncached(const DrvOutput&, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; + + std::optional<std::string> getVersion() override; private: @@ -236,29 +250,12 @@ private: PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); - struct GCState; - - void deleteGarbage(GCState & state, const Path & path); - - void tryToDelete(GCState & state, const Path & path); - - bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path); - - void deletePathRecursive(GCState & state, const Path & path); - - bool isActiveTempFile(const GCState & state, - const Path & path, const string & suffix); - - AutoCloseFD openGCLock(LockType lockType); - void findRoots(const Path & path, unsigned char type, Roots & roots); void findRootsNoTemp(Roots & roots, bool censor); void findRuntimeRoots(Roots & roots, bool censor); - void removeUnusedLinks(const GCState & state); - Path createTempDirInStore(); void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); @@ -267,7 +264,7 @@ private: InodeHash loadInodeHash(); Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); - void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash); + void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash, RepairFlag repair); // Internal versions that are not wrapped in retry_sqlite. bool isValidPath_(State & state, const StorePath & path); @@ -293,6 +290,8 @@ private: const std::string_view pathHash ); + void addBuildLog(const StorePath & drvPath, std::string_view log) override; + friend struct LocalDerivationGoal; friend struct PathSubstitutionGoal; friend struct SubstitutionGoal; @@ -301,7 +300,7 @@ private: typedef std::pair<dev_t, ino_t> Inode; -typedef set<Inode> InodesSeen; +typedef std::set<Inode> InodesSeen; /* "Fix", or canonicalise, the meta-data of the files in a store path diff --git a/src/libstore/lock.hh b/src/libstore/lock.hh index 8fbb67ddc..3d29a7b5b 100644 --- a/src/libstore/lock.hh +++ b/src/libstore/lock.hh @@ -13,7 +13,7 @@ private: AutoCloseFD fdUserLock; bool isEnabled = false; - string user; + std::string user; uid_t uid = 0; gid_t gid = 0; std::vector<gid_t> supplementaryGIDs; @@ -23,7 +23,7 @@ public: void kill(); - string getUser() { return user; } + std::string getUser() { return user; } uid_t getUID() { assert(uid); return uid; } uid_t getGID() { assert(gid); return gid; } std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; } diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 9843ccf04..e87f46980 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -39,22 +39,25 @@ Machine::Machine(decltype(storeUri) storeUri, sshPublicHostKey(sshPublicHostKey) {} -bool Machine::allSupported(const std::set<string> & features) const { +bool Machine::allSupported(const std::set<std::string> & features) const +{ return std::all_of(features.begin(), features.end(), - [&](const string & feature) { + [&](const std::string & feature) { return supportedFeatures.count(feature) || mandatoryFeatures.count(feature); }); } -bool Machine::mandatoryMet(const std::set<string> & features) const { +bool Machine::mandatoryMet(const std::set<std::string> & features) const +{ return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), - [&](const string & feature) { + [&](const std::string & feature) { return features.count(feature); }); } -ref<Store> Machine::openStore() const { +ref<Store> Machine::openStore() const +{ Store::Params storeParams; if (hasPrefix(storeUri, "ssh://")) { storeParams["max-connections"] = "1"; @@ -83,53 +86,87 @@ ref<Store> Machine::openStore() const { return nix::openStore(storeUri, storeParams); } -void parseMachines(const std::string & s, Machines & machines) +static std::vector<std::string> expandBuilderLines(const std::string & builders) { - for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { + std::vector<std::string> result; + for (auto line : tokenizeString<std::vector<std::string>>(builders, "\n;")) { trim(line); line.erase(std::find(line.begin(), line.end(), '#'), line.end()); if (line.empty()) continue; if (line[0] == '@') { - auto file = trim(std::string(line, 1)); + const std::string path = trim(std::string(line, 1)); + std::string text; try { - parseMachines(readFile(file), machines); + text = readFile(path); } catch (const SysError & e) { if (e.errNo != ENOENT) throw; - debug("cannot find machines file '%s'", file); + debug("cannot find machines file '%s'", path); } + + const auto lines = expandBuilderLines(text); + result.insert(end(result), begin(lines), end(lines)); continue; } - auto tokens = tokenizeString<std::vector<string>>(line); - auto sz = tokens.size(); - if (sz < 1) - throw FormatError("bad machine specification '%s'", line); + result.emplace_back(line); + } + return result; +} - auto isSet = [&](size_t n) { - return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; - }; +static Machine parseBuilderLine(const std::string & line) +{ + const auto tokens = tokenizeString<std::vector<std::string>>(line); - machines.emplace_back(tokens[0], - isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, - isSet(2) ? tokens[2] : "", - isSet(3) ? std::stoull(tokens[3]) : 1LL, - isSet(4) ? std::stoull(tokens[4]) : 1LL, - isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, - isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, - isSet(7) ? tokens[7] : ""); - } + auto isSet = [&](size_t fieldIndex) { + return tokens.size() > fieldIndex && tokens[fieldIndex] != "" && tokens[fieldIndex] != "-"; + }; + + auto parseUnsignedIntField = [&](size_t fieldIndex) { + const auto result = string2Int<unsigned int>(tokens[fieldIndex]); + if (!result) { + throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'unsigned int'", fieldIndex, line); + } + return result.value(); + }; + + auto ensureBase64 = [&](size_t fieldIndex) { + const auto & str = tokens[fieldIndex]; + try { + base64Decode(str); + } catch (const Error & e) { + throw FormatError("bad machine specification: a column #%lu in a row: '%s' is not valid base64 string: %s", fieldIndex, line, e.what()); + } + return str; + }; + + if (!isSet(0)) + throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line); + + return { + tokens[0], + isSet(1) ? tokenizeString<std::vector<std::string>>(tokens[1], ",") : std::vector<std::string>{settings.thisSystem}, + isSet(2) ? tokens[2] : "", + isSet(3) ? parseUnsignedIntField(3) : 1U, + isSet(4) ? parseUnsignedIntField(4) : 1U, + isSet(5) ? tokenizeString<std::set<std::string>>(tokens[5], ",") : std::set<std::string>{}, + isSet(6) ? tokenizeString<std::set<std::string>>(tokens[6], ",") : std::set<std::string>{}, + isSet(7) ? ensureBase64(7) : "" + }; +} + +static Machines parseBuilderLines(const std::vector<std::string> & builders) +{ + Machines result; + std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine); + return result; } Machines getMachines() { - static auto machines = [&]() { - Machines machines; - parseMachines(settings.builders, machines); - return machines; - }(); - return machines; + const auto builderLines = expandBuilderLines(settings.builders); + return parseBuilderLines(builderLines); } } diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 341d9bd97..834626de9 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -8,19 +8,19 @@ class Store; struct Machine { - const string storeUri; - const std::vector<string> systemTypes; - const string sshKey; + const std::string storeUri; + const std::vector<std::string> systemTypes; + const std::string sshKey; const unsigned int maxJobs; const unsigned int speedFactor; - const std::set<string> supportedFeatures; - const std::set<string> mandatoryFeatures; + const std::set<std::string> supportedFeatures; + const std::set<std::string> mandatoryFeatures; const std::string sshPublicHostKey; bool enabled = true; - bool allSupported(const std::set<string> & features) const; + bool allSupported(const std::set<std::string> & features) const; - bool mandatoryMet(const std::set<string> & features) const; + bool mandatoryMet(const std::set<std::string> & features) const; Machine(decltype(storeUri) storeUri, decltype(systemTypes) systemTypes, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 6a98934ef..e411cbe04 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -7,6 +7,7 @@ #include "topo-sort.hh" #include "callback.hh" #include "closure.hh" +#include "filetransfer.hh" namespace nix { @@ -111,7 +112,8 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets, downloadSize_ = narSize_ = 0; - ThreadPool pool; + // FIXME: make async. + ThreadPool pool(fileTransferSettings.httpConnections); struct State { @@ -250,12 +252,11 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) { return topoSort(paths, {[&](const StorePath & path) { - StorePathSet references; try { - references = queryPathInfo(path)->references; + return queryPathInfo(path)->references; } catch (InvalidPath &) { + return StorePathSet(); } - return references; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( diff --git a/src/libstore/names.cc b/src/libstore/names.cc index ce808accc..277aabf0f 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -42,7 +42,7 @@ DrvName::~DrvName() { } -bool DrvName::matches(DrvName & n) +bool DrvName::matches(const DrvName & n) { if (name != "*") { if (!regex) { @@ -56,8 +56,8 @@ bool DrvName::matches(DrvName & n) } -string nextComponent(string::const_iterator & p, - const string::const_iterator end) +std::string_view nextComponent(std::string_view::const_iterator & p, + const std::string_view::const_iterator end) { /* Skip any dots and dashes (component separators). */ while (p != end && (*p == '.' || *p == '-')) ++p; @@ -67,18 +67,18 @@ string nextComponent(string::const_iterator & p, /* If the first character is a digit, consume the longest sequence of digits. Otherwise, consume the longest sequence of non-digit, non-separator characters. */ - string s; + auto s = p; if (isdigit(*p)) - while (p != end && isdigit(*p)) s += *p++; + while (p != end && isdigit(*p)) p++; else while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) - s += *p++; + p++; - return s; + return {s, size_t(p - s)}; } -static bool componentsLT(const string & c1, const string & c2) +static bool componentsLT(const std::string_view c1, const std::string_view c2) { auto n1 = string2Int<int>(c1); auto n2 = string2Int<int>(c2); @@ -94,14 +94,14 @@ static bool componentsLT(const string & c1, const string & c2) } -int compareVersions(const string & v1, const string & v2) +int compareVersions(const std::string_view v1, const std::string_view v2) { - string::const_iterator p1 = v1.begin(); - string::const_iterator p2 = v2.begin(); + auto p1 = v1.begin(); + auto p2 = v2.begin(); while (p1 != v1.end() || p2 != v2.end()) { - string c1 = nextComponent(p1, v1.end()); - string c2 = nextComponent(p2, v2.end()); + auto c1 = nextComponent(p1, v1.end()); + auto c2 = nextComponent(p2, v2.end()); if (componentsLT(c1, c2)) return -1; else if (componentsLT(c2, c1)) return 1; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index bc62aac93..3977fc6cc 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -10,26 +10,26 @@ struct Regex; struct DrvName { - string fullName; - string name; - string version; + std::string fullName; + std::string name; + std::string version; unsigned int hits; DrvName(); DrvName(std::string_view s); ~DrvName(); - bool matches(DrvName & n); + bool matches(const DrvName & n); private: std::unique_ptr<Regex> regex; }; -typedef list<DrvName> DrvNames; +typedef std::list<DrvName> DrvNames; -string nextComponent(string::const_iterator & p, - const string::const_iterator end); -int compareVersions(const string & v1, const string & v2); +std::string_view nextComponent(std::string_view::const_iterator & p, + const std::string_view::const_iterator end); +int compareVersions(const std::string_view v1, const std::string_view v2); DrvNames drvNamesFromArgs(const Strings & opArgs); } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 784ebb719..72d41cc94 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -28,7 +28,7 @@ struct NarMember struct NarAccessor : public FSAccessor { - std::shared_ptr<const std::string> nar; + std::optional<const std::string> nar; GetNarBytes getNarBytes; @@ -90,7 +90,7 @@ struct NarAccessor : public FSAccessor void receiveContents(std::string_view data) override { } - void createSymlink(const Path & path, const string & target) override + void createSymlink(const Path & path, const std::string & target) override { createMember(path, NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); @@ -104,7 +104,7 @@ struct NarAccessor : public FSAccessor } }; - NarAccessor(ref<const std::string> nar) : nar(nar) + NarAccessor(std::string && _nar) : nar(_nar) { StringSource source(*nar); NarIndexer indexer(*this, source); @@ -224,9 +224,9 @@ struct NarAccessor : public FSAccessor } }; -ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) +ref<FSAccessor> makeNarAccessor(std::string && nar) { - return make_ref<NarAccessor>(nar); + return make_ref<NarAccessor>(std::move(nar)); } ref<FSAccessor> makeNarAccessor(Source & source) diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 8af1272f6..c2241a04c 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -10,7 +10,7 @@ struct Source; /* Return an object that provides access to the contents of a NAR file. */ -ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); +ref<FSAccessor> makeNarAccessor(std::string && nar); ref<FSAccessor> makeNarAccessor(Source & source); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 4ea30bf47..2d53a449c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -11,7 +11,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & return Error("NAR info file '%1%' is corrupt", whence); }; - auto parseHashField = [&](const string & s) { + auto parseHashField = [&](const std::string & s) { try { return Hash::parseAnyPrefixed(s); } catch (BadHash &) { diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index d95e54af1..8af9b1dde 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -26,7 +26,7 @@ static void makeWritable(const Path & path) struct MakeReadOnly { Path path; - MakeReadOnly(const Path & path) : path(path) { } + MakeReadOnly(const PathView path) : path(path) { } ~MakeReadOnly() { try { @@ -77,7 +77,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa continue; } - string name = dirent->d_name; + std::string name = dirent->d_name; if (name == "." || name == "..") continue; names.push_back(name); } @@ -88,7 +88,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, - const Path & path, InodeHash & inodeHash) + const Path & path, InodeHash & inodeHash, RepairFlag repair) { checkInterrupt(); @@ -110,7 +110,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, if (S_ISDIR(st.st_mode)) { Strings names = readDirectoryIgnoringInodes(path, inodeHash); for (auto & i : names) - optimisePath_(act, stats, path + "/" + i, inodeHash); + optimisePath_(act, stats, path + "/" + i, inodeHash, repair); return; } @@ -151,7 +151,20 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Check if this is a known hash. */ Path linkPath = linksDir + "/" + hash.to_string(Base32, false); - retry: + /* Maybe delete the link, if it has been corrupted. */ + if (pathExists(linkPath)) { + auto stLink = lstat(linkPath); + if (st.st_size != stLink.st_size + || (repair && hash != hashPath(htSHA256, linkPath).first)) + { + // XXX: Consider overwriting linkPath with our valid version. + warn("removing corrupted link '%s'", linkPath); + warn("There may be more corrupted paths." + "\nYou should run `nix-store --verify --check-contents --repair` to fix them all"); + unlink(linkPath.c_str()); + } + } + if (!pathExists(linkPath)) { /* Nope, create a hard link in the links directory. */ if (link(path.c_str(), linkPath.c_str()) == 0) { @@ -187,23 +200,18 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, return; } - if (st.st_size != stLink.st_size) { - warn("removing corrupted link '%s'", linkPath); - unlink(linkPath.c_str()); - goto retry; - } - printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its permissions). */ - bool mustToggle = dirOf(path) != realStoreDir.get(); - if (mustToggle) makeWritable(dirOf(path)); + const Path dirOfPath(dirOf(path)); + bool mustToggle = dirOfPath != realStoreDir.get(); + if (mustToggle) makeWritable(dirOfPath); /* When we're done, make the directory read-only again and reset its timestamp back to 0. */ - MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); Path tempLink = (format("%1%/.tmp-link-%2%-%3%") % realStoreDir % getpid() % random()).str(); @@ -260,7 +268,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) if (!isValidPath(i)) continue; /* path was GC'ed, probably */ { Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); - optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash); + optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); } done++; act.progress(done, paths.size()); @@ -278,12 +286,12 @@ void LocalStore::optimiseStore() stats.filesLinked); } -void LocalStore::optimisePath(const Path & path) +void LocalStore::optimisePath(const Path & path, RepairFlag repair) { OptimiseStats stats; InodeHash inodeHash; - if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash); + if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash, repair); } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index caddba9b1..8c65053e4 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -170,7 +170,7 @@ std::string writeStructuredAttrsShell(const nlohmann::json & json) auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> { if (value.is_string()) - return shellEscape(value); + return shellEscape(value.get<std::string_view>()); if (value.is_number()) { auto f = value.get<float>(); diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index e5a121e00..078c117bd 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -22,9 +22,9 @@ DerivedPath StorePathWithOutputs::toDerivedPath() const std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> ss) { - std::vector<DerivedPath> reqs; - for (auto & s : ss) reqs.push_back(s.toDerivedPath()); - return reqs; + std::vector<DerivedPath> reqs; + for (auto & s : ss) reqs.push_back(s.toDerivedPath()); + return reqs; } @@ -49,9 +49,9 @@ std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s) { size_t n = s.find("!"); return n == s.npos - ? std::make_pair(s, std::set<string>()) + ? std::make_pair(s, std::set<std::string>()) : std::make_pair(((std::string_view) s).substr(0, n), - tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ",")); + tokenizeString<std::set<std::string>>(((std::string_view) s).substr(n + 1), ",")); } diff --git a/src/libstore/path.hh b/src/libstore/path.hh index a152fe8ee..6d91b8e7b 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -63,7 +63,7 @@ public: typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; -typedef std::map<string, StorePath> OutputPathMap; +typedef std::map<std::string, StorePath> OutputPathMap; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 926f4ea1e..42023cd0a 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -74,7 +74,7 @@ PathLocks::PathLocks() } -PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) +PathLocks::PathLocks(const PathSet & paths, const std::string & waitMsg) : deletePaths(false) { lockPaths(paths, waitMsg); @@ -82,7 +82,7 @@ PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) bool PathLocks::lockPaths(const PathSet & paths, - const string & waitMsg, bool wait) + const std::string & waitMsg, bool wait) { assert(fds.empty()); @@ -176,4 +176,17 @@ void PathLocks::setDeletion(bool deletePaths) } +FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg) + : fd(fd) +{ + if (wait) { + if (!lockFile(fd, lockType, false)) { + printInfo("%s", waitMsg); + acquired = lockFile(fd, lockType, true); + } + } else + acquired = lockFile(fd, lockType, false); +} + + } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 411da0222..5e3a734b4 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -20,19 +20,33 @@ class PathLocks { private: typedef std::pair<int, Path> FDPair; - list<FDPair> fds; + std::list<FDPair> fds; bool deletePaths; public: PathLocks(); PathLocks(const PathSet & paths, - const string & waitMsg = ""); + const std::string & waitMsg = ""); bool lockPaths(const PathSet & _paths, - const string & waitMsg = "", + const std::string & waitMsg = "", bool wait = true); ~PathLocks(); void unlock(); void setDeletion(bool deletePaths); }; +struct FdLock +{ + int fd; + bool acquired = false; + + FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg); + + ~FdLock() + { + if (acquired) + lockFile(fd, ltNone, false); + } +}; + } diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 73163424c..3e4188188 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -15,12 +15,12 @@ namespace nix { /* Parse a generation name of the format `<profilename>-<number>-link'. */ -static std::optional<GenerationNumber> parseName(const string & profileName, const string & name) +static std::optional<GenerationNumber> parseName(const std::string & profileName, const std::string & name) { - if (string(name, 0, profileName.size() + 1) != profileName + "-") return {}; - string s = string(name, profileName.size() + 1); - string::size_type p = s.find("-link"); - if (p == string::npos) return {}; + if (name.substr(0, profileName.size() + 1) != profileName + "-") return {}; + auto s = name.substr(profileName.size() + 1); + auto p = s.find("-link"); + if (p == std::string::npos) return {}; if (auto n = string2Int<unsigned int>(s.substr(0, p))) return *n; else @@ -209,13 +209,13 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) } -void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun) +void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun) { if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd') throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec); time_t curTime = time(0); - string strDays = string(timeSpec, 0, timeSpec.size() - 1); + auto strDays = timeSpec.substr(0, timeSpec.size() - 1); auto days = string2Int<int>(strDays); if (!days || *days < 1) @@ -274,7 +274,7 @@ void lockProfile(PathLocks & lock, const Path & profile) } -string optimisticLockProfile(const Path & profile) +std::string optimisticLockProfile(const Path & profile) { return pathExists(profile) ? readLink(profile) : ""; } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index d100c970c..408ca039c 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -42,7 +42,7 @@ void deleteOldGenerations(const Path & profile, bool dryRun); void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun); -void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun); +void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun); void switchLink(Path link, Path target); @@ -66,7 +66,7 @@ void lockProfile(PathLocks & lock, const Path & profile); generally cheap, since the build results are still in the Nix store. Most of the time, only the user environment has to be rebuilt. */ -string optimisticLockProfile(const Path & profile); +std::string optimisticLockProfile(const Path & profile); /* Resolve ~/.nix-profile. If ~/.nix-profile doesn't exist yet, create it. */ diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index f871e6437..d63ec5ea2 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -78,7 +78,7 @@ Realisation Realisation::fromJSON( auto fieldIterator = json.find(fieldName); if (fieldIterator == json.end()) return std::nullopt; - return *fieldIterator; + return {*fieldIterator}; }; auto getField = [&](std::string fieldName) -> std::string { if (auto field = getOptionalField(fieldName)) diff --git a/src/libstore/references.cc b/src/libstore/references.cc index c369b14ac..34dce092c 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -54,12 +54,12 @@ void RefScanSink::operator () (std::string_view data) fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ auto s = tail; - s.append(data.data(), refLength); + auto tailLen = std::min(data.size(), refLength); + s.append(data.data(), tailLen); search(s, hashes, seen); search(data, hashes, seen); - auto tailLen = std::min(data.size(), refLength); auto rest = refLength - tailLen; if (rest < tail.size()) tail = tail.substr(tail.size() - rest); @@ -68,7 +68,7 @@ void RefScanSink::operator () (std::string_view data) std::pair<StorePathSet, HashResult> scanForReferences( - const string & path, + const std::string & path, const StorePathSet & refs) { HashSink hashSink { htSHA256 }; @@ -121,7 +121,7 @@ void RewritingSink::operator () (std::string_view data) s.append(data); size_t j = 0; - while ((j = s.find(from, j)) != string::npos) { + while ((j = s.find(from, j)) != std::string::npos) { matches.push_back(pos + j); s.replace(j, from.size(), to); } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index f43456f0b..0ce335646 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -22,9 +22,18 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & nar, - ref<FSAccessor> narAccessor) +ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) { + if (cacheDir != "") { + try { + /* FIXME: do this asynchronously. */ + writeFile(makeCacheFile(hashPart, "nar"), nar); + } catch (...) { + ignoreException(); + } + } + + auto narAccessor = makeNarAccessor(std::move(nar)); nars.emplace(hashPart, narAccessor); if (cacheDir != "") { @@ -33,14 +42,12 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & JSONPlaceholder jsonRoot(str); listNar(jsonRoot, narAccessor, "", true); writeFile(makeCacheFile(hashPart, "ls"), str.str()); - - /* FIXME: do this asynchronously. */ - writeFile(makeCacheFile(hashPart, "nar"), nar); - } catch (...) { ignoreException(); } } + + return narAccessor; } std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) @@ -55,7 +62,6 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo auto i = nars.find(std::string(storePath.hashPart())); if (i != nars.end()) return {i->second, restPath}; - StringSink sink; std::string listing; Path cacheFile; @@ -86,19 +92,15 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, boo } catch (SysError &) { } try { - *sink.s = nix::readFile(cacheFile); - - auto narAccessor = makeNarAccessor(sink.s); + auto narAccessor = makeNarAccessor(nix::readFile(cacheFile)); nars.emplace(storePath.hashPart(), narAccessor); return {narAccessor, restPath}; - } catch (SysError &) { } } + StringSink sink; store->narFromPath(storePath, sink); - auto narAccessor = makeNarAccessor(sink.s); - addToCache(storePath.hashPart(), *sink.s, narAccessor); - return {narAccessor, restPath}; + return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 594852d0e..99f5544ef 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -20,8 +20,7 @@ class RemoteFSAccessor : public FSAccessor Path makeCacheFile(std::string_view hashPart, const std::string & ext); - void addToCache(std::string_view hashPart, const std::string & nar, - ref<FSAccessor> narAccessor); + ref<FSAccessor> addToCache(std::string_view hashPart, std::string && nar); public: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0df432898..585fc5400 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,11 +1,12 @@ #include "serialise.hh" #include "util.hh" #include "path-with-outputs.hh" +#include "gc-store.hh" #include "remote-fs-accessor.hh" +#include "build-result.hh" #include "remote-store.hh" #include "worker-protocol.hh" #include "archive.hh" -#include "affinity.hh" #include "globals.hh" #include "derivations.hh" #include "pool.hh" @@ -90,6 +91,35 @@ void write(const Store & store, Sink & out, const DrvOutput & drvOutput) } +BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _) +{ + auto path = worker_proto::read(store, from, Phantom<DerivedPath> {}); + BuildResult res { .path = path }; + res.status = (BuildResult::Status) readInt(from); + from + >> res.errorMsg + >> res.timesBuilt + >> res.isNonDeterministic + >> res.startTime + >> res.stopTime; + res.builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {}); + return res; +} + +void write(const Store & store, Sink & to, const BuildResult & res) +{ + worker_proto::write(store, to, res.path); + to + << res.status + << res.errorMsg + << res.timesBuilt + << res.isNonDeterministic + << res.startTime + << res.stopTime; + worker_proto::write(store, to, res.builtOutputs); +} + + std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _) { auto s = readString(from); @@ -173,7 +203,7 @@ void RemoteStore::initConnection(Connection & conn) it. */ conn.closeWrite(); auto msg = conn.from.drain(); - throw Error("protocol mismatch, got '%s'", chomp(*saved.s + msg)); + throw Error("protocol mismatch, got '%s'", chomp(saved.s + msg)); } conn.from >> conn.daemonVersion; @@ -184,15 +214,17 @@ void RemoteStore::initConnection(Connection & conn) conn.to << PROTOCOL_VERSION; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; - if (cpu != -1) - conn.to << 1 << cpu; - else - conn.to << 0; + // Obsolete CPU affinity. + conn.to << 0; } if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) - conn.to << false; + conn.to << false; // obsolete reserveSpace + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 33) { + conn.to.flush(); + conn.daemonNixVersion = readString(conn.from); + } auto ex = conn.processStderr(); if (ex) std::rethrow_exception(ex); @@ -290,6 +322,10 @@ ConnectionHandle RemoteStore::getConnection() return ConnectionHandle(connections->get()); } +void RemoteStore::setOptions() +{ + setOptions(*(getConnection().handle)); +} bool RemoteStore::isValidPathUncached(const StorePath & path) { @@ -496,7 +532,7 @@ std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & ref<const ValidPathInfo> RemoteStore::addCAToStore( Source & dump, - const string & name, + std::string_view name, ContentAddressMethod caMethod, HashType hashType, const StorePathSet & references, @@ -582,10 +618,9 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( } -StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashType, RepairFlag repair) +StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, + FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) { - StorePathSet references; return addCAToStore(dump, name, method, hashType, references, repair)->path; } @@ -662,8 +697,11 @@ void RemoteStore::addMultipleToStore( } -StorePath RemoteStore::addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) +StorePath RemoteStore::addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) { StringSource source(s); return addCAToStore(source, name, TextHashMethod {}, htSHA256, references, repair)->path; @@ -682,23 +720,41 @@ void RemoteStore::registerDrvOutput(const Realisation & info) conn.processStderr(); } -std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id) +void RemoteStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); + try { + callback(nullptr); + } catch (...) { return callback.rethrow(); } + } + conn->to << wopQueryRealisation; conn->to << id.to_string(); conn.processStderr(); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { - auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{}); - if (outPaths.empty()) - return std::nullopt; - return {Realisation{.id = id, .outPath = *outPaths.begin()}}; - } else { - auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{}); - if (realisations.empty()) - return std::nullopt; - return *realisations.begin(); - } + + auto real = [&]() -> std::shared_ptr<const Realisation> { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + auto outPaths = worker_proto::read( + *this, conn->from, Phantom<std::set<StorePath>> {}); + if (outPaths.empty()) + return nullptr; + return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() }); + } else { + auto realisations = worker_proto::read( + *this, conn->from, Phantom<std::set<Realisation>> {}); + if (realisations.empty()) + return nullptr; + return std::make_shared<const Realisation>(*realisations.begin()); + } + }(); + + try { + callback(std::shared_ptr<const Realisation>(real)); + } catch (...) { return callback.rethrow(); } } static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs) @@ -725,17 +781,24 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons } } -void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) +void RemoteStore::copyDrvsFromEvalStore( + const std::vector<DerivedPath> & paths, + std::shared_ptr<Store> evalStore) { if (evalStore && evalStore.get() != this) { /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ RealisedPath::Set drvPaths2; - for (auto & i : drvPaths) + for (auto & i : paths) if (auto p = std::get_if<DerivedPath::Built>(&i)) drvPaths2.insert(p->drvPath); copyClosure(*evalStore, *this, drvPaths2); } +} + +void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) +{ + copyDrvsFromEvalStore(drvPaths, evalStore); auto conn(getConnection()); conn->to << wopBuildPaths; @@ -752,6 +815,91 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod readInt(conn->from); } +std::vector<BuildResult> RemoteStore::buildPathsWithResults( + const std::vector<DerivedPath> & paths, + BuildMode buildMode, + std::shared_ptr<Store> evalStore) +{ + copyDrvsFromEvalStore(paths, evalStore); + + std::optional<ConnectionHandle> conn_(getConnection()); + auto & conn = *conn_; + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { + conn->to << wopBuildPathsWithResults; + writeDerivedPaths(*this, conn, paths); + conn->to << buildMode; + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom<std::vector<BuildResult>> {}); + } else { + // Avoid deadlock. + conn_.reset(); + + // Note: this throws an exception if a build/substitution + // fails, but meh. + buildPaths(paths, buildMode, evalStore); + + std::vector<BuildResult> results; + + for (auto & path : paths) { + std::visit( + overloaded { + [&](const DerivedPath::Opaque & bo) { + results.push_back(BuildResult { + .status = BuildResult::Substituted, + .path = bo, + }); + }, + [&](const DerivedPath::Built & bfd) { + BuildResult res { + .status = BuildResult::Built, + .path = bfd, + }; + + OutputPathMap outputs; + auto drv = evalStore->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive + auto drvOutputs = drv.outputsAndOptPaths(*this); + for (auto & output : bfd.outputs) { + if (!outputHashes.count(output)) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + printStorePath(bfd.drvPath), output); + auto outputId = + DrvOutput{outputHashes.at(output), output}; + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + auto realisation = + queryRealisation(outputId); + if (!realisation) + throw Error( + "cannot operate on an output of unbuilt " + "content-addressed derivation '%s'", + outputId.to_string()); + res.builtOutputs.emplace(realisation->id, *realisation); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + assert(drvOutputs.count(output)); + assert(drvOutputs.at(output).second); + res.builtOutputs.emplace( + outputId, + Realisation { + .id = outputId, + .outPath = *drvOutputs.at(output).second + }); + } + } + + results.push_back(res); + } + }, + path.raw()); + } + + return results; + } +} + BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) @@ -761,7 +909,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res; + BuildResult res { .path = DerivedPath::Built { .drvPath = drvPath } }; res.status = (BuildResult::Status) readInt(conn->from); conn->from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { @@ -802,15 +950,6 @@ void RemoteStore::addIndirectRoot(const Path & path) } -void RemoteStore::syncWithGC() -{ - auto conn(getConnection()); - conn->to << wopSyncWithGC; - conn.processStderr(); - readInt(conn->from); -} - - Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); @@ -905,6 +1044,25 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets, } +void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log) +{ + auto conn(getConnection()); + conn->to << wopAddBuildLog << drvPath.to_string(); + StringSource source(log); + conn.withFramedSink([&](Sink & sink) { + source.drainInto(sink); + }); + readInt(conn->from); +} + + +std::optional<std::string> RemoteStore::getVersion() +{ + auto conn(getConnection()); + return conn->daemonNixVersion; +} + + void RemoteStore::connect() { auto conn(getConnection()); @@ -973,7 +1131,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * auto msg = readNum<uint64_t>(from); if (msg == STDERR_WRITE) { - string s = readString(from); + auto s = readString(from); if (!sink) throw Error("no sink"); (*sink)(s); } @@ -990,7 +1148,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { return std::make_exception_ptr(readError(from)); } else { - string error = readString(from); + auto error = readString(from); unsigned int status = readInt(from); return std::make_exception_ptr(Error(status, error)); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 880be21bf..eb74d88c9 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -4,6 +4,7 @@ #include <string> #include "store-api.hh" +#include "gc-store.hh" namespace nix { @@ -29,7 +30,7 @@ struct RemoteStoreConfig : virtual StoreConfig /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public virtual RemoteStoreConfig, public virtual Store +class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore { public: @@ -66,15 +67,15 @@ public: /* Add a content-addressable store path. `dump` will be drained. */ ref<const ValidPathInfo> addCAToStore( Source & dump, - const string & name, + std::string_view name, ContentAddressMethod caMethod, HashType hashType, const StorePathSet & references, RepairFlag repair); /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ - StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + StorePath addToStoreFromDump(Source & dump, std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; @@ -84,15 +85,24 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; - StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair) override; + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override; void registerDrvOutput(const Realisation & info) override; - std::optional<const Realisation> queryRealisation(const DrvOutput &) override; + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override; + std::vector<BuildResult> buildPathsWithResults( + const std::vector<DerivedPath> & paths, + BuildMode buildMode, + std::shared_ptr<Store> evalStore) override; + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -102,8 +112,6 @@ public: void addIndirectRoot(const Path & path) override; - void syncWithGC() override; - Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -118,6 +126,10 @@ public: StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, uint64_t & downloadSize, uint64_t & narSize) override; + void addBuildLog(const StorePath & drvPath, std::string_view log) override; + + std::optional<std::string> getVersion() override; + void connect() override; unsigned int getProtocol() override; @@ -129,6 +141,7 @@ public: FdSink to; FdSource from; unsigned int daemonVersion; + std::optional<std::string> daemonNixVersion; std::chrono::time_point<std::chrono::steady_clock> startTime; virtual ~Connection(); @@ -150,6 +163,8 @@ protected: virtual void setOptions(Connection & conn); + void setOptions() override; + ConnectionHandle getConnection(); friend struct ConnectionHandle; @@ -162,6 +177,9 @@ private: std::atomic_bool failed{false}; + void copyDrvsFromEvalStore( + const std::vector<DerivedPath> & paths, + std::shared_ptr<Store> evalStore); }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 7accad7f4..844553ad3 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -87,7 +87,11 @@ static void initAWS() }); } -S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint) +S3Helper::S3Helper( + const std::string & profile, + const std::string & region, + const std::string & scheme, + const std::string & endpoint) : config(makeConfig(region, scheme, endpoint)) , client(make_ref<Aws::S3::S3Client>( profile == "" @@ -121,7 +125,10 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy } }; -ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint) +ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig( + const std::string & region, + const std::string & scheme, + const std::string & endpoint) { initAWS(); auto res = make_ref<Aws::Client::ClientConfiguration>(); @@ -385,7 +392,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto compress = [&](std::string compression) { auto compressed = nix::compress(compression, StreamToSourceAdapter(istream).drain()); - return std::make_shared<std::stringstream>(std::move(*compressed)); + return std::make_shared<std::stringstream>(std::move(compressed)); }; if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index 2042bffcf..3f55c74db 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -4,6 +4,8 @@ #include "ref.hh" +#include <optional> + namespace Aws { namespace Client { class ClientConfiguration; } } namespace Aws { namespace S3 { class S3Client; } } @@ -20,7 +22,7 @@ struct S3Helper struct FileTransferResult { - std::shared_ptr<std::string> data; + std::optional<std::string> data; unsigned int durationMs; }; diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 41893e6dd..56b35c3fe 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -100,4 +100,5 @@ ; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin. (allow file-read* - (subpath "/Library/Apple/usr/libexec/oah")) + (subpath "/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/Apple/usr/libexec/oah")) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 447b4179b..e6ecadd7f 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,4 +1,5 @@ #include "sqlite.hh" +#include "globals.hh" #include "util.hh" #include <sqlite3.h> @@ -27,8 +28,12 @@ namespace nix { SQLite::SQLite(const Path & path, bool create) { + // useSQLiteWAL also indicates what virtual file system we need. Using + // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem + // for Linux (WSL) where useSQLiteWAL should be false by default. + const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK) throw Error("cannot open SQLite database '%s'", path); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) @@ -66,7 +71,7 @@ uint64_t SQLite::getLastInsertedRowId() return sqlite3_last_insert_rowid(db); } -void SQLiteStmt::create(sqlite3 * db, const string & sql) +void SQLiteStmt::create(sqlite3 * db, const std::string & sql) { checkInterrupt(); assert(!stmt); diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 93f72675d..1bbad71f2 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -29,7 +29,7 @@ void SSHMaster::addCommonSSHOpts(Strings & args) if (!sshPublicHostKey.empty()) { Path fileName = (Path) *state->tmpDir + "/host-key"; auto p = host.rfind("@"); - string thost = p != string::npos ? string(host, p + 1) : host; + std::string thost = p != std::string::npos ? std::string(host, p + 1) : host; writeFile(fileName, thost + " " + base64Decode(sshPublicHostKey) + "\n"); args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName}); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a57eab3e6..6f4b6b98c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -40,7 +40,7 @@ Path Store::followLinksToStore(std::string_view _path) const Path path = absPath(std::string(_path)); while (!isInStore(path)) { if (!isLink(path)) break; - string target = readLink(path); + auto target = readLink(path); path = absPath(target, dirOf(path)); } if (!isInStore(path)) @@ -139,8 +139,8 @@ StorePath Store::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = std::string { type } + ":" + std::string { hash } - + ":" + storeDir + ":" + std::string { name }; + auto s = std::string(type) + ":" + std::string(hash) + + ":" + storeDir + ":" + std::string(name); auto h = compressHash(hashString(htSHA256, s), 20); return StorePath(h, name); } @@ -165,7 +165,7 @@ StorePath Store::makeOutputPath(std::string_view id, ambiguous. */ static std::string makeType( const Store & store, - string && type, + std::string && type, const PathReferences<StorePath> & references) { for (auto & i : references.references) { @@ -235,7 +235,9 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, } -StorePath Store::computeStorePathForText(const string & name, const string & s, +StorePath Store::computeStorePathForText( + std::string_view name, + std::string_view s, const StorePathSet & references) const { return makeTextPath(name, TextInfo { @@ -245,8 +247,14 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, } -StorePath Store::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) +StorePath Store::addToStore( + std::string_view name, + const Path & _srcPath, + FileIngestionMethod method, + HashType hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) { Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { @@ -255,7 +263,7 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath, else readFile(srcPath, sink); }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair); + return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); } @@ -373,8 +381,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, StringSet StoreConfig::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); - if (settings.isExperimentalFeatureEnabled("ca-derivations")) + + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) res.insert("ca-derivations"); + + if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix)) + res.insert("recursive-nix"); + return res; } @@ -432,11 +445,9 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) bool Store::isValidPath(const StorePath & storePath) { - std::string hashPart(storePath.hashPart()); - { auto state_(state.lock()); - auto res = state_->pathInfoCache.get(hashPart); + auto res = state_->pathInfoCache.get(std::string(storePath.to_string())); if (res && res->isKnownNow()) { stats.narInfoReadAverted++; return res->didExist(); @@ -444,11 +455,11 @@ bool Store::isValidPath(const StorePath & storePath) } if (diskCache) { - auto res = diskCache->lookupNarInfo(getUri(), hashPart); + auto res = diskCache->lookupNarInfo(getUri(), std::string(storePath.hashPart())); if (res.first != NarInfoDiskCache::oUnknown) { stats.narInfoReadAverted++; auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, + state_->pathInfoCache.upsert(std::string(storePath.to_string()), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second }); return res.first == NarInfoDiskCache::oValid; } @@ -458,7 +469,7 @@ bool Store::isValidPath(const StorePath & storePath) if (diskCache && !valid) // FIXME: handle valid = true case. - diskCache->upsertNarInfo(getUri(), hashPart, 0); + diskCache->upsertNarInfo(getUri(), std::string(storePath.hashPart()), 0); return valid; } @@ -505,13 +516,11 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual) void Store::queryPathInfo(const StorePath & storePath, Callback<ref<const ValidPathInfo>> callback) noexcept { - std::string hashPart; + auto hashPart = std::string(storePath.hashPart()); try { - hashPart = storePath.hashPart(); - { - auto res = state.lock()->pathInfoCache.get(hashPart); + auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string())); if (res && res->isKnownNow()) { stats.narInfoReadAverted++; if (!res->didExist()) @@ -526,7 +535,7 @@ void Store::queryPathInfo(const StorePath & storePath, stats.narInfoReadAverted++; { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, + state_->pathInfoCache.upsert(std::string(storePath.to_string()), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || !goodStorePath(storePath, res.second->path)) @@ -551,7 +560,7 @@ void Store::queryPathInfo(const StorePath & storePath, { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); + state_->pathInfoCache.upsert(std::string(storePath.to_string()), PathInfoCacheValue { .value = info }); } if (!info || !goodStorePath(storePath, info->path)) { @@ -564,6 +573,74 @@ void Store::queryPathInfo(const StorePath & storePath, }}); } +void Store::queryRealisation(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept +{ + + try { + if (diskCache) { + auto [cacheOutcome, maybeCachedRealisation] + = diskCache->lookupRealisation(getUri(), id); + switch (cacheOutcome) { + case NarInfoDiskCache::oValid: + debug("Returning a cached realisation for %s", id.to_string()); + callback(maybeCachedRealisation); + return; + case NarInfoDiskCache::oInvalid: + debug( + "Returning a cached missing realisation for %s", + id.to_string()); + callback(nullptr); + return; + case NarInfoDiskCache::oUnknown: + break; + } + } + } catch (...) { + return callback.rethrow(); + } + + auto callbackPtr + = std::make_shared<decltype(callback)>(std::move(callback)); + + queryRealisationUncached( + id, + { [this, id, callbackPtr]( + std::future<std::shared_ptr<const Realisation>> fut) { + try { + auto info = fut.get(); + + if (diskCache) { + if (info) + diskCache->upsertRealisation(getUri(), *info); + else + diskCache->upsertAbsentRealisation(getUri(), id); + } + + (*callbackPtr)(std::shared_ptr<const Realisation>(info)); + + } catch (...) { + callbackPtr->rethrow(); + } + } }); +} + +std::shared_ptr<const Realisation> Store::queryRealisation(const DrvOutput & id) +{ + using RealPtr = std::shared_ptr<const Realisation>; + std::promise<RealPtr> promise; + + queryRealisation(id, + {[&](std::future<RealPtr> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + + return promise.get_future().get(); +} void Store::substitutePaths(const StorePathSet & paths) { @@ -637,10 +714,10 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string Store::makeValidityRegistration(const StorePathSet & paths, +std::string Store::makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash) { - string s = ""; + std::string s = ""; for (auto & i : paths) { s += printStorePath(i) + "\n"; @@ -710,11 +787,11 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store for (auto & storePath : storePaths) { auto jsonPath = jsonList.object(); - jsonPath.attr("path", printStorePath(storePath)); try { auto info = queryPathInfo(storePath); + jsonPath.attr("path", printStorePath(info->path)); jsonPath .attr("narHash", info->narHash.to_string(hashBase, true)) .attr("narSize", info->narSize); @@ -768,6 +845,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store } } catch (InvalidPath &) { + jsonPath.attr("path", printStorePath(storePath)); jsonPath.attr("valid", false); } } @@ -877,7 +955,7 @@ std::map<StorePath, StorePath> copyPaths( for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if<Realisation>(&path.raw)) { - settings.requireExperimentalFeature("ca-derivations"); + settings.requireExperimentalFeature(Xp::CaDerivations); toplevelRealisations.insert(*realisation); } } @@ -909,7 +987,7 @@ std::map<StorePath, StorePath> copyPaths( // Don't fail if the remote doesn't support CA derivations is it might // not be within our control to change that, and we might still want // to at least copy the output paths. - if (e.missingFeature == "ca-derivations") + if (e.missingFeature == Xp::CaDerivations) ignoreException(); else throw; @@ -1031,7 +1109,7 @@ std::map<StorePath, StorePath> copyPaths( nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what())); + printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what()); showProgress(); return; } @@ -1061,13 +1139,28 @@ void copyClosure( copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); } +void copyClosure( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths, + RepairFlag repair, + CheckSigsFlag checkSigs, + SubstituteFlag substitute) +{ + if (&srcStore == &dstStore) return; + + StorePathSet closure; + srcStore.computeFSClosure(storePaths, closure); + copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); +} + std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven) { std::string path; getline(str, path); if (str.eof()) { return {}; } if (!hashGiven) { - string s; + std::string s; getline(str, s); auto narHash = Hash::parseAny(s, htSHA256); getline(str, s); @@ -1080,7 +1173,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre std::string deriver; getline(str, deriver); if (deriver != "") info.deriver = store.parseStorePath(deriver); - string s; + std::string s; getline(str, s); auto n = string2Int<int>(s); if (!n) throw Error("number expected"); @@ -1104,7 +1197,7 @@ std::string Store::showPaths(const StorePathSet & paths) } -string showPaths(const PathSet & paths) +std::string showPaths(const PathSet & paths) { return concatStringsSep(", ", quoteStrings(paths)); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a6a4d0c73..1a320bcb1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -76,103 +76,10 @@ enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; const uint32_t exportMagic = 0x4558494e; -typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots; - - -struct GCOptions -{ - /* Garbage collector operation: - - - `gcReturnLive': return the set of paths reachable from - (i.e. in the closure of) the roots. - - - `gcReturnDead': return the set of paths not reachable from - the roots. - - - `gcDeleteDead': actually delete the latter set. - - - `gcDeleteSpecific': delete the paths listed in - `pathsToDelete', insofar as they are not reachable. - */ - typedef enum { - gcReturnLive, - gcReturnDead, - gcDeleteDead, - gcDeleteSpecific, - } GCAction; - - GCAction action{gcDeleteDead}; - - /* If `ignoreLiveness' is set, then reachability from the roots is - ignored (dangerous!). However, the paths must still be - unreferenced *within* the store (i.e., there can be no other - store paths that depend on them). */ - bool ignoreLiveness{false}; - - /* For `gcDeleteSpecific', the paths to delete. */ - StorePathSet pathsToDelete; - - /* Stop after at least `maxFreed' bytes have been freed. */ - uint64_t maxFreed{std::numeric_limits<uint64_t>::max()}; -}; - - -struct GCResults -{ - /* Depending on the action, the GC roots, or the paths that would - be or have been deleted. */ - PathSet paths; - - /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the - number of bytes that would be or was freed. */ - uint64_t bytesFreed = 0; -}; - - enum BuildMode { bmNormal, bmRepair, bmCheck }; +struct BuildResult; -struct BuildResult -{ - /* Note: don't remove status codes, and only add new status codes - at the end of the list, to prevent client/server - incompatibilities in the nix-store --serve protocol. */ - enum Status { - Built = 0, - Substituted, - AlreadyValid, - PermanentFailure, - InputRejected, - OutputRejected, - TransientFailure, // possibly transient - CachedFailure, // no longer used - TimedOut, - MiscFailure, - DependencyFailed, - LogLimitExceeded, - NotDeterministic, - } status = MiscFailure; - std::string errorMsg; - - /* How many times this build was performed. */ - unsigned int timesBuilt = 0; - - /* If timesBuilt > 1, whether some builds did not produce the same - result. (Note that 'isNonDeterministic = false' does not mean - the build is deterministic, just that we don't have evidence of - non-determinism.) */ - bool isNonDeterministic = false; - - DrvOutputs builtOutputs; - - /* The start/stop times of the build (or one of the rounds, if it - was repeated). */ - time_t startTime = 0, stopTime = 0; - - bool success() { - return status == Built || status == Substituted || status == AlreadyValid; - } -}; typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; @@ -234,7 +141,6 @@ protected: struct State { - // FIXME: fix key LRUCache<std::string, PathInfoCacheValue> pathInfoCache; }; @@ -325,7 +231,9 @@ public: simply yield a different store path, so other users wouldn't be affected), but it has some backwards compatibility issues (the hashing scheme changes), so I'm not doing that for now. */ - StorePath computeStorePathForText(const string & name, const string & s, + StorePath computeStorePathForText( + std::string_view name, + std::string_view s, const StorePathSet & references) const; /* Check whether a path is valid. */ @@ -366,6 +274,14 @@ public: void queryPathInfo(const StorePath & path, Callback<ref<const ValidPathInfo>> callback) noexcept; + /* Query the information about a realisation. */ + std::shared_ptr<const Realisation> queryRealisation(const DrvOutput &); + + /* Asynchronous version of queryRealisation(). */ + void queryRealisation(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept; + + /* Check whether the given valid path info is sufficiently attested, by either being signed by a trusted public key or content-addressed, in order to be included in the given store. @@ -390,11 +306,11 @@ protected: virtual void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0; + virtual void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept = 0; public: - virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0; - /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) @@ -447,9 +363,14 @@ public: validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see libutil/archive.hh). */ - virtual StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); + virtual StorePath addToStore( + std::string_view name, + const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, + RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()); /* Copy the contents of a path to the store and register the validity the resulting path, using a constant amount of @@ -464,14 +385,18 @@ public: false). `dump` may be drained */ // FIXME: remove? - virtual StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ - virtual StorePath addTextToStore(const string & name, const string & s, - const StorePathSet & references, RepairFlag repair = NoRepair) = 0; + virtual StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair = NoRepair) = 0; /** * Add a mapping indicating that `deriver!outputName` maps to the output path @@ -503,6 +428,16 @@ public: BuildMode buildMode = bmNormal, std::shared_ptr<Store> evalStore = nullptr); + /* Like `buildPaths()`, but return a vector of `BuildResult`s + corresponding to each element in `paths`. Note that in case of + a build/substitution error, this function won't throw an + exception, but return a `BuildResult` containing an error + message. */ + virtual std::vector<BuildResult> buildPathsWithResults( + const std::vector<DerivedPath> & paths, + BuildMode buildMode = bmNormal, + std::shared_ptr<Store> evalStore = nullptr); + /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). @@ -549,50 +484,10 @@ public: virtual void addTempRoot(const StorePath & path) { debug("not creating temporary root, store doesn't support GC"); } - /* Add an indirect root, which is merely a symlink to `path' from - /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed - to be a symlink to a store path. The garbage collector will - automatically remove the indirect root when it finds that - `path' has disappeared. */ - virtual void addIndirectRoot(const Path & path) - { unsupported("addIndirectRoot"); } - - /* Acquire the global GC lock, then immediately release it. This - function must be called after registering a new permanent root, - but before exiting. Otherwise, it is possible that a running - garbage collector doesn't see the new root and deletes the - stuff we've just built. By acquiring the lock briefly, we - ensure that either: - - - The collector is already running, and so we block until the - collector is finished. The collector will know about our - *temporary* locks, which should include whatever it is we - want to register as a permanent lock. - - - The collector isn't running, or it's just started but hasn't - acquired the GC lock yet. In that case we get and release - the lock right away, then exit. The collector scans the - permanent root and sees ours. - - In either case the permanent root is seen by the collector. */ - virtual void syncWithGC() { }; - - /* Find the roots of the garbage collector. Each root is a pair - (link, storepath) where `link' is the path of the symlink - outside of the Nix store that point to `storePath'. If - 'censor' is true, privacy-sensitive information about roots - found in /proc is censored. */ - virtual Roots findRoots(bool censor) - { unsupported("findRoots"); } - - /* Perform a garbage collection. */ - virtual void collectGarbage(const GCOptions & options, GCResults & results) - { unsupported("collectGarbage"); } - /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or `nix-store --register-validity'. */ - string makeValidityRegistration(const StorePathSet & paths, + std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); /* Write a JSON representation of store path metadata, such as the @@ -708,8 +603,11 @@ public: /* Return the build log of the specified store path, if available, or null otherwise. */ - virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path) - { return nullptr; } + virtual std::optional<std::string> getBuildLog(const StorePath & path) + { return std::nullopt; } + + virtual void addBuildLog(const StorePath & path, std::string_view log) + { unsupported("addBuildLog"); } /* Hack to allow long-running processes like hydra-queue-runner to occasionally flush their path info cache. */ @@ -741,6 +639,14 @@ public: virtual void createUser(const std::string & userName, uid_t userId) { } + /* + * Synchronises the options of the client with those of the daemon + * (a no-op when there’s no daemon) + */ + virtual void setOptions() { } + + virtual std::optional<std::string> getVersion() { return {}; } + protected: Stats stats; @@ -791,6 +697,13 @@ void copyClosure( CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); +void copyClosure( + Store & srcStore, Store & dstStore, + const StorePathSet & paths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); + /* Remove the temporary roots file for this process. Any temporary root becomes garbage after this point unless it has been registered as a (permanent) root. */ @@ -876,7 +789,7 @@ struct RegisterStoreImplementation /* Display a set of paths in human-readable form (i.e., between quotes and separated by commas). */ -string showPaths(const PathSet & paths); +std::string showPaths(const PathSet & paths); std::optional<ValidPathInfo> decodeValidPathInfo( diff --git a/src/libstore/tests/machines.cc b/src/libstore/tests/machines.cc new file mode 100644 index 000000000..f51052b14 --- /dev/null +++ b/src/libstore/tests/machines.cc @@ -0,0 +1,169 @@ +#include "machines.hh" +#include "globals.hh" + +#include <gmock/gmock-matchers.h> + +using testing::Contains; +using testing::ElementsAre; +using testing::EndsWith; +using testing::Eq; +using testing::Field; +using testing::SizeIs; + +using nix::absPath; +using nix::FormatError; +using nix::getMachines; +using nix::Machine; +using nix::Machines; +using nix::pathExists; +using nix::Settings; +using nix::settings; + +class Environment : public ::testing::Environment { + public: + void SetUp() override { settings.thisSystem = "TEST_ARCH-TEST_OS"; } +}; + +testing::Environment* const foo_env = + testing::AddGlobalTestEnvironment(new Environment); + +TEST(machines, getMachinesWithEmptyBuilders) { + settings.builders = ""; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesUriOnly) { + settings.builders = "nix@scratchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); +} + +TEST(machines, getMachinesDefaults) { + settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); +} + +TEST(machines, getMachinesWithNewLineSeparator) { + settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(2)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithSemicolonSeparator) { + settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + EXPECT_THAT(actual, SizeIs(2)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { + settings.builders = "nix@scratchy.labs.cs.uu.nl i686-linux " + "/home/nix/.ssh/id_scratchy_auto 8 3 kvm " + "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark"))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, Eq("SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="))); +} + +TEST(machines, + getMachinesWithCorrectCompleteSingleBuilderWithTabColumnDelimiter) { + settings.builders = + "nix@scratchy.labs.cs.uu.nl\ti686-linux\t/home/nix/.ssh/" + "id_scratchy_auto\t8\t3\tkvm\tbenchmark\tSSH+HOST+PUBLIC+" + "KEY+BASE64+ENCODED=="; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark"))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, Eq("SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="))); +} + +TEST(machines, getMachinesWithMultiOptions) { + settings.builders = "nix@scratchy.labs.cs.uu.nl Arch1,Arch2 - - - " + "SupportedFeature1,SupportedFeature2 " + "MandatoryFeature1,MandatoryFeature2"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2"))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2"))); +} + +TEST(machines, getMachinesWithIncorrectFormat) { + settings.builders = "nix@scratchy.labs.cs.uu.nl - - eight"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - -1"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 three"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 -3"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 3 - - BAD_BASE64"; + EXPECT_THROW(getMachines(), FormatError); +} + +TEST(machines, getMachinesWithCorrectFileReference) { + auto path = absPath("src/libstore/tests/test-data/machines.valid"); + ASSERT_TRUE(pathExists(path)); + + settings.builders = std::string("@") + path; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(3)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { + auto path = "/dev/null"; + ASSERT_TRUE(pathExists(path)); + + settings.builders = std::string("@") + path; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesWithIncorrectFileReference) { + settings.builders = std::string("@") + absPath("/not/a/file"); + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { + settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); + EXPECT_THROW(getMachines(), FormatError); +} diff --git a/src/libstore/tests/test-data/machines.bad_format b/src/libstore/tests/test-data/machines.bad_format new file mode 100644 index 000000000..7255a1216 --- /dev/null +++ b/src/libstore/tests/test-data/machines.bad_format @@ -0,0 +1 @@ +nix@scratchy.labs.cs.uu.nl - - eight diff --git a/src/libstore/tests/test-data/machines.valid b/src/libstore/tests/test-data/machines.valid new file mode 100644 index 000000000..1a6c8017c --- /dev/null +++ b/src/libstore/tests/test-data/machines.valid @@ -0,0 +1,3 @@ +nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm +nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 +nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFDQVFDWWV5R1laNTNzd1VjMUZNSHBWL1BCcXlKaFR5S1JoRkpWWVRpRHlQN2h5c1JGa0w4VDlLOGdhL2Y2L3c3QjN2SjNHSFRIUFkybENiUEdZbGNLd2h6M2ZRbFNNOEViNi95b3ZLajdvM1FsMEx5Y0dzdGJvRmcwWkZKNldncUxsR0ltS0NobUlxOGZ3TW5ZTWUxbnRQeTBUZFZjSU1tOTV3YzF3SjBMd2c3cEVMRmtHazdkeTVvYnM4a3lGZ0pORDVRSmFwQWJjeWp4Z1QzdzdMcktNZ2xzeWhhd01JNVpkMGZsQTVudW5OZ3pid3plYVhLaUsyTW0vdGJXYTU1YTd4QmNYdHpIZGlPSWdSajJlRWxaMGh5bk10YjBmcklsdmxIcEtLaVFaZ3pQdCtIVXQ2bXpRMkRVME52MGYyYnNSU0krOGpJU2pQcmdlcVVHRldMUzVIUTg2N2xSMlpiaWtyclhZNTdqbVFEZk5DRHY1VFBHZU9UekFEd2pjMDc2aFZ3VFJCd3VTZFhtaWNxTS95b3lrWitkV1dnZ25MenE5QU1tdlNZcDhmZkZDcS9CSDBZNUFXWTFHay9vS3hMVTNaOWt3ZDd2UWNFQWFCQ2dxdnVZRGdTaHE1RlhndDM3OVZESWtEL05ZSTg2QXVvajVDRmVNTzlRM2pJSlRadlh6c1VldjVoSnA2djcxSVh5ODVtbTY5R20zcXdicVE1SjVQZDU1Um56SitpaW5BNjZxTEFSc0Y4amNsSnd5ekFXclBoYU9DRVY2bjVMeVhVazhzMW9EVVR4V1pWN25rVkFTbHJ0MllGcjN5dzdjRTRXQVhsemhHcDhocmdLMVVkMUlyeDVnZWRaSnBWcy9uNWVybmJFMUxmb2x5UHUvRUFIWlh6VGd4dHVDUFNobXc9PQo= diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 02e81b022..5c38323cd 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -56,14 +56,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() auto conn = make_ref<Connection>(); /* Connect to a daemon that does the privileged work for us. */ - conn->fd = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!conn->fd) - throw SysError("cannot create Unix domain socket"); - closeOnExec(conn->fd.get()); + conn->fd = createUnixDomainSocket(); nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 93cf546d2..87088a3ac 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 32) +#define PROTOCOL_VERSION (1 << 8 | 34) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -56,6 +56,8 @@ typedef enum { wopRegisterDrvOutput = 42, wopQueryRealisation = 43, wopAddMultipleToStore = 44, + wopAddBuildLog = 45, + wopBuildPathsWithResults = 46, } WorkerOp; @@ -90,6 +92,7 @@ MAKE_WORKER_PROTO(, ContentAddress); MAKE_WORKER_PROTO(, DerivedPath); MAKE_WORKER_PROTO(, Realisation); MAKE_WORKER_PROTO(, DrvOutput); +MAKE_WORKER_PROTO(, BuildResult); MAKE_WORKER_PROTO(template<typename T>, std::vector<T>); MAKE_WORKER_PROTO(template<typename T>, std::set<T>); |