diff options
Diffstat (limited to 'src/libstore')
45 files changed, 1194 insertions, 1141 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3a2d84861..f8eff508c 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -40,14 +40,14 @@ void BinaryCacheStore::init() upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) { - size_t colon = line.find(':'); - if (colon == std::string::npos) continue; + size_t colon= line.find(':'); + if (colon ==std::string::npos) continue; auto name = line.substr(0, colon); auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { if (value != storeDir) - throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") - % getUri() % value % storeDir); + throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", + getUri(), value, storeDir); } else if (name == "WantMassQuery") { wantMassQuery.setDefault(value == "1" ? "true" : "false"); } else if (name == "Priority") { @@ -93,7 +93,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) { - return storePathToHash(printStorePath(storePath)) + ".narinfo"; + return std::string(storePath.hashPart()) + ".narinfo"; } void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) @@ -102,7 +102,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - auto hashPart = storePathToHash(printStorePath(narInfo->path)); + std::string hashPart(narInfo->path.hashPart()); { auto state_(state.lock()); @@ -113,9 +113,12 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); } -void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, +void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { + // FIXME: See if we can use the original source to reduce memory usage. + auto nar = make_ref<std::string>(narSource.drain()); + if (!repair && isValidPath(info.path)) return; /* Verify that all references are valid. This may do some .narinfo @@ -161,7 +164,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str } } - upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json"); + upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); } /* Compress the NAR. */ @@ -284,7 +287,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) try { getFile(info->url, *decompressor); } catch (NoSuchBinaryCacheFile & e) { - throw SubstituteGone(e.what()); + throw SubstituteGone(e.info()); } decompressor->finish(); @@ -327,7 +330,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { // FIXME: some cut&paste from LocalStore::addToStore(). @@ -336,7 +339,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath small files. */ StringSink sink; Hash h; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); } else { @@ -345,9 +348,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(recursive, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, h, name)); - addToStore(info, sink.s, repair, CheckSigs, nullptr); + auto source = StringSource { *sink.s }; + addToStore(info, source, repair, CheckSigs, nullptr); return std::move(info.path); } @@ -356,12 +360,13 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s const StorePathSet & references, RepairFlag repair) { ValidPathInfo info(computeStorePathForText(name, s, references)); - info.references = cloneStorePathSet(references); + info.references = references; if (repair || !isValidPath(info.path)) { StringSink sink; dumpString(s, sink); - addToStore(info, sink.s, repair, CheckSigs, nullptr); + auto source = StringSource { *sink.s }; + addToStore(info, source, repair, CheckSigs, nullptr); } return std::move(info.path); @@ -390,14 +395,14 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) { - auto drvPath = path.clone(); + auto drvPath = path; if (!path.isDerivation()) { try { auto info = queryPathInfo(path); // FIXME: add a "Log" field to .narinfo if (!info->deriver) return nullptr; - drvPath = info->deriver->clone(); + drvPath = *info->deriver; } catch (InvalidPath &) { return nullptr; } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index aa13c1cb4..52ef8aa7a 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -74,12 +74,12 @@ public: std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } - void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, + void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0e3a23a4d..53a0958aa 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -7,7 +7,7 @@ #include "affinity.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "download.hh" +#include "filetransfer.hh" #include "finally.hh" #include "compression.hh" #include "json.hh" @@ -33,7 +33,6 @@ #include <sys/types.h> #include <sys/stat.h> #include <sys/utsname.h> -#include <sys/select.h> #include <sys/resource.h> #include <sys/socket.h> #include <sys/un.h> @@ -43,6 +42,7 @@ #include <errno.h> #include <cstring> #include <termios.h> +#include <poll.h> #include <pwd.h> #include <grp.h> @@ -104,13 +104,10 @@ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; -class Goal : public std::enable_shared_from_this<Goal> +struct Goal : public std::enable_shared_from_this<Goal> { -public: typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; -protected: - /* Backlink to the worker. */ Worker & worker; @@ -138,6 +135,9 @@ protected: /* Whether the goal is finished. */ ExitCode exitCode; + /* Exception containing an error message, if any. */ + std::optional<Error> ex; + Goal(Worker & worker) : worker(worker) { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; @@ -149,7 +149,6 @@ protected: trace("goal destroyed"); } -public: virtual void work() = 0; void addWaitee(GoalPtr waitee); @@ -173,21 +172,14 @@ public: return name; } - ExitCode getExitCode() - { - return exitCode; - } - /* Callback in case of a timeout. It should wake up its waiters, get rid of any running child processes that are being monitored by the worker (important!), etc. */ - virtual void timedOut() = 0; + virtual void timedOut(Error && ex) = 0; virtual string key() = 0; -protected: - - virtual void amDone(ExitCode result); + void amDone(ExitCode result, std::optional<Error> ex = {}); }; @@ -303,7 +295,7 @@ public: /* Make a goal (with caching). */ GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(StorePath && drvPath, + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); @@ -355,13 +347,13 @@ public: contents. */ bool pathContentsGood(const StorePath & path); - void markContentsGood(StorePath && path); + void markContentsGood(const StorePath & path); void updateProgress() { actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize); + act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } }; @@ -392,8 +384,7 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) assert(waitees.find(waitee) != waitees.end()); waitees.erase(waitee); - trace(format("waitee '%1%' done; %2% left") % - waitee->name % waitees.size()); + trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; @@ -418,12 +409,20 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) } -void Goal::amDone(ExitCode result) +void Goal::amDone(ExitCode result, std::optional<Error> ex) { trace("done"); assert(exitCode == ecBusy); assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; + + if (ex) { + if (!waiters.empty()) + logError(ex->info()); + else + this->ex = std::move(*ex); + } + for (auto & i : waiters) { GoalPtr goal = i.lock(); if (goal) goal->waiteeDone(shared_from_this(), result); @@ -453,7 +452,7 @@ static void commonChildInit(Pipe & logPipe) that e.g. ssh cannot open /dev/tty) and it doesn't receive terminal signals. */ if (setsid() == -1) - throw SysError(format("creating a new session")); + throw SysError("creating a new session"); /* Dup the write side of the logger pipe into stderr. */ if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) @@ -466,7 +465,7 @@ static void commonChildInit(Pipe & logPipe) /* Reroute stdin to /dev/null. */ int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); if (fdDevNull == -1) - throw SysError(format("cannot open '%1%'") % pathNullDevice); + throw SysError("cannot open '%1%'", pathNullDevice); if (dup2(fdDevNull, STDIN_FILENO) == -1) throw SysError("cannot dup null device into stdin"); close(fdDevNull); @@ -488,12 +487,18 @@ void handleDiffHook( auto diffRes = runProgram(diffHookOptions); if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first))); + throw ExecError(diffRes.first, + "diff-hook program '%1%' %2%", + diffHook, + statusToString(diffRes.first)); if (diffRes.second != "") printError(chomp(diffRes.second)); } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); + ErrorInfo ei = error.info(); + ei.hint = hintfmt("diff hook execution failed: %s", + (error.info().hint.has_value() ? error.info().hint->str() : "")); + logError(ei); } } } @@ -507,9 +512,10 @@ private: Path fnUserLock; AutoCloseFD fdUserLock; + bool isEnabled = false; string user; - uid_t uid; - gid_t gid; + uid_t uid = 0; + gid_t gid = 0; std::vector<gid_t> supplementaryGIDs; public: @@ -522,7 +528,9 @@ public: uid_t getGID() { assert(gid); return gid; } std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; } - bool enabled() { return uid != 0; } + bool findFreeUser(); + + bool enabled() { return isEnabled; } }; @@ -530,42 +538,46 @@ public: UserLock::UserLock() { assert(settings.buildUsersGroup != ""); + createDirs(settings.nixStateDir + "/userpool"); +} + +bool UserLock::findFreeUser() { + if (enabled()) return true; /* Get the members of the build-users-group. */ struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - throw Error(format("the group '%1%' specified in 'build-users-group' does not exist") - % settings.buildUsersGroup); + throw Error("the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup); gid = gr->gr_gid; /* Copy the result of getgrnam. */ Strings users; for (char * * p = gr->gr_mem; *p; ++p) { - debug(format("found build user '%1%'") % *p); + debug("found build user '%1%'", *p); users.push_back(*p); } if (users.empty()) - throw Error(format("the build users group '%1%' has no members") - % settings.buildUsersGroup); + throw Error("the build users group '%1%' has no members", + settings.buildUsersGroup); /* Find a user account that isn't currently in use for another build. */ for (auto & i : users) { - debug(format("trying user '%1%'") % i); + debug("trying user '%1%'", i); struct passwd * pw = getpwnam(i.c_str()); if (!pw) - throw Error(format("the user '%1%' in the group '%2%' does not exist") - % i % settings.buildUsersGroup); + throw Error("the user '%1%' in the group '%2%' does not exist", + i, settings.buildUsersGroup); - createDirs(settings.nixStateDir + "/userpool"); fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fd) - throw SysError(format("opening user lock '%1%'") % fnUserLock); + throw SysError("opening user lock '%1%'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { fdUserLock = std::move(fd); @@ -574,8 +586,8 @@ UserLock::UserLock() /* Sanity check... */ if (uid == getuid() || uid == geteuid()) - throw Error(format("the Nix user should not be a member of '%1%'") - % settings.buildUsersGroup); + throw Error("the Nix user should not be a member of '%1%'", + settings.buildUsersGroup); #if __linux__ /* Get the list of supplementary groups of this build user. This @@ -585,21 +597,19 @@ UserLock::UserLock() int err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(), &ngroups); if (err == -1) - throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name); + throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); supplementaryGIDs.resize(ngroups); #endif - return; + isEnabled = true; + return true; } } - throw Error(format("all build users are currently in use; " - "consider creating additional users and adding them to the '%1%' group") - % settings.buildUsersGroup); + return false; } - void UserLock::kill() { killUser(uid); @@ -858,6 +868,9 @@ private: std::unique_ptr<Activity> act; + /* Activity that denotes waiting for a lock. */ + std::unique_ptr<Activity> actLock; + std::map<ActivityId, Activity> builderActivities; /* The remote machine on which we're building. */ @@ -887,16 +900,16 @@ private: friend struct RestrictedStore; public: - DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, + DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, + DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); /* Whether we need to perform hash rewriting if there are valid output paths. */ bool needsHashRewrite(); - void timedOut() override; + void timedOut(Error && ex) override; string key() override { @@ -911,7 +924,7 @@ public: StorePath getDrvPath() { - return drvPath.clone(); + return drvPath; } /* Add wanted outputs to an already existing derivation goal. */ @@ -928,6 +941,7 @@ private: void closureRepaired(); void inputsRealised(); void tryToBuild(); + void tryLocalBuild(); void buildDone(); /* Is the build hook willing to perform the build? */ @@ -994,12 +1008,11 @@ private: void repairClosure(); - void amDone(ExitCode result) override - { - Goal::amDone(result); - } + void started(); - void done(BuildResult::Status status, const string & msg = ""); + void done( + BuildResult::Status status, + std::optional<Error> ex = {}); StorePathSet exportReferences(const StorePathSet & storePaths); }; @@ -1008,11 +1021,11 @@ private: const Path DerivationGoal::homeDir = "/homeless-shelter"; -DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, +DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(true) - , drvPath(std::move(drvPath)) + , drvPath(drvPath) , wantedOutputs(wantedOutputs) , buildMode(buildMode) { @@ -1025,11 +1038,11 @@ DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOut } -DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, +DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(false) - , drvPath(std::move(drvPath)) + , drvPath(drvPath) , buildMode(buildMode) { this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv)); @@ -1093,10 +1106,10 @@ void DerivationGoal::killChild() } -void DerivationGoal::timedOut() +void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut); + done(BuildResult::TimedOut, ex); } @@ -1144,8 +1157,7 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - printError("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)); - done(BuildResult::MiscFailure); + done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); return; } @@ -1181,7 +1193,7 @@ void DerivationGoal::haveDerivation() return; } - parsedDrv = std::make_unique<ParsedDerivation>(drvPath.clone(), *drv); + parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build @@ -1289,14 +1301,18 @@ void DerivationGoal::repairClosure() if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path.clone(), i.clone()); + outputsToDrv.insert_or_assign(j.second.path, i); } /* Check each path (slow!). */ for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - printError("found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); + logError({ + .name = "Corrupt path in closure", + .hint = hintfmt( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) + }); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) addWaitee(worker.makeSubstitutionGoal(i, Repair)); @@ -1330,9 +1346,9 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - printError("cannot build derivation '%s': %s dependencies couldn't be built", - worker.store.printStorePath(drvPath), nrFailed); - done(BuildResult::DependencyFailed); + done(BuildResult::DependencyFailed, Error( + "%s dependencies of derivation '%s' failed to build", + nrFailed, worker.store.printStorePath(drvPath))); return; } @@ -1386,6 +1402,19 @@ void DerivationGoal::inputsRealised() result = BuildResult(); } +void DerivationGoal::started() { + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + fmt("building '%s'", worker.store.printStorePath(drvPath)); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); + worker.updateProgress(); +} void DerivationGoal::tryToBuild() { @@ -1401,10 +1430,15 @@ void DerivationGoal::tryToBuild() lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); return; } + actLock.reset(); + /* Now check again whether the outputs are valid. This is because another process may have started building in parallel. After it has finished and released the locks, we can (and should) @@ -1420,7 +1454,7 @@ void DerivationGoal::tryToBuild() return; } - missingPaths = cloneStorePathSet(drv->outputPaths()); + missingPaths = drv->outputPaths(); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); @@ -1437,26 +1471,13 @@ void DerivationGoal::tryToBuild() supported for local builds. */ bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); - auto started = [&]() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); - worker.updateProgress(); - }; - /* Is the build hook willing to accept this job? */ if (!buildLocally) { switch (tryBuildHook()) { case rpAccept: /* Yes, it has started doing so. Wait until we get EOF from the hook. */ + actLock.reset(); result.startTime = time(0); // inexact state = &DerivationGoal::buildDone; started(); @@ -1464,6 +1485,9 @@ void DerivationGoal::tryToBuild() case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); return; @@ -1473,6 +1497,8 @@ void DerivationGoal::tryToBuild() } } + actLock.reset(); + /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ @@ -1483,17 +1509,48 @@ void DerivationGoal::tryToBuild() return; } + state = &DerivationGoal::tryLocalBuild; + worker.wakeUp(shared_from_this()); +} + +void DerivationGoal::tryLocalBuild() { + + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { +#if defined(__linux__) || defined(__APPLE__) + if (!buildUser) buildUser = std::make_unique<UserLock>(); + + if (buildUser->findFreeUser()) { + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); + } else { + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + worker.waitForAWhile(shared_from_this()); + return; + } +#else + /* Don't know how to block the creation of setuid/setgid + binaries on this platform. */ + throw Error("build users are not supported on this platform for security reasons"); +#endif + } + + actLock.reset(); + try { /* Okay, we have to build. */ startBuilder(); } catch (BuildError & e) { - printError(e.msg()); outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, e.msg()); + done(BuildResult::InputRejected, e); return; } @@ -1605,10 +1662,10 @@ void DerivationGoal::buildDone() } auto msg = fmt("builder for '%s' %s", - worker.store.printStorePath(drvPath), + yellowtxt(worker.store.printStorePath(drvPath)), statusToString(status)); - if (!settings.verboseBuild && !logTail.empty()) { + if (!logger->isVerbose() && !logTail.empty()) { msg += (format("; last %d log lines:") % logTail.size()).str(); for (auto & line : logTail) msg += "\n " + line; @@ -1657,11 +1714,7 @@ void DerivationGoal::buildDone() } void flushLine() { - if (settings.verboseBuild) { - printError("post-build-hook: " + currentLine); - } else { - act.result(resPostBuildLogLine, currentLine); - } + act.result(resPostBuildLogLine, currentLine); currentLine.clear(); } @@ -1680,6 +1733,7 @@ void DerivationGoal::buildDone() } if (buildMode == bmCheck) { + deleteTmpDir(true); done(BuildResult::Built); return; } @@ -1709,8 +1763,6 @@ void DerivationGoal::buildDone() outputLocks.unlock(); } catch (BuildError & e) { - printError(e.msg()); - outputLocks.unlock(); BuildResult::Status st = BuildResult::MiscFailure; @@ -1729,7 +1781,7 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, e.msg()); + done(st, e); return; } @@ -1772,7 +1824,7 @@ HookReply DerivationGoal::tryBuildHook() } } - debug(format("hook reply is '%1%'") % reply); + debug("hook reply is '%1%'", reply); if (reply == "decline") return rpDecline; @@ -1788,8 +1840,12 @@ HookReply DerivationGoal::tryBuildHook() } catch (SysError & e) { if (e.errNo == EPIPE) { - printError("build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))); + logError({ + .name = "Build hook died", + .hint = hintfmt( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))) + }); worker.hook = 0; return rpDecline; } else @@ -1845,14 +1901,14 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) if (!inputPaths.count(storePath)) throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - worker.store.computeFSClosure(singleton(storePath), paths); + worker.store.computeFSClosure({storePath}, paths); } /* If there are derivations in the graph, then include their outputs as well. This is useful if you want to do things like passing all build-time dependencies of some path to a derivation that builds a NixOS DVD image. */ - auto paths2 = cloneStorePathSet(paths); + auto paths2 = paths; for (auto & j : paths2) { if (j.isDerivation()) { @@ -1942,22 +1998,6 @@ void DerivationGoal::startBuilder() #endif } - /* If `build-users-group' is not empty, then we have to build as - one of the members of that group. */ - if (settings.buildUsersGroup != "" && getuid() == 0) { -#if defined(__linux__) || defined(__APPLE__) - buildUser = std::make_unique<UserLock>(); - - /* Make sure that no other processes are executing under this - uid. */ - buildUser->kill(); -#else - /* Don't know how to block the creation of setuid/setgid - binaries on this platform. */ - throw Error("build users are not supported on this platform for security reasons"); -#endif - } - /* Create a temporary directory where the build will take place. */ tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); @@ -1985,7 +2025,7 @@ void DerivationGoal::startBuilder() string s = get(drv->env, "exportReferencesGraph").value_or(""); Strings ss = tokenizeString<Strings>(s); if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); + throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); for (Strings::iterator i = ss.begin(); i != ss.end(); ) { string fileName = *i++; static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); @@ -1997,7 +2037,7 @@ void DerivationGoal::startBuilder() /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences(singleton(storePath)), false, false)); + exportReferences({storePath}), false, false)); } } @@ -2034,7 +2074,7 @@ void DerivationGoal::startBuilder() worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); } catch (InvalidPath & e) { } catch (Error & e) { - throw Error(format("while processing 'sandbox-paths': %s") % e.what()); + throw Error("while processing 'sandbox-paths': %s", e.what()); } for (auto & i : closure) { auto p = worker.store.printStorePath(i); @@ -2081,10 +2121,10 @@ void DerivationGoal::startBuilder() printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError(format("cannot create '%1%'") % chrootRootDir); + throw SysError("cannot create '%1%'", chrootRootDir); if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % chrootRootDir); + throw SysError("cannot change ownership of '%1%'", chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need this. (Of course they should really respect $TMPDIR @@ -2128,7 +2168,7 @@ void DerivationGoal::startBuilder() chmod_(chrootStoreDir, 01775); if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir); + throw SysError("cannot change ownership of '%1%'", chrootStoreDir); for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); @@ -2161,7 +2201,7 @@ void DerivationGoal::startBuilder() if (needsHashRewrite()) { if (pathExists(homeDir)) - throw Error(format("directory '%1%' exists; please remove it") % homeDir); + throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); /* We're not doing a chroot build, but we have some valid output paths. Since we can't just overwrite or delete @@ -2182,7 +2222,7 @@ void DerivationGoal::startBuilder() for (auto & i : missingPaths) if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) { addHashRewrite(i); - redirectedBadOutputs.insert(i.clone()); + redirectedBadOutputs.insert(i); } } @@ -2206,8 +2246,7 @@ void DerivationGoal::startBuilder() if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { state = stExtraChrootDirs; } else { - throw Error(format("unknown pre-build hook command '%1%'") - % line); + throw Error("unknown pre-build hook command '%1%'", line); } } else if (state == stExtraChrootDirs) { if (line == "") { @@ -2229,7 +2268,7 @@ void DerivationGoal::startBuilder() startDaemon(); /* Run the builder. */ - printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); + printMsg(lvlChatty, "executing builder '%1%'", drv->builder); /* Create the log file. */ Path logFile = openLogFile(); @@ -2249,10 +2288,13 @@ void DerivationGoal::startBuilder() if (chown(slaveName.c_str(), buildUser->getUID(), 0)) throw SysError("changing owner of pseudoterminal slave"); - } else { + } +#if __APPLE__ + else { if (grantpt(builderOut.readSide.get())) throw SysError("granting access to pseudoterminal slave"); } +#endif #if 0 // Mount the pt in the sandbox so that the "tty" command works. @@ -2465,7 +2507,7 @@ void DerivationGoal::initTmpDir() { auto hash = hashString(htSHA256, i.first); string fn = ".attr-" + hash.to_string(Base32, false); Path p = tmpDir + "/" + fn; - writeFile(p, i.second); + writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; } @@ -2675,8 +2717,8 @@ struct RestrictedStore : public LocalFSStore StorePathSet queryAllValidPaths() override { StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p.clone()); - for (auto & p : goal.addedPaths) paths.insert(p.clone()); + for (auto & p : goal.inputPaths) paths.insert(p); + for (auto & p : goal.addedPaths) paths.insert(p); return paths; } @@ -2705,14 +2747,11 @@ struct RestrictedStore : public LocalFSStore StorePathSet queryDerivationOutputs(const StorePath & path) override { throw Error("queryDerivationOutputs"); } - StringSet queryDerivationOutputNames(const StorePath & path) override - { throw Error("queryDerivationOutputNames"); } - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override { throw Error("addToStore"); } @@ -2725,9 +2764,9 @@ struct RestrictedStore : public LocalFSStore } StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { - auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); goal.addDependency(path); return path; } @@ -2767,7 +2806,7 @@ struct RestrictedStore : public LocalFSStore auto drv = derivationFromPath(path.path); for (auto & output : drv.outputs) if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path.clone()); + newPaths.insert(output.second.path); } else if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } @@ -2812,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore if (goal.isAllowed(path.path)) allowed.emplace_back(path); else - unknown.insert(path.path.clone()); + unknown.insert(path.path); } next->queryMissing(allowed, willBuild, willSubstitute, @@ -2907,7 +2946,7 @@ void DerivationGoal::addDependency(const StorePath & path) { if (isAllowed(path)) return; - addedPaths.insert(path.clone()); + addedPaths.insert(path); /* If we're doing a sandbox build, then we have to make the path appear in the sandbox. */ @@ -2967,7 +3006,7 @@ void DerivationGoal::chownToBuilder(const Path & path) { if (!buildUser) return; if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % path); + throw SysError("cannot change ownership of '%1%'", path); } @@ -3104,7 +3143,7 @@ void DerivationGoal::runChild() /* Bind-mount chroot directory to itself, to treat it as a different filesystem from /, as needed for pivot_root. */ if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError(format("unable to bind mount '%1%'") % chrootRootDir); + throw SysError("unable to bind mount '%1%'", chrootRootDir); /* Bind-mount the sandbox's Nix store onto itself so that we can mark it as a "shared" subtree, allowing bind @@ -3151,7 +3190,7 @@ void DerivationGoal::runChild() // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the - // potential impurities introduced in fixed outputs. + // potential impurities introduced in fixed-outputs. writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); ss.push_back("/etc/services"); @@ -3166,7 +3205,7 @@ void DerivationGoal::runChild() filesystem that we want in the chroot environment. */ auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug(format("bind mounting '%1%' to '%2%'") % source % target); + debug("bind mounting '%1%' to '%2%'", source, target); struct stat st; if (stat(source.c_str(), &st) == -1) { if (optional && errno == ENOENT) @@ -3238,16 +3277,16 @@ void DerivationGoal::runChild() /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) - throw SysError(format("cannot change directory to '%1%'") % chrootRootDir); + throw SysError("cannot change directory to '%1%'", chrootRootDir); if (mkdir("real-root", 0) == -1) throw SysError("cannot create real-root directory"); if (pivot_root(".", "real-root") == -1) - throw SysError(format("cannot pivot old root directory onto '%1%'") % (chrootRootDir + "/real-root")); + throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); if (chroot(".") == -1) - throw SysError(format("cannot change root directory to '%1%'") % chrootRootDir); + throw SysError("cannot change root directory to '%1%'", chrootRootDir); if (umount2("real-root", MNT_DETACH) == -1) throw SysError("cannot unmount real root filesystem"); @@ -3268,7 +3307,7 @@ void DerivationGoal::runChild() #endif if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError(format("changing into '%1%'") % tmpDir); + throw SysError("changing into '%1%'", tmpDir); /* Close all other file descriptors. */ closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); @@ -3407,9 +3446,9 @@ void DerivationGoal::runChild() sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { if (i.first != i.second.source) - throw Error(format( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") - % i.first % i.second.source); + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); string path = i.first; struct stat st; @@ -3500,7 +3539,7 @@ void DerivationGoal::runChild() else if (drv->builder == "builtin:unpack-channel") builtinUnpackChannel(drv2); else - throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); + throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); @@ -3510,7 +3549,7 @@ void DerivationGoal::runChild() execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - throw SysError(format("executing '%1%'") % drv->builder); + throw SysError("executing '%1%'", drv->builder); } catch (std::exception & e) { writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); @@ -3529,13 +3568,36 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv if (store.isStorePath(i)) result.insert(store.parseStorePath(i)); else if (drv.outputs.count(i)) - result.insert(drv.outputs.find(i)->second.path.clone()); + result.insert(drv.outputs.find(i)->second.path); else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; } +static void moveCheckToStore(const Path & src, const Path & dst) +{ + /* For the rename of directory to succeed, we must be running as root or + the directory must be made temporarily writable (to update the + directory's parent link ".."). */ + struct stat st; + if (lstat(src.c_str(), &st) == -1) { + throw SysError("getting attributes of path '%1%'", src); + } + + bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + + if (changePerm) + chmod_(src, st.st_mode | S_IWUSR); + + if (rename(src.c_str(), dst.c_str())) + throw SysError("renaming '%1%' to '%2%'", src, dst); + + if (changePerm) + chmod_(dst, st.st_mode); +} + + void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output @@ -3564,9 +3626,9 @@ void DerivationGoal::registerOutputs() output paths, and any paths that have been built via recursive Nix calls. */ StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p.clone()); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path.clone()); - for (auto & p : addedPaths) referenceablePaths.insert(p.clone()); + for (auto & p : inputPaths) referenceablePaths.insert(p); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & p : addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all @@ -3595,7 +3657,7 @@ void DerivationGoal::registerOutputs() replaceValidPath(path, actualPath); else if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1) - throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); + throw SysError("moving build output '%1%' from the sandbox to the Nix store", path); } if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); } @@ -3616,13 +3678,16 @@ void DerivationGoal::registerOutputs() user. */ if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError(format("suspicious ownership or permission on '%1%'; rejecting this build output") % path); + throw BuildError("suspicious ownership or permission on '%1%'; rejecting this build output", path); #endif /* Apply hash rewriting if necessary. */ bool rewritten = false; if (!outputRewrites.empty()) { - printError(format("warning: rewriting hashes in '%1%'; cross fingers") % path); + logWarning({ + .name = "Rewriting hashes", + .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) + }); /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or @@ -3647,21 +3712,25 @@ void DerivationGoal::registerOutputs() if (fixedOutput) { - bool recursive; Hash h; - i.second.parseHashInfo(recursive, h); + FileIngestionMethod outputHashMode; Hash h; + i.second.parseHashInfo(outputHashMode, h); - if (!recursive) { + if (outputHashMode == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( - format("output path '%1%' should be a non-executable regular file") % path); + "output path '%1%' should be a non-executable regular file " + "since recursive hashing is not enabled (outputHashMode=flat)", + path); } /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); + Hash h2 = outputHashMode == FileIngestionMethod::Recursive + ? hashPath(h.type, actualPath).first + : hashFile(h.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); if (h != h2) { @@ -3670,7 +3739,7 @@ void DerivationGoal::registerOutputs() worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); + worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); @@ -3690,7 +3759,7 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = makeFixedOutputCA(recursive, h2); + ca = makeFixedOutputCA(outputHashMode, h2); } /* Get rid of all weird permissions. This also checks that @@ -3714,8 +3783,7 @@ void DerivationGoal::registerOutputs() if (settings.runDiffHook || settings.keepFailed) { Path dst = worker.store.toRealPath(path + checkSuffix); deletePath(dst); - if (rename(actualPath.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + moveCheckToStore(actualPath, dst); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), @@ -3723,10 +3791,10 @@ void DerivationGoal::registerOutputs() path, dst, worker.store.printStorePath(drvPath), tmpDir); throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", - worker.store.printStorePath(drvPath), path, dst); + worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst); } else throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", - worker.store.printStorePath(drvPath), path); + worker.store.printStorePath(drvPath), worker.store.toRealPath(path)); } /* Since we verified the build, it's now ultimately trusted. */ @@ -3759,7 +3827,7 @@ void DerivationGoal::registerOutputs() info.narHash = hash.first; info.narSize = hash.second; info.references = std::move(references); - info.deriver = drvPath.clone(); + info.deriver = drvPath; info.ultimate = true; info.ca = ca; worker.store.signPathInfo(info); @@ -3783,10 +3851,10 @@ void DerivationGoal::registerOutputs() result.isNonDeterministic = true; Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); - auto msg = prevExists - ? fmt("output '%s' of '%s' differs from '%s' from previous round", + hintformat hint = prevExists + ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : fmt("output '%s' of '%s' differs from previous round", + : hintfmt("output '%s' of '%s' differs from previous round", worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); handleDiffHook( @@ -3796,9 +3864,13 @@ void DerivationGoal::registerOutputs() worker.store.printStorePath(drvPath), tmpDir); if (settings.enforceDeterminism) - throw NotDeterministic(msg); + throw NotDeterministic(hint); + + logError({ + .name = "Output determinism error", + .hint = hint + }); - printError(msg); curRound = nrRounds; // we know enough, bail out early } } @@ -3870,23 +3942,23 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) uint64_t closureSize = 0; StorePathSet pathsDone; std::queue<StorePath> pathsLeft; - pathsLeft.push(path.clone()); + pathsLeft.push(path); while (!pathsLeft.empty()) { - auto path = pathsLeft.front().clone(); + auto path = pathsLeft.front(); pathsLeft.pop(); - if (!pathsDone.insert(path.clone()).second) continue; + if (!pathsDone.insert(path).second) continue; auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; for (auto & ref : i->second.references) - pathsLeft.push(ref.clone()); + pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; for (auto & ref : info->references) - pathsLeft.push(ref.clone()); + pathsLeft.push(ref); } } @@ -3912,7 +3984,9 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); - auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references); + auto used = recursive + ? getClosure(info.path).first + : info.references; if (recursive && checks.ignoreSelfRefs) used.erase(info.path); @@ -3922,10 +3996,10 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) for (auto & i : used) if (allowed) { if (!spec.count(i)) - badPaths.insert(i.clone()); + badPaths.insert(i); } else { if (spec.count(i)) - badPaths.insert(i.clone()); + badPaths.insert(i); } if (!badPaths.empty()) { @@ -4014,7 +4088,7 @@ Path DerivationGoal::openLogFile() settings.compressLog ? ".bz2" : ""); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError(format("creating log file '%1%'") % logFileName); + if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); logFileSink = std::make_shared<FdSink>(fdLogFile.get()); @@ -4060,11 +4134,11 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - printError( - format("%1% killed after writing more than %2% bytes of log output") - % getName() % settings.maxLogSize); killChild(); - done(BuildResult::LogLimitExceeded); + done( + BuildResult::LogLimitExceeded, + Error("%s killed after writing more than %d bytes of log output", + getName(), settings.maxLogSize)); return; } @@ -4106,13 +4180,8 @@ void DerivationGoal::flushLine() ; else { - if (settings.verboseBuild && - (settings.printRepeatedBuilds || curRound == 1)) - printError(currentLogLine); - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - } + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); act->result(resBuildLogLine, currentLogLine); } @@ -4130,7 +4199,7 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) bool good = worker.store.isValidPath(i.second.path) && (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path.clone()); + if (good == returnValid) result.insert(i.second.path); } return result; } @@ -4146,15 +4215,16 @@ void DerivationGoal::addHashRewrite(const StorePath & path) deletePath(worker.store.printStorePath(p)); inputRewrites[h1] = h2; outputRewrites[h2] = h1; - redirectedOutputs.insert_or_assign(path.clone(), std::move(p)); + redirectedOutputs.insert_or_assign(path, std::move(p)); } -void DerivationGoal::done(BuildResult::Status status, const string & msg) +void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex) { result.status = status; - result.errorMsg = msg; - amDone(result.success() ? ecSuccess : ecFailed); + if (ex) + result.errorMsg = ex->what(); + amDone(result.success() ? ecSuccess : ecFailed, ex); if (result.status == BuildResult::TimedOut) worker.timedOut = true; if (result.status == BuildResult::PermanentFailure) @@ -4220,10 +4290,10 @@ private: GoalState state; public: - SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair); ~SubstitutionGoal(); - void timedOut() override { abort(); }; + void timedOut(Error && ex) override { abort(); }; string key() override { @@ -4246,18 +4316,13 @@ public: void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; - StorePath getStorePath() { return storePath.clone(); } - - void amDone(ExitCode result) override - { - Goal::amDone(result); - } + StorePath getStorePath() { return storePath; } }; -SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair) : Goal(worker) - , storePath(std::move(storePath)) + , storePath(storePath) , repair(repair) { state = &SubstitutionGoal::init; @@ -4352,7 +4417,7 @@ void SubstitutionGoal::tryNext() throw; } catch (Error & e) { if (settings.tryFallback) { - printError(e.what()); + logError(e.info()); tryNext(); return; } @@ -4378,8 +4443,11 @@ void SubstitutionGoal::tryNext() && !sub->isTrusted && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { - printError("warning: substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)); + logWarning({ + .name = "Invalid path signature", + .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), worker.store.printStorePath(storePath)) + }); tryNext(); return; } @@ -4488,7 +4556,7 @@ void SubstitutionGoal::finished() return; } - worker.markContentsGood(storePath.clone()); + worker.markContentsGood(storePath); printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); @@ -4522,7 +4590,6 @@ void SubstitutionGoal::handleEOF(int fd) if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } - ////////////////////////////////////////////////////////////////////// @@ -4559,10 +4626,10 @@ Worker::~Worker() GoalPtr Worker::makeDerivationGoal(const StorePath & path, const StringSet & wantedOutputs, BuildMode buildMode) { - GoalPtr goal = derivationGoals[path.clone()].lock(); // FIXME + GoalPtr goal = derivationGoals[path].lock(); // FIXME if (!goal) { - goal = std::make_shared<DerivationGoal>(path.clone(), wantedOutputs, *this, buildMode); - derivationGoals.insert_or_assign(path.clone(), goal); + goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); + derivationGoals.insert_or_assign(path, goal); wakeUp(goal); } else (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); @@ -4570,10 +4637,10 @@ GoalPtr Worker::makeDerivationGoal(const StorePath & path, } -std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(StorePath && drvPath, +std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto goal = std::make_shared<DerivationGoal>(std::move(drvPath), drv, *this, buildMode); + auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); wakeUp(goal); return goal; } @@ -4581,10 +4648,10 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(StorePath && drv GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) { - GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME + GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { - goal = std::make_shared<SubstitutionGoal>(path.clone(), *this, repair); - substitutionGoals.insert_or_assign(path.clone(), goal); + goal = std::make_shared<SubstitutionGoal>(path, *this, repair); + substitutionGoals.insert_or_assign(path, goal); wakeUp(goal); } return goal; @@ -4613,7 +4680,7 @@ void Worker::removeGoal(GoalPtr goal) topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals (unless keepGoing was set). */ - if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) + if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) topGoals.clear(); } @@ -4739,9 +4806,9 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( - "unable to start any build; either increase '--max-jobs' " - "or enable remote builds"); + if (awake.empty() && 0 == settings.maxBuildJobs) + throw Error("unable to start any build; either increase '--max-jobs' " + "or enable remote builds"); assert(!awake.empty()); } } @@ -4754,7 +4821,6 @@ void Worker::run(const Goals & _topGoals) assert(!settings.keepGoing || children.empty()); } - void Worker::waitForInput() { printMsg(lvlVomit, "waiting for children"); @@ -4766,8 +4832,7 @@ void Worker::waitForInput() terminated. */ bool useTimeout = false; - struct timeval timeout; - timeout.tv_usec = 0; + long timeout = 0; auto before = steady_time_point::clock::now(); /* If we're monitoring for silence on stdout/stderr, or if there @@ -4785,7 +4850,7 @@ void Worker::waitForInput() nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); } if (nearest != steady_time_point::max()) { - timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count()); + timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count()); useTimeout = true; } @@ -4793,33 +4858,29 @@ void Worker::waitForInput() up after a few seconds at most. */ if (!waitingForAWhile.empty()) { useTimeout = true; - if (lastWokenUp == steady_time_point::min()) - printError("waiting for locks or build slots..."); if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout.tv_sec = std::max(1L, + timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>( lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); } else lastWokenUp = steady_time_point::min(); if (useTimeout) - vomit("sleeping %d seconds", timeout.tv_sec); + vomit("sleeping %d seconds", timeout); /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ - fd_set fds; - FD_ZERO(&fds); - int fdMax = 0; + std::vector<struct pollfd> pollStatus; + std::map <int, int> fdToPollStatus; for (auto & i : children) { for (auto & j : i.fds) { - if (j >= FD_SETSIZE) - throw Error("reached FD_SETSIZE limit"); - FD_SET(j, &fds); - if (j >= fdMax) fdMax = j + 1; + pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); + fdToPollStatus[j] = pollStatus.size() - 1; } } - if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { + if (poll(pollStatus.data(), pollStatus.size(), + useTimeout ? timeout * 1000 : -1) == -1) { if (errno == EINTR) return; throw SysError("waiting for input"); } @@ -4840,20 +4901,20 @@ void Worker::waitForInput() set<int> fds2(j->fds); std::vector<unsigned char> buffer(4096); for (auto & k : fds2) { - if (FD_ISSET(k, &fds)) { + if (pollStatus.at(fdToPollStatus.at(k)).revents) { ssize_t rd = read(k, buffer.data(), buffer.size()); // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? if (rd == 0 || (rd == -1 && errno == EIO)) { - debug(format("%1%: got EOF") % goal->getName()); + debug("%1%: got EOF", goal->getName()); goal->handleEOF(k); j->fds.erase(k); } else if (rd == -1) { if (errno != EINTR) throw SysError("%s: read failed", goal->getName()); } else { - printMsg(lvlVomit, format("%1%: read %2% bytes") - % goal->getName() % rd); + printMsg(lvlVomit, "%1%: read %2% bytes", + goal->getName(), rd); string data((char *) buffer.data(), rd); j->lastOutput = after; goal->handleChildOutput(k, data); @@ -4861,26 +4922,24 @@ void Worker::waitForInput() } } - if (goal->getExitCode() == Goal::ecBusy && + if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { - printError( - format("%1% timed out after %2% seconds of silence") - % goal->getName() % settings.maxSilentTime); - goal->timedOut(); + goal->timedOut(Error( + "%1% timed out after %2% seconds of silence", + goal->getName(), settings.maxSilentTime)); } - else if (goal->getExitCode() == Goal::ecBusy && + else if (goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { - printError( - format("%1% timed out after %2% seconds") - % goal->getName() % settings.buildTimeout); - goal->timedOut(); + goal->timedOut(Error( + "%1% timed out after %2% seconds", + goal->getName(), settings.buildTimeout)); } } @@ -4937,15 +4996,19 @@ bool Worker::pathContentsGood(const StorePath & path) Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } - pathContentsGoodCache.insert_or_assign(path.clone(), res); - if (!res) printError("path '%s' is corrupted or missing!", store.printStorePath(path)); + pathContentsGoodCache.insert_or_assign(path, res); + if (!res) + logError({ + .name = "Corrupted path", + .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) + }); return res; } -void Worker::markContentsGood(StorePath && path) +void Worker::markContentsGood(const StorePath & path) { - pathContentsGoodCache.insert_or_assign(std::move(path), true); + pathContentsGoodCache.insert_or_assign(path, true); } @@ -4982,24 +5045,35 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, worker.run(goals); StorePathSet failed; + std::optional<Error> ex; for (auto & i : goals) { - if (i->getExitCode() != Goal::ecSuccess) { + if (i->ex) { + if (ex) + logError(i->ex->info()); + else + ex = i->ex; + } + if (i->exitCode != Goal::ecSuccess) { DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); if (i2) failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath()); } } - if (!failed.empty()) + if (failed.size() == 1 && ex) { + ex->status = worker.exitStatus(); + throw *ex; + } else if (!failed.empty()) { + if (ex) logError(ex->info()); throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + } } - BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath.clone(), drv, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); BuildResult result; @@ -5020,7 +5094,7 @@ void LocalStore::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {StorePathWithOutputs(path)}); + primeCache(*this, {{path}}); Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); @@ -5028,8 +5102,13 @@ void LocalStore::ensurePath(const StorePath & path) worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + if (goal->exitCode != Goal::ecSuccess) { + if (goal->ex) { + goal->ex->status = worker.exitStatus(); + throw *goal->ex; + } else + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + } } @@ -5041,7 +5120,7 @@ void LocalStore::repairPath(const StorePath & path) worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) { + if (goal->exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 1b802d908..802fb87bc 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -22,7 +22,10 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, srcFiles = readDirectory(srcDir); } catch (SysError & e) { if (e.errNo == ENOTDIR) { - printError("warning: not including '%s' in the user environment because it's not a directory", srcDir); + logWarning({ + .name = "Create links - directory", + .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) + }); return; } throw; @@ -41,7 +44,10 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw SysError("getting status of '%1%'", srcFile); } catch (SysError & e) { if (e.errNo == ENOENT || e.errNo == ENOTDIR) { - printError("warning: skipping dangling symlink '%s'", dstFile); + logWarning({ + .name = "Create links - skipping symlink", + .hint = hintfmt("skipping dangling symlink '%s'", dstFile) + }); continue; } throw; @@ -72,15 +78,15 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, if (!S_ISDIR(lstat(target).st_mode)) throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); + throw SysError("unlinking '%1%'", dstFile); if (mkdir(dstFile.c_str(), 0755) == -1) - throw SysError(format("creating directory '%1%'")); + throw SysError("creating directory '%1%'", dstFile); createLinks(state, target, dstFile, state.priorities[dstFile]); createLinks(state, srcFile, dstFile, priority); continue; } } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); + throw SysError("getting status of '%1%'", dstFile); } else { @@ -99,11 +105,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, if (prevPriority < priority) continue; if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); + throw SysError("unlinking '%1%'", dstFile); } else if (S_ISDIR(dstSt.st_mode)) throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); + throw SysError("getting status of '%1%'", dstFile); } createSymlink(srcFile, dstFile); diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index f6ae5d2e6..2048f8f87 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -1,5 +1,5 @@ #include "builtins.hh" -#include "download.hh" +#include "filetransfer.hh" #include "store-api.hh" #include "archive.hh" #include "compression.hh" @@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto getAttr = [&](const string & name) { auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; @@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto mainUrl = getAttr("url"); bool unpack = get(drv.env, "unpack").value_or("") == "1"; - /* Note: have to use a fresh downloader here because we're in + /* Note: have to use a fresh fileTransfer here because we're in a forked process. */ - auto downloader = makeDownloader(); + auto fileTransfer = makeFileTransfer(); auto fetch = [&](const std::string & url) { @@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) /* No need to do TLS verification, because we check the hash of the result anyway. */ - DownloadRequest request(url); + FileTransferRequest request(url); request.verifyTLS = false; request.decompress = false; auto decompressor = makeDecompressionSink( unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); - downloader->download(std::move(request), *decompressor); + fileTransfer->download(std::move(request), *decompressor); decompressor->finish(); }); @@ -54,7 +54,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto executable = drv.env.find("executable"); if (executable != drv.env.end() && executable->second == "1") { if (chmod(storePath.c_str(), 0755) == -1) - throw SysError(format("making '%1%' executable") % storePath); + throw SysError("making '%1%' executable", storePath); } }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 8e9f9d71b..e370e278c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -73,6 +73,18 @@ struct TunnelLogger : public Logger enqueueMsg(*buf.s); } + void logEI(const ErrorInfo & ei) override + { + if (ei.level > verbosity) return; + + std::stringstream oss; + oss << ei; + + StringSink buf; + buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n"); + enqueueMsg(*buf.s); + } + /* startWork() means that we're starting an operation for which we want to send out stderr to the client. */ void startWork() @@ -281,7 +293,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto path = store->parseStorePath(readString(from)); logger->startWork(); StorePathSet paths; // FIXME - paths.insert(path.clone()); + paths.insert(path); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); to << (res.count(path) != 0); @@ -315,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, StorePathSet paths; if (op == wopQueryReferences) for (auto & i : store->queryPathInfo(path)->references) - paths.insert(i.clone()); + paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); else if (op == wopQueryValidDerivers) @@ -329,8 +341,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopQueryDerivationOutputNames: { auto path = store->parseStorePath(readString(from)); logger->startWork(); - StringSet names; - names = store->queryDerivationOutputNames(path); + auto names = store->readDerivation(path).outputNames(); logger->stopWork(); to << names; break; @@ -355,20 +366,26 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddToStore: { - bool fixed, recursive; std::string s, baseName; - from >> baseName >> fixed /* obsolete */ >> recursive >> s; - /* Compatibility hack. */ - if (!fixed) { - s = "sha256"; - recursive = true; + FileIngestionMethod method; + { + bool fixed; uint8_t recursive; + from >> baseName >> fixed /* obsolete */ >> recursive >> s; + if (recursive > (uint8_t) FileIngestionMethod::Recursive) + throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); + method = FileIngestionMethod { recursive }; + /* Compatibility hack. */ + if (!fixed) { + s = "sha256"; + method = FileIngestionMethod::Recursive; + } } HashType hashAlgo = parseHashType(s); TeeSource savedNAR(from); RetrieveRegularNARSink savedRegular; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { /* Get the entire NAR dump from the client and save it to a string so that we can pass it to addToStoreFromDump(). */ @@ -380,7 +397,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + auto path = store->addToStoreFromDump( + method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s, + baseName, + method, + hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -572,9 +593,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - StorePathSet paths; - paths.insert(path.clone()); // FIXME - store->querySubstitutablePathInfos(paths, infos); + store->querySubstitutablePathInfos({path}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -735,7 +754,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } default: - throw Error(format("invalid operation %1%") % op); + throw Error("invalid operation %1%", op); } } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 973ddc86a..b95f7bfdc 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,13 +9,13 @@ namespace nix { -void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const +void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const { - recursive = false; + recursive = FileIngestionMethod::Flat; string algo = hashAlgo; if (string(algo, 0, 2) == "r:") { - recursive = true; + recursive = FileIngestionMethod::Recursive; algo = string(algo, 2); } @@ -27,28 +27,6 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const } -BasicDerivation::BasicDerivation(const BasicDerivation & other) - : platform(other.platform) - , builder(other.builder) - , args(other.args) - , env(other.env) -{ - for (auto & i : other.outputs) - outputs.insert_or_assign(i.first, - DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); - for (auto & i : other.inputSrcs) - inputSrcs.insert(i.clone()); -} - - -Derivation::Derivation(const Derivation & other) - : BasicDerivation(other) -{ - for (auto & i : other.inputDrvs) - inputDrvs.insert_or_assign(i.first.clone(), i.second); -} - - const StorePath & BasicDerivation::findOutput(const string & id) const { auto i = outputs.find(id); @@ -67,9 +45,9 @@ bool BasicDerivation::isBuiltin() const StorePath writeDerivation(ref<Store> store, const Derivation & drv, std::string_view name, RepairFlag repair) { - auto references = cloneStorePathSet(drv.inputSrcs); + auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs) - references.insert(i.first.clone()); + references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ @@ -87,7 +65,7 @@ static void expect(std::istream & str, const string & s) char s2[s.size()]; str.read(s2, s.size()); if (string(s2, s.size()) != s) - throw FormatError(format("expected string '%1%'") % s); + throw FormatError("expected string '%1%'", s); } @@ -114,7 +92,7 @@ static Path parsePath(std::istream & str) { string s = parseString(str); if (s.size() == 0 || s[0] != '/') - throw FormatError(format("bad path '%1%' in derivation") % s); + throw FormatError("bad path '%1%' in derivation", s); return s; } @@ -155,7 +133,11 @@ static Derivation parseDerivation(const Store & store, const string & s) expect(str, ","); auto hashAlgo = parseString(str); expect(str, ","); auto hash = parseString(str); expect(str, ")"); - drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); + drv.outputs.emplace(id, DerivationOutput { + .path = std::move(path), + .hashAlgo = std::move(hashAlgo), + .hash = std::move(hash) + }); } /* Parse the list of input derivations. */ @@ -196,7 +178,7 @@ Derivation readDerivation(const Store & store, const Path & drvPath) try { return parseDerivation(store, readFile(drvPath)); } catch (FormatError & e) { - throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); + throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); } } @@ -204,6 +186,12 @@ Derivation readDerivation(const Store & store, const Path & drvPath) Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); + return readDerivation(drvPath); +} + + +Derivation Store::readDerivation(const StorePath & drvPath) +{ auto accessor = getFSAccessor(); try { return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); @@ -377,8 +365,8 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput auto h = drvHashes.find(i.first); if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); - h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, - readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first; + h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store, + store.readDerivation(i.first), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } @@ -405,11 +393,20 @@ StorePathSet BasicDerivation::outputPaths() const { StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path.clone()); + paths.insert(i.second.path); return paths; } +StringSet BasicDerivation::outputNames() const +{ + StringSet names; + for (auto & i : outputs) + names.insert(i.first); + return names; +} + + Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) { drv.outputs.clear(); @@ -419,7 +416,11 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) auto path = store.parseStorePath(readString(in)); auto hashAlgo = readString(in); auto hash = readString(in); - drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); + drv.outputs.emplace(name, DerivationOutput { + .path = std::move(path), + .hashAlgo = std::move(hashAlgo), + .hash = std::move(hash) + }); } drv.inputSrcs = readStorePaths<StorePathSet>(store, in); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index a3651de23..856aae59f 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -17,12 +17,7 @@ struct DerivationOutput StorePath path; std::string hashAlgo; /* hash used for expected hash computation */ std::string hash; /* expected hash, may be null */ - DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash) - : path(std::move(path)) - , hashAlgo(std::move(hashAlgo)) - , hash(std::move(hash)) - { } - void parseHashInfo(bool & recursive, Hash & hash) const; + void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; }; typedef std::map<string, DerivationOutput> DerivationOutputs; @@ -43,7 +38,6 @@ struct BasicDerivation StringPairs env; BasicDerivation() { } - explicit BasicDerivation(const BasicDerivation & other); virtual ~BasicDerivation() { }; /* Return the path corresponding to the output identifier `id' in @@ -58,6 +52,8 @@ struct BasicDerivation /* Return the output paths of a derivation. */ StorePathSet outputPaths() const; + /* Return the output names of a derivation. */ + StringSet outputNames() const; }; struct Derivation : BasicDerivation @@ -69,8 +65,6 @@ struct Derivation : BasicDerivation std::map<std::string, StringSet> * actualInputs = nullptr) const; Derivation() { } - Derivation(Derivation && other) = default; - explicit Derivation(const Derivation & other); }; diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4692d1a7b..cb9da027d 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,3 +1,4 @@ +#include "serialise.hh" #include "store-api.hh" #include "archive.hh" #include "worker-protocol.hh" @@ -56,7 +57,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashAndWriteSink.currentHash(); if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(), hash.to_string()); + printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic @@ -100,9 +101,11 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces if (readInt(source) == 1) readString(source); - addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); + // Can't use underlying source, which would have been exhausted + auto source = StringSource { *tee.source.data }; + addToStore(info, source, NoRepair, checkSigs, accessor); - res.push_back(info.path.clone()); + res.push_back(info.path); } return res; diff --git a/src/libstore/download.cc b/src/libstore/filetransfer.cc index 149c84765..c954ace7f 100644 --- a/src/libstore/download.cc +++ b/src/libstore/filetransfer.cc @@ -1,14 +1,10 @@ -#include "download.hh" +#include "filetransfer.hh" #include "util.hh" #include "globals.hh" -#include "hash.hh" #include "store-api.hh" -#include "archive.hh" #include "s3.hh" #include "compression.hh" -#include "pathlocks.hh" #include "finally.hh" -#include "tarfile.hh" #ifdef ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -31,13 +27,9 @@ using namespace std::string_literals; namespace nix { -DownloadSettings downloadSettings; +FileTransferSettings fileTransferSettings; -static GlobalConfig::Register r1(&downloadSettings); - -CachedDownloadRequest::CachedDownloadRequest(const std::string & uri) - : uri(uri), ttl(settings.tarballTtl) -{ } +static GlobalConfig::Register r1(&fileTransferSettings); std::string resolveUri(const std::string & uri) { @@ -47,21 +39,21 @@ std::string resolveUri(const std::string & uri) return uri; } -struct CurlDownloader : public Downloader +struct curlFileTransfer : public FileTransfer { CURLM * curlm = 0; std::random_device rd; std::mt19937 mt19937; - struct DownloadItem : public std::enable_shared_from_this<DownloadItem> + struct TransferItem : public std::enable_shared_from_this<TransferItem> { - CurlDownloader & downloader; - DownloadRequest request; - DownloadResult result; + curlFileTransfer & fileTransfer; + FileTransferRequest request; + FileTransferResult result; Activity act; bool done = false; // whether either the success or failure function has been called - Callback<DownloadResult> callback; + Callback<FileTransferResult> callback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object std::string status; @@ -80,19 +72,36 @@ struct CurlDownloader : public Downloader curl_off_t writtenToSink = 0; - DownloadItem(CurlDownloader & downloader, - const DownloadRequest & request, - Callback<DownloadResult> && callback) - : downloader(downloader) + /* Get the HTTP status code, or 0 for other protocols. */ + long getHTTPStatus() + { + long httpStatus = 0; + long protocol = 0; + curl_easy_getinfo(req, CURLINFO_PROTOCOL, &protocol); + if (protocol == CURLPROTO_HTTP || protocol == CURLPROTO_HTTPS) + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + return httpStatus; + } + + TransferItem(curlFileTransfer & fileTransfer, + const FileTransferRequest & request, + Callback<FileTransferResult> && callback) + : fileTransfer(fileTransfer) , request(request) - , act(*logger, lvlTalkative, actDownload, + , act(*logger, lvlTalkative, actFileTransfer, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { if (this->request.dataCallback) { - writtenToSink += len; - this->request.dataCallback((char *) data, len); + auto httpStatus = getHTTPStatus(); + + /* Only write data to the sink if this is a + successful response. */ + if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { + writtenToSink += len; + this->request.dataCallback((char *) data, len); + } } else this->result.data->append((char *) data, len); }) @@ -103,17 +112,17 @@ struct CurlDownloader : public Downloader requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); } - ~DownloadItem() + ~TransferItem() { if (req) { if (active) - curl_multi_remove_handle(downloader.curlm, req); + curl_multi_remove_handle(fileTransfer.curlm, req); curl_easy_cleanup(req); } if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)); + fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -157,7 +166,7 @@ struct CurlDownloader : public Downloader static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb); + return ((TransferItem *) userp)->writeCallback(contents, size, nmemb); } size_t headerCallback(void * contents, size_t size, size_t nmemb) @@ -199,7 +208,7 @@ struct CurlDownloader : public Downloader static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb); + return ((TransferItem *) userp)->headerCallback(contents, size, nmemb); } int progressCallback(double dltotal, double dlnow) @@ -214,7 +223,7 @@ struct CurlDownloader : public Downloader static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) { - return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); + return ((TransferItem *) userp)->progressCallback(dltotal, dlnow); } static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) @@ -238,7 +247,7 @@ struct CurlDownloader : public Downloader static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) { - return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); + return ((TransferItem *) userp)->readCallback(buffer, size, nitems); } void init() @@ -249,7 +258,7 @@ struct CurlDownloader : public Downloader if (verbosity >= lvlVomit) { curl_easy_setopt(req, CURLOPT_VERBOSE, 1); - curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); + curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); } curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); @@ -258,19 +267,19 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt(req, CURLOPT_USERAGENT, ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + - (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); #endif #if LIBCURL_VERSION_NUM >= 0x072f00 - if (downloadSettings.enableHttp2) + if (fileTransferSettings.enableHttp2) curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); else curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); #endif - curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); + curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper); curl_easy_setopt(req, CURLOPT_WRITEDATA, this); - curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper); + curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper); curl_easy_setopt(req, CURLOPT_HEADERDATA, this); curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); @@ -298,10 +307,10 @@ struct CurlDownloader : public Downloader curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); } - curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get()); curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); - curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get()); /* If no file exist in the specified path, curl continues to work anyway as if netrc support was disabled. */ @@ -317,8 +326,7 @@ struct CurlDownloader : public Downloader void finish(CURLcode code) { - long httpStatus = 0; - curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + auto httpStatus = getHTTPStatus(); char * effectiveUriCStr; curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); @@ -345,7 +353,7 @@ struct CurlDownloader : public Downloader failEx(writeException); else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 0 /* other protocol */)) { result.cached = httpStatus == 304; act.progress(result.bodySize, result.bodySize); @@ -390,6 +398,7 @@ struct CurlDownloader : public Downloader case CURLE_SSL_CACERT_BADFILE: case CURLE_TOO_MANY_REDIRECTS: case CURLE_WRITE_ERROR: + case CURLE_UNSUPPORTED_PROTOCOL: err = Misc; break; default: // Shut up warnings @@ -401,14 +410,14 @@ struct CurlDownloader : public Downloader auto exc = code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted - ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) : httpStatus != 0 - ? DownloadError(err, + ? FileTransferError(err, fmt("unable to %s '%s': HTTP error %d", request.verb(), request.uri, httpStatus) + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) ) - : DownloadError(err, + : FileTransferError(err, fmt("unable to %s '%s': %s (%d)", request.verb(), request.uri, curl_easy_strerror(code), code)); @@ -422,13 +431,13 @@ struct CurlDownloader : public Downloader || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); if (writtenToSink) warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else warn("%s; retrying in %d ms", exc.what(), ms); embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); - downloader.enqueueItem(shared_from_this()); + fileTransfer.enqueueItem(shared_from_this()); } else fail(exc); @@ -439,12 +448,12 @@ struct CurlDownloader : public Downloader struct State { struct EmbargoComparator { - bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) { + bool operator() (const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2) { return i1->embargo > i2->embargo; } }; bool quit = false; - std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming; + std::priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator> incoming; }; Sync<State> state_; @@ -456,7 +465,7 @@ struct CurlDownloader : public Downloader std::thread workerThread; - CurlDownloader() + curlFileTransfer() : mt19937(rd()) { static std::once_flag globalInit; @@ -469,7 +478,7 @@ struct CurlDownloader : public Downloader #endif #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, - downloadSettings.httpConnections.get()); + fileTransferSettings.httpConnections.get()); #endif wakeupPipe.create(); @@ -478,7 +487,7 @@ struct CurlDownloader : public Downloader workerThread = std::thread([&]() { workerThreadEntry(); }); } - ~CurlDownloader() + ~curlFileTransfer() { stopWorkerThread(); @@ -504,7 +513,7 @@ struct CurlDownloader : public Downloader stopWorkerThread(); }); - std::map<CURL *, std::shared_ptr<DownloadItem>> items; + std::map<CURL *, std::shared_ptr<TransferItem>> items; bool quit = false; @@ -517,7 +526,7 @@ struct CurlDownloader : public Downloader int running; CURLMcode mc = curl_multi_perform(curlm, &running); if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc)); + throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc)); /* Set the promises of any finished requests. */ CURLMsg * msg; @@ -547,7 +556,7 @@ struct CurlDownloader : public Downloader vomit("download thread waiting for %d ms", sleepTimeMs); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); + throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc)); nextWakeup = std::chrono::steady_clock::time_point(); @@ -561,7 +570,7 @@ struct CurlDownloader : public Downloader throw SysError("reading curl wakeup socket"); } - std::vector<std::shared_ptr<DownloadItem>> incoming; + std::vector<std::shared_ptr<TransferItem>> incoming; auto now = std::chrono::steady_clock::now(); { @@ -599,7 +608,11 @@ struct CurlDownloader : public Downloader workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - printError("unexpected error in download thread: %s", e.what()); + logError({ + .name = "File transfer", + .hint = hintfmt("unexpected error in download thread: %s", + e.what()) + }); } { @@ -609,7 +622,7 @@ struct CurlDownloader : public Downloader } } - void enqueueItem(std::shared_ptr<DownloadItem> item) + void enqueueItem(std::shared_ptr<TransferItem> item) { if (item->request.data && !hasPrefix(item->request.uri, "http://") @@ -641,8 +654,8 @@ struct CurlDownloader : public Downloader } #endif - void enqueueDownload(const DownloadRequest & request, - Callback<DownloadResult> callback) override + void enqueueFileTransfer(const FileTransferRequest & request, + Callback<FileTransferResult> callback) override { /* Ugly hack to support s3:// URIs. */ if (hasPrefix(request.uri, "s3://")) { @@ -660,9 +673,9 @@ struct CurlDownloader : public Downloader // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); - DownloadResult res; + FileTransferResult res; if (!s3Res.data) - throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); res.data = s3Res.data; callback(std::move(res)); #else @@ -672,26 +685,26 @@ struct CurlDownloader : public Downloader return; } - enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback))); + enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback))); } }; -ref<Downloader> getDownloader() +ref<FileTransfer> getFileTransfer() { - static ref<Downloader> downloader = makeDownloader(); - return downloader; + static ref<FileTransfer> fileTransfer = makeFileTransfer(); + return fileTransfer; } -ref<Downloader> makeDownloader() +ref<FileTransfer> makeFileTransfer() { - return make_ref<CurlDownloader>(); + return make_ref<curlFileTransfer>(); } -std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request) +std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) { - auto promise = std::make_shared<std::promise<DownloadResult>>(); - enqueueDownload(request, - {[promise](std::future<DownloadResult> fut) { + auto promise = std::make_shared<std::promise<FileTransferResult>>(); + enqueueFileTransfer(request, + {[promise](std::future<FileTransferResult> fut) { try { promise->set_value(fut.get()); } catch (...) { @@ -701,15 +714,21 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & return promise->get_future(); } -DownloadResult Downloader::download(const DownloadRequest & request) +FileTransferResult FileTransfer::download(const FileTransferRequest & request) +{ + return enqueueFileTransfer(request).get(); +} + +FileTransferResult FileTransfer::upload(const FileTransferRequest & request) { - return enqueueDownload(request).get(); + /* Note: this method is the same as download, but helps in readability */ + return enqueueFileTransfer(request).get(); } -void Downloader::download(DownloadRequest && request, Sink & sink) +void FileTransfer::download(FileTransferRequest && request, Sink & sink) { /* Note: we can't call 'sink' via request.dataCallback, because - that would cause the sink to execute on the downloader + that would cause the sink to execute on the fileTransfer thread. If 'sink' is a coroutine, this will fail. Also, if the sink is expensive (e.g. one that does decompression and writing to the Nix store), it would stall the download thread too much. @@ -755,8 +774,8 @@ void Downloader::download(DownloadRequest && request, Sink & sink) state->avail.notify_one(); }; - enqueueDownload(request, - {[_state](std::future<DownloadResult> fut) { + enqueueFileTransfer(request, + {[_state](std::future<FileTransferResult> fut) { auto state(_state->lock()); state->quit = true; try { @@ -801,140 +820,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink) } } -CachedDownloadResult Downloader::downloadCached( - ref<Store> store, const CachedDownloadRequest & request) -{ - auto url = resolveUri(request.uri); - - auto name = request.name; - if (name == "") { - auto p = url.rfind('/'); - if (p != string::npos) name = string(url, p + 1); - } - - std::optional<StorePath> expectedStorePath; - if (request.expectedHash) { - expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); - if (store->isValidPath(*expectedStorePath)) { - CachedDownloadResult result; - result.storePath = store->printStorePath(*expectedStorePath); - result.path = store->toRealPath(result.storePath); - return result; - } - } - - Path cacheDir = getCacheDir() + "/nix/tarballs"; - createDirs(cacheDir); - - string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); - - Path dataFile = cacheDir + "/" + urlHash + ".info"; - Path fileLink = cacheDir + "/" + urlHash + "-file"; - - PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); - - std::optional<StorePath> storePath; - - string expectedETag; - - bool skip = false; - - CachedDownloadResult result; - - if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = store->parseStorePath(readLink(fileLink)); - // FIXME - store->addTempRoot(*storePath); - if (store->isValidPath(*storePath)) { - auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); - if (ss.size() >= 3 && ss[0] == url) { - time_t lastChecked; - if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { - skip = true; - result.effectiveUri = request.uri; - result.etag = ss[1]; - } else if (!ss[1].empty()) { - debug(format("verifying previous ETag '%1%'") % ss[1]); - expectedETag = ss[1]; - } - } - } else - storePath.reset(); - } - - if (!skip) { - - try { - DownloadRequest request2(url); - request2.expectedETag = expectedETag; - auto res = download(request2); - result.effectiveUri = res.effectiveUri; - result.etag = res.etag; - - if (!res.cached) { - StringSink sink; - dumpString(*res.data, sink); - Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); - ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); - info.narHash = hashString(htSHA256, *sink.s); - info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(false, hash); - store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - storePath = info.path.clone(); - } - - assert(storePath); - replaceSymlink(store->printStorePath(*storePath), fileLink); - - writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); - } catch (DownloadError & e) { - if (!storePath) throw; - warn("warning: %s; using cached result", e.msg()); - result.etag = expectedETag; - } - } - - if (request.unpack) { - Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked"; - PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); - std::optional<StorePath> unpackedStorePath; - if (pathExists(unpackedLink)) { - unpackedStorePath = store->parseStorePath(readLink(unpackedLink)); - // FIXME - store->addTempRoot(*unpackedStorePath); - if (!store->isValidPath(*unpackedStorePath)) - unpackedStorePath.reset(); - } - if (!unpackedStorePath) { - printInfo("unpacking '%s'...", url); - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir); - auto members = readDirectory(tmpDir); - if (members.size() != 1) - throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); - auto topDir = tmpDir + "/" + members.begin()->name; - unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); - } - replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink); - storePath = std::move(*unpackedStorePath); - } - - if (expectedStorePath && *storePath != *expectedStorePath) { - unsigned int statusCode = 102; - Hash gotHash = request.unpack - ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first - : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))); - throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - url, request.expectedHash.to_string(), gotHash.to_string()); - } - - result.storePath = store->printStorePath(*storePath); - result.path = store->toRealPath(result.storePath); - return result; -} - - bool isUri(const string & s) { if (s.compare(0, 8, "channel:") == 0) return true; diff --git a/src/libstore/download.hh b/src/libstore/filetransfer.hh index 5a131c704..11dca2fe0 100644 --- a/src/libstore/download.hh +++ b/src/libstore/filetransfer.hh @@ -9,7 +9,7 @@ namespace nix { -struct DownloadSettings : Config +struct FileTransferSettings : Config { Setting<bool> enableHttp2{this, true, "http2", "Whether to enable HTTP/2 support."}; @@ -31,15 +31,15 @@ struct DownloadSettings : Config "How often Nix will attempt to download a file before giving up."}; }; -extern DownloadSettings downloadSettings; +extern FileTransferSettings fileTransferSettings; -struct DownloadRequest +struct FileTransferRequest { std::string uri; std::string expectedETag; bool verifyTLS = true; bool head = false; - size_t tries = downloadSettings.tries; + size_t tries = fileTransferSettings.tries; unsigned int baseRetryTimeMs = 250; ActivityId parentAct; bool decompress = true; @@ -47,7 +47,7 @@ struct DownloadRequest std::string mimeType; std::function<void(char *, size_t)> dataCallback; - DownloadRequest(const std::string & uri) + FileTransferRequest(const std::string & uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() @@ -56,7 +56,7 @@ struct DownloadRequest } }; -struct DownloadResult +struct FileTransferResult { bool cached = false; std::string etag; @@ -65,74 +65,53 @@ struct DownloadResult uint64_t bodySize = 0; }; -struct CachedDownloadRequest -{ - std::string uri; - bool unpack = false; - std::string name; - Hash expectedHash; - unsigned int ttl; - - CachedDownloadRequest(const std::string & uri); - CachedDownloadRequest() = delete; -}; - -struct CachedDownloadResult -{ - // Note: 'storePath' may be different from 'path' when using a - // chroot store. - Path storePath; - Path path; - std::optional<std::string> etag; - std::string effectiveUri; -}; - class Store; -struct Downloader +struct FileTransfer { - virtual ~Downloader() { } + virtual ~FileTransfer() { } - /* Enqueue a download request, returning a future to the result of - the download. The future may throw a DownloadError + /* Enqueue a data transfer request, returning a future to the result of + the download. The future may throw a FileTransferError exception. */ - virtual void enqueueDownload(const DownloadRequest & request, - Callback<DownloadResult> callback) = 0; + virtual void enqueueFileTransfer(const FileTransferRequest & request, + Callback<FileTransferResult> callback) = 0; - std::future<DownloadResult> enqueueDownload(const DownloadRequest & request); + std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request); /* Synchronously download a file. */ - DownloadResult download(const DownloadRequest & request); + FileTransferResult download(const FileTransferRequest & request); + + /* Synchronously upload a file. */ + FileTransferResult upload(const FileTransferRequest & request); /* Download a file, writing its data to a sink. The sink will be invoked on the thread of the caller. */ - void download(DownloadRequest && request, Sink & sink); - - /* Check if the specified file is already in ~/.cache/nix/tarballs - and is more recent than ‘tarball-ttl’ seconds. Otherwise, - use the recorded ETag to verify if the server has a more - recent version, and if so, download it to the Nix store. */ - CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request); + void download(FileTransferRequest && request, Sink & sink); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; -/* Return a shared Downloader object. Using this object is preferred +/* Return a shared FileTransfer object. Using this object is preferred because it enables connection reuse and HTTP/2 multiplexing. */ -ref<Downloader> getDownloader(); +ref<FileTransfer> getFileTransfer(); -/* Return a new Downloader object. */ -ref<Downloader> makeDownloader(); +/* Return a new FileTransfer object. */ +ref<FileTransfer> makeFileTransfer(); -class DownloadError : public Error +class FileTransferError : public Error { public: - Downloader::Error error; - DownloadError(Downloader::Error error, const FormatOrString & fs) - : Error(fs), error(error) + FileTransfer::Error error; + template<typename... Args> + FileTransferError(FileTransfer::Error error, const Args & ... args) + : Error(args...), error(error) { } }; bool isUri(const string & s); +/* Resolve deprecated 'channel:<foo>' URLs. */ +std::string resolveUri(const std::string & uri); + } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0c3d89611..57fb20845 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -38,10 +38,10 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType) AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fdGCLock) - throw SysError(format("opening global GC lock '%1%'") % fnGCLock); + throw SysError("opening global GC lock '%1%'", fnGCLock); if (!lockFile(fdGCLock.get(), lockType, false)) { - printError(format("waiting for the big garbage collector lock...")); + printInfo("waiting for the big garbage collector lock..."); lockFile(fdGCLock.get(), lockType, true); } @@ -65,8 +65,8 @@ static void makeSymlink(const Path & link, const Path & target) /* Atomically replace the old one. */ if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError(format("cannot rename '%1%' to '%2%'") - % tempLink % link); + throw SysError("cannot rename '%1%' to '%2%'", + tempLink , link); } @@ -91,15 +91,15 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, Path gcRoot(canonPath(_gcRoot)); if (isInStore(gcRoot)) - throw Error(format( + throw Error( "creating a garbage collector root (%1%) in the Nix store is forbidden " - "(are you running nix-build inside the store?)") % gcRoot); + "(are you running nix-build inside the store?)", gcRoot); if (indirect) { /* Don't clobber the link if it already exists and doesn't point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot); + throw Error("cannot create symlink '%1%'; already exists", gcRoot); makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); } @@ -109,10 +109,10 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") - throw Error(format( + throw Error( "path '%1%' is not a valid garbage collector root; " - "it's not in the directory '%2%'") - % gcRoot % rootsDir); + "it's not in the directory '%2%'", + gcRoot, rootsDir); } if (baseNameOf(gcRoot) == std::string(storePath.to_string())) @@ -128,11 +128,13 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, gcroots directory. */ if (settings.checkRootReachability) { auto roots = findRoots(false); - if (roots[storePath.clone()].count(gcRoot) == 0) - printError( - "warning: '%1%' is not in a directory where the garbage collector looks for roots; " - "therefore, '%2%' might be removed by the garbage collector", - gcRoot, printStorePath(storePath)); + if (roots[storePath].count(gcRoot) == 0) + logWarning({ + .name = "GC root", + .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " + "therefore, '%2%' might be removed by the garbage collector", + gcRoot, printStorePath(storePath)) + }); } /* Grab the global GC root, causing us to block while a GC is in @@ -170,7 +172,7 @@ void LocalStore::addTempRoot(const StorePath & path) way. */ struct stat st; if (fstat(state->fdTempRoots.get(), &st) == -1) - throw SysError(format("statting '%1%'") % fnTempRoots); + throw SysError("statting '%1%'", fnTempRoots); if (st.st_size == 0) break; /* The garbage collector deleted this file before we could @@ -202,6 +204,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) /* Read the `temproots' directory for per-process temporary root files. */ for (auto & i : readDirectory(tempRootsDir)) { + if (i.name[0] == '.') { + // Ignore hidden files. Some package managers (notably portage) create + // those to keep the directory alive. + continue; + } Path path = tempRootsDir + "/" + i.name; pid_t pid = std::stoi(i.name); @@ -211,7 +218,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) if (!*fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; - throw SysError(format("opening temporary roots file '%1%'") % path); + throw SysError("opening temporary roots file '%1%'", path); } /* This should work, but doesn't, for some reason. */ @@ -222,7 +229,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) only succeed if the owning process has died. In that case we don't care about its temporary roots. */ if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); + printInfo("removing stale temporary roots file '%1%'", path); unlink(path.c_str()); writeFull(fd->get(), "d"); continue; @@ -398,7 +405,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (!fdDir) { if (errno == ENOENT || errno == EACCES) continue; - throw SysError(format("opening %1%") % fdStr); + throw SysError("opening %1%", fdStr); } struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { @@ -408,13 +415,13 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (errno) { if (errno == ESRCH) continue; - throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); + throw SysError("iterating /proc/%1%/fd", ent->d_name); } fdDir.reset(); try { auto mapFile = fmt("/proc/%s/maps", ent->d_name); - auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n"); + auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile), "\n"); for (const auto & line : mapLines) { auto match = std::smatch{}; if (std::regex_match(line, match, mapRegex)) @@ -422,7 +429,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } auto envFile = fmt("/proc/%s/environ", ent->d_name); - auto envString = readFile(envFile, true); + auto envString = readFile(envFile); auto env_end = std::sregex_iterator{}; for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) unchecked[i->str()].emplace(envFile); @@ -471,9 +478,9 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (!isValidPath(path)) continue; debug("got additional root '%1%'", pathS); if (censor) - roots[path.clone()].insert(censored); + roots[path].insert(censored); else - roots[path.clone()].insert(links.begin(), links.end()); + roots[path].insert(links.begin(), links.end()); } } @@ -536,7 +543,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) struct stat st; if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT) return; - throw SysError(format("getting status of %1%") % realPath); + throw SysError("getting status of %1%", realPath); } printInfo(format("deleting '%1%'") % path); @@ -554,10 +561,10 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) // size. try { if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("making '%1%' writable") % realPath); + throw SysError("making '%1%' writable", realPath); Path tmp = trashDir + "/" + std::string(baseNameOf(path)); if (rename(realPath.c_str(), tmp.c_str())) - throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp); + throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp); state.bytesInvalidated += size; } catch (SysError & e) { if (e.errNo == ENOSPC) { @@ -585,11 +592,11 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.roots.count(path)) { debug("cannot delete '%1%' because it's a root", printStorePath(path)); - state.alive.insert(path.clone()); + state.alive.insert(path); return true; } - visited.insert(path.clone()); + visited.insert(path); if (!isValidPath(path)) return false; @@ -603,7 +610,7 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.gcKeepDerivations && path.isDerivation()) { for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i.clone()); + incoming.insert(i); } /* If keep-outputs is set, then don't delete this path if there @@ -611,13 +618,13 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.gcKeepOutputs) { auto derivers = queryValidDerivers(path); for (auto & i : derivers) - incoming.insert(i.clone()); + incoming.insert(i); } for (auto & i : incoming) if (i != path) if (canReachRoot(state, visited, i)) { - state.alive.insert(path.clone()); + state.alive.insert(path); return true; } @@ -661,7 +668,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) ‘nix-store --delete’ doesn't have the unexpected effect of recursing into derivations and outputs. */ for (auto & i : visited) - state.dead.insert(i.clone()); + state.dead.insert(i); if (state.shouldDelete) deletePathRecursive(state, path); } @@ -676,7 +683,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) void LocalStore::removeUnusedLinks(const GCState & state) { AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + if (!dir) throw SysError("opening directory '%1%'", linksDir); long long actualSize = 0, unsharedSize = 0; @@ -689,7 +696,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (lstat(path.c_str(), &st) == -1) - throw SysError(format("statting '%1%'") % path); + throw SysError("statting '%1%'", path); if (st.st_nlink != 1) { actualSize += st.st_size; @@ -700,14 +707,14 @@ void LocalStore::removeUnusedLinks(const GCState & state) printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); if (unlink(path.c_str()) == -1) - throw SysError(format("deleting '%1%'") % path); + throw SysError("deleting '%1%'", path); state.results.bytesFreed += st.st_size; } struct stat st; if (stat(linksDir.c_str(), &st) == -1) - throw SysError(format("statting '%1%'") % linksDir); + throw SysError("statting '%1%'", linksDir); long long overhead = st.st_blocks * 512ULL; printInfo(format("note: currently hard linking saves %.2f MiB") @@ -742,12 +749,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ - printError("finding garbage collector roots..."); + printInfo("finding garbage collector roots..."); Roots rootMap; if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true); - for (auto & i : rootMap) state.roots.insert(i.first.clone()); + for (auto & i : rootMap) state.roots.insert(i.first); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths @@ -756,8 +763,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) Roots tempRoots; findTempRoots(fds, tempRoots, true); for (auto & root : tempRoots) { - state.tempRoots.insert(root.first.clone()); - state.roots.insert(root.first.clone()); + state.tempRoots.insert(root.first); + state.roots.insert(root.first); } /* After this point the set of roots or temporary roots cannot @@ -794,14 +801,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } else if (options.maxFreed > 0) { if (state.shouldDelete) - printError("deleting garbage..."); + printInfo("deleting garbage..."); else - printError("determining live/dead paths..."); + printInfo("determining live/dead paths..."); try { AutoCloseDir dir(opendir(realStoreDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir); + if (!dir) throw SysError("opening directory '%1%'", realStoreDir); /* Read the store and immediately delete all paths that aren't valid. When using --max-freed etc., deleting @@ -863,7 +870,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Clean up the links directory. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { - printError("deleting unused links..."); + printInfo("deleting unused links..."); removeUnusedLinks(state); } @@ -884,7 +891,7 @@ void LocalStore::autoGC(bool sync) if (statvfs(realStoreDir.c_str(), &st)) throw SysError("getting filesystem info about '%s'", realStoreDir); - return (uint64_t) st.f_bavail * st.f_bsize; + return (uint64_t) st.f_bavail * st.f_frsize; }; std::shared_future<void> future; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 7e97f3c22..bee94cbd8 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -31,6 +31,7 @@ Settings::Settings() , nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))) , nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) + , nixUserConfFiles(getUserConfigFiles()) , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) @@ -77,11 +78,27 @@ void loadConfFile() ~/.nix/nix.conf or the command line. */ globalConfig.resetOverriden(); + auto files = settings.nixUserConfFiles; + for (auto file = files.rbegin(); file != files.rend(); file++) { + globalConfig.applyConfigFile(*file); + } +} + +std::vector<Path> getUserConfigFiles() +{ + // Use the paths specified in NIX_USER_CONF_FILES if it has been defined + auto nixConfFiles = getEnv("NIX_USER_CONF_FILES"); + if (nixConfFiles.has_value()) { + return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":"); + } + + // Use the paths specified by the XDG spec + std::vector<Path> files; auto dirs = getConfigDirs(); - // Iterate over them in reverse so that the ones appearing first in the path take priority - for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { - globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + for (auto & dir : dirs) { + files.insert(files.end(), dir + "/nix/nix.conf"); } + return files; } unsigned int Settings::getDefaultCores() @@ -113,7 +130,7 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name) void Settings::requireExperimentalFeature(const std::string & name) { if (!isExperimentalFeatureEnabled(name)) - throw Error("experimental Nix feature '%s' is disabled", name); + throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name); } bool Settings::isWSL1() @@ -150,21 +167,24 @@ template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out) template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category) { - args.mkFlag() - .longName(name) - .description("Enable sandboxing.") - .handler([=](std::vector<std::string> ss) { override(smEnabled); }) - .category(category); - args.mkFlag() - .longName("no-" + name) - .description("Disable sandboxing.") - .handler([=](std::vector<std::string> ss) { override(smDisabled); }) - .category(category); - args.mkFlag() - .longName("relaxed-" + name) - .description("Enable sandboxing, but allow builds to disable it.") - .handler([=](std::vector<std::string> ss) { override(smRelaxed); }) - .category(category); + args.addFlag({ + .longName = name, + .description = "Enable sandboxing.", + .category = category, + .handler = {[=]() { override(smEnabled); }} + }); + args.addFlag({ + .longName = "no-" + name, + .description = "Disable sandboxing.", + .category = category, + .handler = {[=]() { override(smDisabled); }} + }); + args.addFlag({ + .longName = "relaxed-" + name, + .description = "Enable sandboxing, but allow builds to disable it.", + .category = category, + .handler = {[=]() { override(smRelaxed); }} + }); } void MaxBuildJobsSetting::set(const std::string & str) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 3aa3653f3..2fbcafff8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -53,9 +53,12 @@ public: /* The directory where state is stored. */ Path nixStateDir; - /* The directory where configuration files are stored. */ + /* The directory where system configuration files are stored. */ Path nixConfDir; + /* A list of user configuration files to load. */ + std::vector<Path> nixUserConfFiles; + /* The directory where internal helper programs are stored. */ Path nixLibexecDir; @@ -268,7 +271,7 @@ public: "listed in 'trusted-public-keys'."}; Setting<StringSet> extraPlatforms{this, - std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{}, + std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, "extra-platforms", "Additional platforms that can be built on the local system. " "These may be supported natively (e.g. armv7 on some aarch64 CPUs " @@ -351,12 +354,21 @@ public: Setting<Paths> pluginFiles{this, {}, "plugin-files", "Plugins to dynamically load at nix initialization time."}; + Setting<std::string> githubAccessToken{this, "", "github-access-token", + "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."}; + Setting<Strings> experimentalFeatures{this, {}, "experimental-features", "Experimental Nix features to enable."}; bool isExperimentalFeatureEnabled(const std::string & name); void requireExperimentalFeature(const std::string & name); + + Setting<bool> allowDirty{this, true, "allow-dirty", + "Whether to allow dirty Git/Mercurial trees."}; + + Setting<bool> warnDirty{this, true, "warn-dirty", + "Whether to warn about dirty Git/Mercurial trees."}; }; @@ -369,6 +381,9 @@ void initPlugins(); void loadConfFile(); +// Used by the Settings constructor +std::vector<Path> getUserConfigFiles(); + extern const string nixVersion; } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 011794c62..451a64785 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,5 +1,5 @@ #include "binary-cache-store.hh" -#include "download.hh" +#include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" @@ -85,14 +85,14 @@ protected: checkEnabled(); try { - DownloadRequest request(cacheUri + "/" + path); + FileTransferRequest request(cacheUri + "/" + path); request.head = true; - getDownloader()->download(request); + getFileTransfer()->download(request); return true; - } catch (DownloadError & e) { + } catch (FileTransferError & e) { /* S3 buckets return 403 if a file doesn't exist and the bucket is unlistable, so treat 403 as 404. */ - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return false; maybeDisable(); throw; @@ -103,19 +103,19 @@ protected: const std::string & data, const std::string & mimeType) override { - auto req = DownloadRequest(cacheUri + "/" + path); + auto req = FileTransferRequest(cacheUri + "/" + path); req.data = std::make_shared<string>(data); // FIXME: inefficient req.mimeType = mimeType; try { - getDownloader()->download(req); - } catch (DownloadError & e) { + getFileTransfer()->upload(req); + } catch (FileTransferError & e) { throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); } } - DownloadRequest makeRequest(const std::string & path) + FileTransferRequest makeRequest(const std::string & path) { - DownloadRequest request(cacheUri + "/" + path); + FileTransferRequest request(cacheUri + "/" + path); return request; } @@ -124,9 +124,9 @@ protected: checkEnabled(); auto request(makeRequest(path)); try { - getDownloader()->download(std::move(request), sink); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + getFileTransfer()->download(std::move(request), sink); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); maybeDisable(); throw; @@ -142,12 +142,12 @@ protected: auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); - getDownloader()->enqueueDownload(request, - {[callbackPtr, this](std::future<DownloadResult> result) { + getFileTransfer()->enqueueFileTransfer(request, + {[callbackPtr, this](std::future<FileTransferResult> result) { try { (*callbackPtr)(result.get().data); - } catch (DownloadError & e) { - if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) return (*callbackPtr)(std::shared_ptr<std::string>()); maybeDisable(); callbackPtr->rethrow(); @@ -174,4 +174,3 @@ static RegisterStoreImplementation regStore([]( }); } - diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 458266be0..45c70fad6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -195,7 +195,7 @@ struct LegacySSHStore : public Store { unsupported("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override { unsupported("addToStore"); } @@ -256,7 +256,7 @@ struct LegacySSHStore : public Store conn->to.flush(); for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) - out.insert(i.clone()); + out.insert(i); } StorePathSet queryValidPaths(const StorePathSet & paths, diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 363be1443..48aca478c 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -74,7 +74,7 @@ static void atomicWrite(const Path & path, const std::string & s) AutoDelete del(tmp, false); writeFile(tmp, s); if (rename(tmp.c_str(), path.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % tmp % path); + throw SysError("renaming '%1%' to '%2%'", tmp, path); del.cancel(); } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index aa5abd835..dd96d2578 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -22,7 +22,7 @@ struct LocalStoreAccessor : public FSAccessor { Path storePath = store->toStorePath(path); if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + throw InvalidPath("path '%1%' is not a valid store path", storePath); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -33,11 +33,11 @@ struct LocalStoreAccessor : public FSAccessor struct stat st; if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); } if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error(format("file '%1%' has unsupported type") % path); + throw Error("file '%1%' has unsupported type", path); return { S_ISREG(st.st_mode) ? Type::tRegular : @@ -90,13 +90,13 @@ const string LocalFSStore::drvsLogDir = "drvs"; std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_) { - auto path = path_.clone(); + auto path = path_; if (!path.isDerivation()) { try { auto info = queryPathInfo(path); if (!info->deriver) return nullptr; - path = info->deriver->clone(); + path = *info->deriver; } catch (InvalidPath &) { return nullptr; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ae7513ad8..0df4374b9 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -87,18 +87,22 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist") - % settings.buildUsersGroup); + logError({ + .name = "'build-users-group' not found", + .hint = hintfmt( + "warning: the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup) + }); else { struct stat st; if (stat(realStoreDir.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % realStoreDir); + throw SysError("getting attributes of path '%1%'", realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) - throw SysError(format("changing ownership of path '%1%'") % realStoreDir); + throw SysError("changing ownership of path '%1%'", realStoreDir); if (chmod(realStoreDir.c_str(), perm) == -1) - throw SysError(format("changing permissions on path '%1%'") % realStoreDir); + throw SysError("changing permissions on path '%1%'", realStoreDir); } } } @@ -109,12 +113,12 @@ LocalStore::LocalStore(const Params & params) struct stat st; while (path != "/") { if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); if (S_ISLNK(st.st_mode)) - throw Error(format( + throw Error( "the path '%1%' is a symlink; " - "this is not allowed for the Nix store and its parent directories") - % path); + "this is not allowed for the Nix store and its parent directories", + path); path = dirOf(path); } } @@ -147,7 +151,7 @@ LocalStore::LocalStore(const Params & params) globalLock = openLockFile(globalLockPath.c_str(), true); if (!lockFile(globalLock.get(), ltRead, false)) { - printError("waiting for the big Nix store lock..."); + printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -155,8 +159,8 @@ LocalStore::LocalStore(const Params & params) upgrade. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) - throw Error(format("current Nix store schema is version %1%, but I only support %2%") - % curSchema % nixSchemaVersion); + throw Error("current Nix store schema is version %1%, but I only support %2%", + curSchema, nixSchemaVersion); else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; @@ -178,7 +182,7 @@ LocalStore::LocalStore(const Params & params) "please upgrade Nix to version 1.11 first."); if (!lockFile(globalLock.get(), ltWrite, false)) { - printError("waiting for exclusive access to the Nix store..."); + printInfo("waiting for exclusive access to the Nix store..."); lockFile(globalLock.get(), ltWrite, true); } @@ -256,7 +260,7 @@ LocalStore::~LocalStore() } if (future.valid()) { - printError("waiting for auto-GC to finish on exit..."); + printInfo("waiting for auto-GC to finish on exit..."); future.get(); } @@ -284,7 +288,7 @@ int LocalStore::getSchema() if (pathExists(schemaPath)) { string s = readFile(schemaPath); if (!string2Int(s, curSchema)) - throw Error(format("'%1%' is corrupt") % schemaPath); + throw Error("'%1%' is corrupt", schemaPath); } return curSchema; } @@ -293,7 +297,7 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { if (access(dbDir.c_str(), R_OK | W_OK)) - throw SysError(format("Nix database directory '%1%' is not writable") % dbDir); + throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ string dbPath = dbDir + "/db.sqlite"; @@ -367,7 +371,7 @@ void LocalStore::makeStoreWritable() throw SysError("setting up a private mount namespace"); if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError(format("remounting %1% writable") % realStoreDir); + throw SysError("remounting %1% writable", realStoreDir); } #endif } @@ -388,7 +392,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct | 0444 | (st.st_mode & S_IXUSR ? 0111 : 0); if (chmod(path.c_str(), mode) == -1) - throw SysError(format("changing mode of '%1%' to %2$o") % path % mode); + throw SysError("changing mode of '%1%' to %2$o", path, mode); } } @@ -406,7 +410,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct #else if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) #endif - throw SysError(format("changing modification time of '%1%'") % path); + throw SysError("changing modification time of '%1%'", path); } } @@ -415,7 +419,7 @@ void canonicaliseTimestampAndPermissions(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); canonicaliseTimestampAndPermissions(path, st); } @@ -430,17 +434,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe setattrlist() to remove other attributes as well. */ if (lchflags(path.c_str(), 0)) { if (errno != ENOTSUP) - throw SysError(format("clearing flags of path '%1%'") % path); + throw SysError("clearing flags of path '%1%'", path); } #endif struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); /* Really make sure that the path is of a supported type. */ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error(format("file '%1%' has an unsupported type") % path); + throw Error("file '%1%' has an unsupported type", path); #if __linux__ /* Remove extended attributes / ACLs. */ @@ -474,7 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { assert(!S_ISDIR(st.st_mode)); if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) - throw BuildError(format("invalid ownership on file '%1%'") % path); + throw BuildError("invalid ownership on file '%1%'", path); mode_t mode = st.st_mode & ~S_IFMT; assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); return; @@ -498,8 +502,8 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (!S_ISLNK(st.st_mode) && chown(path.c_str(), geteuid(), getegid()) == -1) #endif - throw SysError(format("changing owner of '%1%' to %2%") - % path % geteuid()); + throw SysError("changing owner of '%1%' to %2%", + path, geteuid()); } if (S_ISDIR(st.st_mode)) { @@ -518,11 +522,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino be a symlink, since we can't change its ownership. */ struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); if (st.st_uid != geteuid()) { assert(S_ISLNK(st.st_mode)); - throw Error(format("wrong ownership of top-level store path '%1%'") % path); + throw Error("wrong ownership of top-level store path '%1%'", path); } } @@ -557,10 +561,10 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - bool recursive; Hash h; - out->second.parseHashInfo(recursive, h); + FileIngestionMethod method; Hash h; + out->second.parseHashInfo(method, h); - check(makeFixedOutputPath(recursive, h, drvName), out->second.path, "out"); + check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); } else { @@ -580,7 +584,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -595,7 +599,7 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string())); + auto drv = readDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -615,7 +619,7 @@ uint64_t LocalStore::addValidPath(State & state, { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), + state_->pathInfoCache.upsert(std::string(info.path.hashPart()), PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) }); } @@ -627,7 +631,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { - auto info = std::make_shared<ValidPathInfo>(path.clone()); + auto info = std::make_shared<ValidPathInfo>(path); callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { auto state(_state.lock()); @@ -680,7 +684,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -717,7 +721,7 @@ StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteF { StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i.clone()); + if (isValidPath(i)) res.insert(i); return res; } @@ -785,26 +789,9 @@ StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) } -StringSet LocalStore::queryDerivationOutputNames(const StorePath & path) -{ - return retrySQLite<StringSet>([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); - - StringSet outputNames; - while (useQueryDerivationOutputs.next()) - outputNames.insert(useQueryDerivationOutputs.getStr(0)); - - return outputNames; - }); -} - - std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart) { - if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); + if (hashPart.size() != StorePath::HashLen) throw Error("invalid hash part"); Path prefix = storeDir + "/" + hashPart; @@ -829,7 +816,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) StorePathSet remaining; for (auto & i : paths) - remaining.insert(i.clone()); + remaining.insert(i); StorePathSet res; @@ -843,9 +830,9 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) StorePathSet remaining2; for (auto & path : remaining) if (valid.count(path)) - res.insert(path.clone()); + res.insert(path); else - remaining2.insert(path.clone()); + remaining2.insert(path); std::swap(remaining, remaining2); } @@ -867,16 +854,16 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto info = sub->queryPathInfo(path); auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); - infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ - info->deriver ? info->deriver->clone() : std::optional<StorePath>(), - cloneStorePathSet(info->references), + infos.insert_or_assign(path, SubstitutablePathInfo{ + info->deriver, + info->references, narInfo ? narInfo->fileSize : 0, info->narSize}); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { if (settings.tryFallback) - printError(e.what()); + logError(e.info()); else throw; } @@ -913,7 +900,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) updatePathInfo(*state, i); else addValidPath(*state, i, false); - paths.insert(i.path.clone()); + paths.insert(i.path); } for (auto & i : infos) { @@ -928,8 +915,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string()))); + checkDerivationOutputs(i.path, readDerivation(i.path)); } /* Do a topological sort of the paths. This will throw an @@ -956,7 +942,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(storePathToHash(printStorePath(path))); + state_->pathInfoCache.erase(std::string(path.hashPart())); } } @@ -1008,7 +994,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (info.ca == "" || !info.references.count(info.path)) hashSink = std::make_unique<HashSink>(htSHA256); else - hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path))); + hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart())); LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { size_t n = source.read(data, len); @@ -1022,7 +1008,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string()); + printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1043,11 +1029,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, - bool recursive, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) { Hash h = hashString(hashAlgo, dump); - auto dstPath = makeFixedOutputPath(recursive, h, name); + auto dstPath = makeFixedOutputPath(method, h, name); addTempRoot(dstPath); @@ -1067,7 +1053,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam autoGC(); - if (recursive) { + if (method == FileIngestionMethod::Recursive) { StringSource source(dump); restorePath(realPath, source); } else @@ -1080,7 +1066,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ HashResult hash; - if (recursive) { + if (method == FileIngestionMethod::Recursive) { hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else @@ -1088,10 +1074,10 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam optimisePath(realPath); // FIXME: combine with hashPath() - ValidPathInfo info(dstPath.clone()); + ValidPathInfo info(dstPath); info.narHash = hash.first; info.narSize = hash.second; - info.ca = makeFixedOutputCA(recursive, h); + info.ca = makeFixedOutputCA(method, h); registerValidPath(info); } @@ -1103,7 +1089,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); @@ -1111,12 +1097,12 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (recursive) + if (method == FileIngestionMethod::Recursive) dumpPath(srcPath, sink, filter); else sink.s = make_ref<std::string>(readFile(srcPath)); - return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); + return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair); } @@ -1151,11 +1137,11 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, optimisePath(realPath); - ValidPathInfo info(dstPath.clone()); + ValidPathInfo info(dstPath); info.narHash = narHash; info.narSize = sink.s->size(); - info.references = cloneStorePathSet(references); - info.ca = "text:" + hash.to_string(); + info.references = references; + info.ca = "text:" + hash.to_string(Base32, true); registerValidPath(info); } @@ -1205,7 +1191,7 @@ void LocalStore::invalidatePathChecked(const StorePath & path) bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { - printError(format("reading the Nix store...")); + printInfo(format("reading the Nix store...")); bool errors = false; @@ -1237,12 +1223,15 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) Path linkPath = linksDir + "/" + link.name; string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); if (hash != link.name) { - printError( - "link '%s' was modified! expected hash '%s', got '%s'", - linkPath, link.name, hash); + logError({ + .name = "Invalid hash", + .hint = hintfmt( + "link '%s' was modified! expected hash '%s', got '%s'", + linkPath, link.name, hash) + }); if (repair) { if (unlink(linkPath.c_str()) == 0) - printError("removed link '%s'", linkPath); + printInfo("removed link '%s'", linkPath); else throw SysError("removing corrupt link '%s'", linkPath); } else { @@ -1266,14 +1255,17 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->ca == "" || !info->references.count(info->path)) hashSink = std::make_unique<HashSink>(info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path))); + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(), current.first.to_string()); + logError({ + .name = "Invalid hash - path modified", + .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) + }); if (repair) repairPath(i); else errors = true; } else { @@ -1281,14 +1273,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Fill in missing hashes. */ if (info->narHash == nullHash) { - printError("fixing missing hash on '%s'", printStorePath(i)); + printInfo("fixing missing hash on '%s'", printStorePath(i)); info->narHash = current.first; update = true; } /* Fill in missing narSize fields (from old stores). */ if (info->narSize == 0) { - printError("updating size field on '%s' to %s", printStorePath(i), current.second); + printInfo("updating size field on '%s' to %s", printStorePath(i), current.second); info->narSize = current.second; update = true; } @@ -1304,7 +1296,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* It's possible that the path got GC'ed, so ignore errors on invalid paths. */ if (isValidPath(i)) - printError("error: %s", e.msg()); + logError(e.info()); else warn(e.msg()); errors = true; @@ -1324,7 +1316,10 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, if (!done.insert(pathS).second) return; if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); + logError({ + .name = "Nix path not found", + .hint = hintfmt("path '%s' is not in the Nix store", pathS) + }); return; } @@ -1343,16 +1338,19 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, } if (canInvalidate) { - printError("path '%s' disappeared, removing from database...", pathS); + printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); invalidatePath(*state, path); } else { - printError("path '%s' disappeared, but it still has valid referrers!", pathS); + logError({ + .name = "Missing path with referrers", + .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) + }); if (repair) try { repairPath(path); } catch (Error & e) { - warn(e.msg()); + logWarning(e.info()); errors = true; } else errors = true; @@ -1392,7 +1390,7 @@ static void makeMutable(const Path & path) AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); if (fd == -1) { if (errno == ELOOP) return; // it's a symlink - throw SysError(format("opening file '%1%'") % path); + throw SysError("opening file '%1%'", path); } unsigned int flags = 0, old; @@ -1410,7 +1408,7 @@ static void makeMutable(const Path & path) void LocalStore::upgradeStore7() { if (getuid() != 0) return; - printError("removing immutable bits from the Nix store (this may take a while)..."); + printInfo("removing immutable bits from the Nix store (this may take a while)..."); makeMutable(realStoreDir); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 16aeab0ad..e17cc45ae 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -135,8 +135,6 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const StorePath & path) override; - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -149,7 +147,7 @@ public: std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive, HashType hashAlgo, + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; /* Like addToStore(), but the contents of the path are contained @@ -157,7 +155,7 @@ public: true) or simply the contents of a regular file (if recursive == false). */ StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index ac68c2342..aec4ed493 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil libnixrust +libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) @@ -31,7 +31,8 @@ ifeq ($(HAVE_SECCOMP), 1) libstore_LDFLAGS += -lseccomp endif -libstore_CXXFLAGS = \ +libstore_CXXFLAGS += \ + -I src/libutil -I src/libstore \ -DNIX_PREFIX=\"$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ -DNIX_DATA_DIR=\"$(datadir)\" \ diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9c47fe524..e68edb38c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -103,7 +103,7 @@ void Store::computeFSClosure(const StorePath & startPath, StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { StorePathSet paths; - paths.insert(startPath.clone()); + paths.insert(startPath); computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers); } @@ -141,11 +141,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); - state->willBuild.insert(drvPath.clone()); + state->willBuild.insert(drvPath); } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second })); }; auto checkOutput = [&]( @@ -157,9 +157,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - StorePathSet paths; // FIXME - paths.insert(outPath.clone()); - querySubstitutablePathInfos(paths, infos); + querySubstitutablePathInfos({outPath}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -170,10 +168,10 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (drvState->done) return; assert(drvState->left); drvState->left--; - drvState->outPaths.insert(outPath.clone()); + drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone()))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } )); } } } @@ -190,12 +188,12 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (!isValidPath(path.path)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path.path.clone()); + state->unknown.insert(path.path); return; } auto drv = make_ref<Derivation>(derivationFromPath(path.path)); - ParsedDerivation parsedDrv(path.path.clone(), *drv); + ParsedDerivation parsedDrv(StorePath(path.path), *drv); PathSet invalid; for (auto & j : drv->outputs) @@ -216,13 +214,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - StorePathSet paths; // FIXME - paths.insert(path.path.clone()); - querySubstitutablePathInfos(paths, infos); + querySubstitutablePathInfos({path.path}, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path.path.clone()); + state->unknown.insert(path.path); return; } @@ -231,13 +227,13 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, { auto state(state_.lock()); - state->willSubstitute.insert(path.path.clone()); + state->willSubstitute.insert(path.path); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref })); } }; @@ -260,12 +256,12 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) throw BuildError("cycle detected in the references of '%s' from '%s'", printStorePath(path), printStorePath(*parent)); - if (!visited.insert(path.clone()).second) return; - parents.insert(path.clone()); + if (!visited.insert(path).second) return; + parents.insert(path); StorePathSet references; try { - references = cloneStorePathSet(queryPathInfo(path)->references); + references = queryPathInfo(path)->references; } catch (InvalidPath &) { } @@ -275,7 +271,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) if (i != path && paths.count(i)) dfsVisit(i, &path); - sorted.push_back(path.clone()); + sorted.push_back(path); parents.erase(path); }; diff --git a/src/libstore/names.cc b/src/libstore/names.cc new file mode 100644 index 000000000..d1c8a6101 --- /dev/null +++ b/src/libstore/names.cc @@ -0,0 +1,107 @@ +#include "names.hh" +#include "util.hh" + + +namespace nix { + + +DrvName::DrvName() +{ + name = ""; +} + + +/* Parse a derivation name. The `name' part of a derivation name is + everything up to but not including the first dash *not* followed by + a letter. The `version' part is the rest (excluding the separating + dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', + '2.0.48'). */ +DrvName::DrvName(std::string_view s) : hits(0) +{ + name = fullName = std::string(s); + for (unsigned int i = 0; i < s.size(); ++i) { + /* !!! isalpha/isdigit are affected by the locale. */ + if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) { + name = s.substr(0, i); + version = s.substr(i + 1); + break; + } + } +} + + +bool DrvName::matches(DrvName & n) +{ + if (name != "*") { + if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended)); + if (!std::regex_match(n.name, *regex)) return false; + } + if (version != "" && version != n.version) return false; + return true; +} + + +string nextComponent(string::const_iterator & p, + const string::const_iterator end) +{ + /* Skip any dots and dashes (component separators). */ + while (p != end && (*p == '.' || *p == '-')) ++p; + + if (p == end) return ""; + + /* If the first character is a digit, consume the longest sequence + of digits. Otherwise, consume the longest sequence of + non-digit, non-separator characters. */ + string s; + if (isdigit(*p)) + while (p != end && isdigit(*p)) s += *p++; + else + while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) + s += *p++; + + return s; +} + + +static bool componentsLT(const string & c1, const string & c2) +{ + int n1, n2; + bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); + + if (c1Num && c2Num) return n1 < n2; + else if (c1 == "" && c2Num) return true; + else if (c1 == "pre" && c2 != "pre") return true; + else if (c2 == "pre") return false; + /* Assume that `2.3a' < `2.3.1'. */ + else if (c2Num) return true; + else if (c1Num) return false; + else return c1 < c2; +} + + +int compareVersions(const string & v1, const string & v2) +{ + string::const_iterator p1 = v1.begin(); + string::const_iterator p2 = v2.begin(); + + while (p1 != v1.end() || p2 != v2.end()) { + string c1 = nextComponent(p1, v1.end()); + string c2 = nextComponent(p2, v2.end()); + if (componentsLT(c1, c2)) return -1; + else if (componentsLT(c2, c1)) return 1; + } + + return 0; +} + + +DrvNames drvNamesFromArgs(const Strings & opArgs) +{ + DrvNames result; + for (auto & i : opArgs) + result.push_back(DrvName(i)); + return result; +} + + +} diff --git a/src/libstore/names.hh b/src/libstore/names.hh new file mode 100644 index 000000000..00e14b8c7 --- /dev/null +++ b/src/libstore/names.hh @@ -0,0 +1,32 @@ +#pragma once + +#include <memory> + +#include "types.hh" +#include <regex> + +namespace nix { + +struct DrvName +{ + string fullName; + string name; + string version; + unsigned int hits; + + DrvName(); + DrvName(std::string_view s); + bool matches(DrvName & n); + +private: + std::unique_ptr<std::regex> regex; +}; + +typedef list<DrvName> DrvNames; + +string nextComponent(string::const_iterator & p, + const string::const_iterator end); +int compareVersions(const string & v1, const string & v2); +DrvNames drvNamesFromArgs(const Strings & opArgs); + +} diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index b74480684..ca663d837 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -184,7 +184,7 @@ struct NarAccessor : public FSAccessor auto i = get(path); if (i.type != FSAccessor::Type::tDirectory) - throw Error(format("path '%1%' inside NAR file is not a directory") % path); + throw Error("path '%1%' inside NAR file is not a directory", path); StringSet res; for (auto & child : i.children) @@ -197,7 +197,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); if (i.type != FSAccessor::Type::tRegular) - throw Error(format("path '%1%' inside NAR file is not a regular file") % path); + throw Error("path '%1%' inside NAR file is not a regular file", path); if (getNarBytes) return getNarBytes(i.start, i.size); @@ -209,7 +209,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); if (i.type != FSAccessor::Type::tSymlink) - throw Error(format("path '%1%' inside NAR file is not a symlink") % path); + throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } }; diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 442541330..552970248 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -189,7 +189,7 @@ public: return {oInvalid, 0}; auto namePart = queryNAR.getStr(1); - auto narInfo = make_ref<NarInfo>(StorePath::fromBaseName(hashPart + "-" + namePart)); + auto narInfo = make_ref<NarInfo>(StorePath(hashPart + "-" + namePart)); narInfo->url = queryNAR.getStr(2); narInfo->compression = queryNAR.getStr(3); if (!queryNAR.isNull(4)) @@ -198,9 +198,9 @@ public: narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath::fromBaseName(r)); + narInfo->references.insert(StorePath(r)); if (!queryNAR.isNull(9)) - narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9)); + narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); narInfo->ca = queryNAR.getStr(11); @@ -230,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string()) + (info->narHash.to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1375094b5..bb4448c90 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,10 +4,10 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack + : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack { auto corrupt = [&]() { - throw Error(format("NAR info file '%1%' is corrupt") % whence); + throw Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { @@ -56,11 +56,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString<Strings>(value, " "); if (!references.empty()) corrupt(); for (auto & r : refs) - references.insert(StorePath::fromBaseName(r)); + references.insert(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") - deriver = StorePath::fromBaseName(value); + deriver = StorePath(value); } else if (name == "System") system = value; @@ -87,10 +87,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32) + "\n"; + res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32) + "\n"; + res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8ac382e9d..b2b2412a3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -19,9 +19,9 @@ static void makeWritable(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("changing writability of '%1%'") % path); + throw SysError("changing writability of '%1%'", path); } @@ -47,7 +47,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() InodeHash inodeHash; AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + if (!dir) throw SysError("opening directory '%1%'", linksDir); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -55,7 +55,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() // We don't care if we hit non-hash files, anything goes inodeHash.insert(dirent->d_ino); } - if (errno) throw SysError(format("reading directory '%1%'") % linksDir); + if (errno) throw SysError("reading directory '%1%'", linksDir); printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); @@ -68,7 +68,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa Strings names; AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); + if (!dir) throw SysError("opening directory '%1%'", path); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -83,7 +83,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa if (name == "." || name == "..") continue; names.push_back(name); } - if (errno) throw SysError(format("reading directory '%1%'") % path); + if (errno) throw SysError("reading directory '%1%'", path); return names; } @@ -96,7 +96,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); #if __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for @@ -130,7 +130,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - printError(format("skipping suspicious writable file '%1%'") % path); + logWarning({ + .name = "Suspicious file", + .hint = hintfmt("skipping suspicious writable file '%1%'", path) + }); return; } @@ -150,7 +153,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); + debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); /* Check if this is a known hash. */ Path linkPath = linksDir + "/" + hash.to_string(Base32, false); @@ -186,7 +189,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, current file with a hard link to that file. */ struct stat stLink; if (lstat(linkPath.c_str(), &stLink)) - throw SysError(format("getting attributes of path '%1%'") % linkPath); + throw SysError("getting attributes of path '%1%'", linkPath); if (st.st_ino == stLink.st_ino) { debug(format("'%1%' is already linked to '%2%'") % path % linkPath); @@ -194,7 +197,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } if (st.st_size != stLink.st_size) { - printError(format("removing corrupted link '%1%'") % linkPath); + logWarning({ + .name = "Corrupted link", + .hint = hintfmt("removing corrupted link '%1%'", linkPath) + }); unlink(linkPath.c_str()); goto retry; } @@ -229,7 +235,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - printError(format("unable to unlink '%1%'") % tempLink); + logError({ + .name = "Unlink error", + .hint = hintfmt("unable to unlink '%1%'", tempLink) + }); if (errno == EMLINK) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it @@ -238,7 +247,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); + throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); } stats.filesLinked++; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 45c033c66..24f848e46 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -4,8 +4,8 @@ namespace nix { -ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) - : drvPath(std::move(drvPath)), drv(drv) +ParsedDerivation::ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv) + : drvPath(drvPath), drv(drv) { /* Parse the __json attribute, if any. */ auto jsonAttr = drv.env.find("__json"); diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index d0cc70f9c..6ee172d81 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -12,7 +12,7 @@ class ParsedDerivation public: - ParsedDerivation(StorePath && drvPath, BasicDerivation & drv); + ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv); ~ParsedDerivation(); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 9a28aa96a..b3d8ce95c 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,38 +2,38 @@ namespace nix { -extern "C" { - rust::Result<StorePath> ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir); - rust::Result<StorePath> ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir); - rust::Result<StorePath> ffi_StorePath_fromBaseName(rust::StringSlice baseName); - rust::String ffi_StorePath_to_string(const StorePath & _this); - StorePath ffi_StorePath_clone(const StorePath & _this); - rust::StringSlice ffi_StorePath_name(const StorePath & _this); -} - -StorePath StorePath::make(std::string_view path, std::string_view storeDir) -{ - return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap(); -} - -StorePath StorePath::make(unsigned char hash[20], std::string_view name) -{ - return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap(); -} +MakeError(BadStorePath, Error); -StorePath StorePath::fromBaseName(std::string_view baseName) +static void checkName(std::string_view path, std::string_view name) { - return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap(); + if (name.empty()) + throw BadStorePath("store path '%s' has an empty name", path); + if (name.size() > 211) + throw BadStorePath("store path '%s' has a name longer than 211 characters", path); + for (auto c : name) + if (!((c >= '0' && c <= '9') + || (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '=')) + throw BadStorePath("store path '%s' contains illegal character '%s'", path, c); } -rust::String StorePath::to_string() const +StorePath::StorePath(std::string_view _baseName) + : baseName(_baseName) { - return ffi_StorePath_to_string(*this); + if (baseName.size() < HashLen + 1) + throw BadStorePath("'%s' is too short to be a valid store path", baseName); + for (auto c : hashPart()) + if (c == 'e' || c == 'o' || c == 'u' || c == 't' + || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'))) + throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c); + checkName(baseName, name()); } -StorePath StorePath::clone() const +StorePath::StorePath(const Hash & hash, std::string_view _name) + : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) { - return ffi_StorePath_clone(*this); + checkName(baseName, name()); } bool StorePath::isDerivation() const @@ -41,18 +41,14 @@ bool StorePath::isDerivation() const return hasSuffix(name(), drvExtension); } -std::string_view StorePath::name() const -{ - return ffi_StorePath_name(*this); -} - -StorePath StorePath::dummy( - StorePath::make( - (unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")); +StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath Store::parseStorePath(std::string_view path) const { - return StorePath::make(path, storeDir); + auto p = canonPath(std::string(path)); + if (dirOf(p) != storeDir) + throw BadStorePath("path '%s' is not in the Nix store", p); + return StorePath(baseNameOf(p)); } std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const @@ -78,9 +74,7 @@ StorePathSet Store::parseStorePathSet(const PathSet & paths) const std::string Store::printStorePath(const StorePath & path) const { - auto s = storeDir + "/"; - s += (std::string_view) path.to_string(); - return s; + return (storeDir + "/").append(path.to_string()); } PathSet Store::printStorePathSet(const StorePathSet & paths) const @@ -90,29 +84,6 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const return res; } -StorePathSet cloneStorePathSet(const StorePathSet & paths) -{ - StorePathSet res; - for (auto & p : paths) - res.insert(p.clone()); - return res; -} - -StorePathSet storePathsToSet(const StorePaths & paths) -{ - StorePathSet res; - for (auto & p : paths) - res.insert(p.clone()); - return res; -} - -StorePathSet singleton(const StorePath & path) -{ - StorePathSet res; - res.insert(path.clone()); - return res; -} - std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s) { size_t n = s.find("!"); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index c90bb1fff..aaebd3ec3 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,59 +1,59 @@ #pragma once -#include "rust-ffi.hh" +#include "types.hh" namespace nix { -/* See path.rs. */ -struct StorePath; - class Store; +struct Hash; -extern "C" { - void ffi_StorePath_drop(void *); - bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b); - bool ffi_StorePath_eq(const StorePath & a, const StorePath & b); - unsigned char * ffi_StorePath_hash_data(const StorePath & p); -} - -struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> +class StorePath { - StorePath() = delete; + std::string baseName; + +public: - static StorePath make(std::string_view path, std::string_view storeDir); + /* Size of the hash part of store paths, in base-32 characters. */ + constexpr static size_t HashLen = 32; // i.e. 160 bits - static StorePath make(unsigned char hash[20], std::string_view name); + StorePath() = delete; - static StorePath fromBaseName(std::string_view baseName); + StorePath(std::string_view baseName); - rust::String to_string() const; + StorePath(const Hash & hash, std::string_view name); + + std::string_view to_string() const + { + return baseName; + } bool operator < (const StorePath & other) const { - return ffi_StorePath_less_than(*this, other); + return baseName < other.baseName; } bool operator == (const StorePath & other) const { - return ffi_StorePath_eq(*this, other); + return baseName == other.baseName; } bool operator != (const StorePath & other) const { - return !(*this == other); + return baseName != other.baseName; } - StorePath clone() const; - /* Check whether a file name ends with the extension for derivations. */ bool isDerivation() const; - std::string_view name() const; + std::string_view name() const + { + return std::string_view(baseName).substr(HashLen + 1); + } - unsigned char * hashData() const + std::string_view hashPart() const { - return ffi_StorePath_hash_data(*this); + return std::string_view(baseName).substr(0, HashLen); } static StorePath dummy; @@ -62,34 +62,19 @@ struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; -StorePathSet cloneStorePathSet(const StorePathSet & paths); -StorePathSet storePathsToSet(const StorePaths & paths); - -StorePathSet singleton(const StorePath & path); - -/* Size of the hash part of store paths, in base-32 characters. */ -const size_t storePathHashLen = 32; // i.e. 160 bits - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; +enum struct FileIngestionMethod : uint8_t { + Flat = false, + Recursive = true +}; + struct StorePathWithOutputs { StorePath path; std::set<std::string> outputs; - StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {}) - : path(path.clone()), outputs(outputs) - { } - - StorePathWithOutputs(StorePath && path, std::set<std::string> && outputs) - : path(std::move(path)), outputs(std::move(outputs)) - { } - - StorePathWithOutputs(const StorePathWithOutputs & other) - : path(other.path.clone()), outputs(other.outputs) - { } - std::string to_string(const Store & store) const; }; @@ -102,7 +87,7 @@ namespace std { template<> struct hash<nix::StorePath> { std::size_t operator()(const nix::StorePath & path) const noexcept { - return * (std::size_t *) path.hashData(); + return * (std::size_t *) path.to_string().data(); } }; diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 2635e3940..926f4ea1e 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -20,7 +20,7 @@ AutoCloseFD openLockFile(const Path & path, bool create) fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); if (!fd && (create || errno != ENOENT)) - throw SysError(format("opening lock file '%1%'") % path); + throw SysError("opening lock file '%1%'", path); return fd; } @@ -51,7 +51,7 @@ bool lockFile(int fd, LockType lockType, bool wait) while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); + throw SysError("acquiring/releasing lock"); else return false; } @@ -60,7 +60,7 @@ bool lockFile(int fd, LockType lockType, bool wait) checkInterrupt(); if (errno == EWOULDBLOCK) return false; if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); + throw SysError("acquiring/releasing lock"); } } @@ -124,7 +124,7 @@ bool PathLocks::lockPaths(const PathSet & paths, hasn't been unlinked). */ struct stat st; if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); + throw SysError("statting lock file '%1%'", lockPath); if (st.st_size != 0) /* This lock file has been unlinked, so we're holding a lock on a deleted file. This means that other @@ -160,7 +160,8 @@ void PathLocks::unlock() if (close(i.first) == -1) printError( - format("error (ignored): cannot close lock file on '%1%'") % i.second); + "error (ignored): cannot close lock file on '%1%'", + i.second); debug(format("lock released on '%1%'") % i.second); } diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 2bef51878..6cfe393a4 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -50,7 +50,7 @@ Generations findGenerations(Path profile, int & curGen) gen.number = n; struct stat st; if (lstat(gen.path.c_str(), &st) != 0) - throw SysError(format("statting '%1%'") % gen.path); + throw SysError("statting '%1%'", gen.path); gen.creationTime = st.st_mtime; gens.push_back(gen); } @@ -117,7 +117,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) static void removeFile(const Path & path) { if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink '%1%'") % path); + throw SysError("cannot unlink '%1%'", path); } @@ -149,7 +149,7 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens Generations gens = findGenerations(profile, curGen); if (gensToDelete.find(curGen) != gensToDelete.end()) - throw Error(format("cannot delete current generation of profile %1%'") % profile); + throw Error("cannot delete current generation of profile %1%'", profile); for (auto & i : gens) { if (gensToDelete.find(i.number) == gensToDelete.end()) continue; @@ -226,7 +226,7 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b int days; if (!string2Int(strDays, days) || days < 1) - throw Error(format("invalid number of days specifier '%1%'") % timeSpec); + throw Error("invalid number of days specifier '%1%'", timeSpec); time_t oldTime = curTime - days * 24 * 3600; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 102e15921..a10d536a3 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -92,7 +92,7 @@ PathSet scanForReferences(const string & path, auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); if (pos == string::npos) - throw Error(format("bad reference '%1%'") % i); + throw Error("bad reference '%1%'", i); string s = string(baseName, 0, pos); assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 5a2d103b9..bd698d781 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -19,7 +19,7 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) { assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); + return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext); } void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, @@ -51,7 +51,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) std::string restPath = std::string(path, storePath.size()); if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + throw InvalidPath("path '%1%' is not a valid store path", storePath); auto i = nars.find(storePath); if (i != nars.end()) return {i->second, restPath}; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8c55da268..fc5ab5865 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -116,11 +116,11 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() struct sockaddr_un addr; addr.sun_family = AF_UNIX; if (socketPath.size() + 1 >= sizeof(addr.sun_path)) - throw Error(format("socket path '%1%' is too long") % socketPath); + throw Error("socket path '%1%' is too long", socketPath); strcpy(addr.sun_path, socketPath.c_str()); if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at '%1%'") % socketPath); + throw SysError("cannot connect to daemon at '%1%'", socketPath); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); @@ -268,7 +268,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i.clone()); + if (isValidPath(i)) res.insert(i); return res; } else { conn->to << wopQueryValidPaths; @@ -296,7 +296,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & i : paths) { conn->to << wopHasSubstitutes << printStorePath(i); conn.processStderr(); - if (readInt(conn->from)) res.insert(i.clone()); + if (readInt(conn->from)) res.insert(i); } return res; } else { @@ -329,7 +329,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, info.references = readStorePaths<StorePathSet>(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.clone(), std::move(info)); + infos.insert_or_assign(i, std::move(info)); } } else { @@ -365,14 +365,14 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, } catch (Error & e) { // Ugly backwards compatibility hack. if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.what()); + throw InvalidPath(e.info()); throw; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = std::make_shared<ValidPathInfo>(path.clone()); + info = std::make_shared<ValidPathInfo>(StorePath(path)); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash(readString(conn->from), htSHA256); @@ -396,7 +396,7 @@ void RemoteStore::queryReferrers(const StorePath & path, conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) - referrers.insert(i.clone()); + referrers.insert(i); } @@ -418,15 +418,6 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputNames << printStorePath(path); - conn.processStderr(); - return readStrings<PathSet>(conn->from); -} - - std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); @@ -484,7 +475,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, - bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); @@ -492,10 +483,12 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, Path srcPath(absPath(_srcPath)); - conn->to << wopAddToStore << name - << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ - << (recursive ? 1 : 0) - << printHashType(hashAlgo); + conn->to + << wopAddToStore + << name + << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (method == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(hashAlgo); try { conn->to.written = 0; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f301a97d8..80c8e9f11 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -51,8 +51,6 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const StorePath & path) override; - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -65,7 +63,7 @@ public: std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f2e4b63e0..427dd48ce 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -6,7 +6,7 @@ #include "nar-info-disk-cache.hh" #include "globals.hh" #include "compression.hh" -#include "download.hh" +#include "filetransfer.hh" #include "istringstream_nocopy.hh" #include <aws/core/Aws.h> @@ -32,8 +32,10 @@ namespace nix { struct S3Error : public Error { Aws::S3::S3Errors err; - S3Error(Aws::S3::S3Errors err, const FormatOrString & fs) - : Error(fs), err(err) { }; + + template<typename... Args> + S3Error(Aws::S3::S3Errors err, const Args & ... args) + : Error(args...), err(err) { }; }; /* Helper: given an Outcome<R, E>, return R in case of success, or @@ -109,7 +111,9 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); if (retry) printError("AWS error '%s' (%s), will retry in %d ms", - error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); + error.GetExceptionName(), + error.GetMessage(), + CalculateDelayBeforeNextRetry(error, attemptedRetries)); return retry; } }; @@ -132,7 +136,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region return res; } -S3Helper::DownloadResult S3Helper::getObject( +S3Helper::FileTransferResult S3Helper::getObject( const std::string & bucketName, const std::string & key) { debug("fetching 's3://%s/%s'...", bucketName, key); @@ -146,7 +150,7 @@ S3Helper::DownloadResult S3Helper::getObject( return Aws::New<std::stringstream>("STRINGSTREAM"); }); - DownloadResult res; + FileTransferResult res; auto now1 = std::chrono::steady_clock::now(); @@ -249,7 +253,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore // If bucket listing is disabled, 404s turn into 403s || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) return false; - throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage()); + throw Error("AWS error fetching '%s': %s", path, error.GetMessage()); } return true; diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index ef5f23d0f..2042bffcf 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -18,13 +18,13 @@ struct S3Helper ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); - struct DownloadResult + struct FileTransferResult { std::shared_ptr<std::string> data; unsigned int durationMs; }; - DownloadResult getObject( + FileTransferResult getObject( const std::string & bucketName, const std::string & key); }; diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index eb1daafc5..76c822c4e 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -29,7 +29,7 @@ SQLite::SQLite(const Path & path, bool create) { if (sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open SQLite database '%s'") % path); + throw Error("cannot open SQLite database '%s'", path); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) throwSQLiteError(db, "setting timeout"); @@ -204,7 +204,10 @@ void handleSQLiteBusy(const SQLiteBusy & e) if (now > lastWarned + 10) { lastWarned = now; - printError("warning: %s", e.what()); + logWarning({ + .name = "Sqlite busy", + .hint = hintfmt(e.what()) + }); } /* Sleep for a while since retrying the transaction right away diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index fd04c9b07..dd81ab051 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -3,7 +3,7 @@ #include <functional> #include <string> -#include "types.hh" +#include "error.hh" struct sqlite3; struct sqlite3_stmt; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b9e894a9a..aae227bae 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,6 +6,7 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "url.hh" #include <future> @@ -22,7 +23,7 @@ bool Store::isInStore(const Path & path) const Path Store::toStorePath(const Path & path) const { if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw Error("path '%1%' is not in the Nix store", path); Path::size_type slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) return path; @@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const path = absPath(target, dirOf(path)); } if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw NotInStore("path '%1%' is not in the Nix store", path); return path; } @@ -54,15 +55,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const { auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs(followLinksToStorePath(path2), std::move(outputs)); -} - - -string storePathToHash(const Path & path) -{ - auto base = baseNameOf(path); - assert(base.size() >= storePathHashLen); - return string(base, 0, storePathHashLen); + return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; } @@ -141,9 +134,9 @@ StorePath Store::makeStorePath(const string & type, const Hash & hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name); + string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); auto h = compressHash(hashString(htSHA256, s), 20); - return StorePath::make(h.hash, name); + return StorePath(h, name); } @@ -171,19 +164,22 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - bool recursive, + FileIngestionMethod recursive, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive) { + if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); - return makeStorePath("output:out", hashString(htSHA256, - "fixed:out:" + (recursive ? (string) "r:" : "") + - hash.to_string(Base16) + ":"), name); + return makeStorePath("output:out", + hashString(htSHA256, + "fixed:out:" + + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") + + hash.to_string(Base16, true) + ":"), + name); } } @@ -200,10 +196,12 @@ StorePath Store::makeTextPath(std::string_view name, const Hash & hash, std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const + const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const { - Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(recursive, h, name), h); + Hash h = method == FileIngestionMethod::Recursive + ? hashPath(hashAlgo, srcPath, filter).first + : hashFile(hashAlgo, srcPath); + return std::make_pair(makeFixedOutputPath(method, h, name), h); } @@ -237,7 +235,7 @@ bool Store::PathInfoCacheValue::isKnownNow() bool Store::isValidPath(const StorePath & storePath) { - auto hashPart = storePathToHash(printStorePath(storePath)); + std::string hashPart(storePath.hashPart()); { auto state_(state.lock()); @@ -305,7 +303,7 @@ void Store::queryPathInfo(const StorePath & storePath, std::string hashPart; try { - hashPart = storePathToHash(printStorePath(storePath)); + hashPart = storePath.hashPart(); { auto res = state.lock()->pathInfoCache.get(hashPart); @@ -455,7 +453,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string(hashBase)) + .attr("narHash", info->narHash.to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -498,7 +496,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); + jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -547,7 +545,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMod for (auto & path : paths) { if (path.path.isDerivation()) unsupported("buildPaths"); - paths2.insert(path.path.clone()); + paths2.insert(path.path); } if (queryValidPaths(paths2).size() != paths2.size()) @@ -687,21 +685,6 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, } -ValidPathInfo::ValidPathInfo(const ValidPathInfo & other) - : path(other.path.clone()) - , deriver(other.deriver ? other.deriver->clone(): std::optional<StorePath>{}) - , narHash(other.narHash) - , references(cloneStorePathSet(other.references)) - , registrationTime(other.registrationTime) - , narSize(other.narSize) - , id(other.id) - , ultimate(other.ultimate) - , sigs(other.sigs) - , ca(other.ca) -{ -} - - std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven) { std::string path; @@ -754,7 +737,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32) + ";" + + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -769,11 +752,15 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) bool ValidPathInfo::isContentAddressed(const Store & store) const { auto warn = [&]() { - printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); + logWarning( + ErrorInfo{ + .name = "Path not content-addressed", + .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path)) + }); }; if (hasPrefix(ca, "text:")) { - Hash hash(std::string(ca, 5)); + Hash hash(ca.substr(5)); if (store.makeTextPath(path.name(), hash, references) == path) return true; else @@ -781,9 +768,9 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const } else if (hasPrefix(ca, "fixed:")) { - bool recursive = ca.compare(6, 2, "r:") == 0; - Hash hash(std::string(ca, recursive ? 8 : 6)); - auto refs = cloneStorePathSet(references); + FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; + Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6)); + auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; @@ -826,27 +813,14 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(bool recursive, const Hash & hash) +std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) { - return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); + return "fixed:" + + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") + + hash.to_string(Base32, true); } -void Store::addToStore(const ValidPathInfo & info, Source & narSource, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) -{ - addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, accessor); -} - -void Store::addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair, CheckSigsFlag checkSigs, - std::shared_ptr<FSAccessor> accessor) -{ - StringSource source(*nar); - addToStore(info, source, repair, checkSigs, accessor); -} - } @@ -866,27 +840,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_ Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { - for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) { - auto e = s.find('='); - if (e != std::string::npos) { - auto value = s.substr(e + 1); - std::string decoded; - for (size_t i = 0; i < value.size(); ) { - if (value[i] == '%') { - if (i + 2 >= value.size()) - throw Error("invalid URI parameter '%s'", value); - try { - decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); - i += 3; - } catch (...) { - throw Error("invalid URI parameter '%s'", value); - } - } else - decoded += value[i++]; - } - params[s.substr(0, e)] = decoded; - } - } + params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } return {uri, params}; @@ -961,7 +915,7 @@ std::list<ref<Store>> getDefaultSubstituters() try { stores.push_back(openStore(uri)); } catch (Error & e) { - printError("warning: %s", e.what()); + logWarning(e.info()); } }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 396709633..17b582d8f 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -29,6 +29,7 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); +MakeError(NotInStore, Error); class FSAccessor; @@ -41,7 +42,6 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; - /* Magic header of exportPath() output (obsolete). */ const uint32_t exportMagic = 0x4558494e; @@ -187,8 +187,9 @@ struct ValidPathInfo Strings shortRefs() const; + ValidPathInfo(const StorePath & path) : path(path) { } + ValidPathInfo(StorePath && path) : path(std::move(path)) { } - explicit ValidPathInfo(const ValidPathInfo & other); virtual ~ValidPathInfo() { } }; @@ -344,7 +345,7 @@ public: StorePath makeOutputPath(const string & id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(bool recursive, + StorePath makeFixedOutputPath(FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references = {}, bool hasSelfReference = false) const; @@ -356,7 +357,7 @@ public: store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, - const Path & srcPath, bool recursive = true, + const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). @@ -428,10 +429,6 @@ public: virtual StorePathSet queryDerivationOutputs(const StorePath & path) { unsupported("queryDerivationOutputs"); } - /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const StorePath & path) - { unsupported("queryDerivationOutputNames"); } - /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0; @@ -448,24 +445,19 @@ public: /* Import a path into the store. */ virtual void addToStore(const ValidPathInfo & info, Source & narSource, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0); - - // FIXME: remove - virtual void addToStore(const ValidPathInfo & info, const ref<std::string> & nar, - RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, - std::shared_ptr<FSAccessor> accessor = 0); + std::shared_ptr<FSAccessor> accessor = 0) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, - bool recursive = true, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; // FIXME: remove? virtual StorePath addToStoreFromDump(const string & dump, const string & name, - bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); } @@ -590,6 +582,9 @@ public: ensurePath(). */ Derivation derivationFromPath(const StorePath & drvPath); + /* Read a derivation (which must already be valid). */ + Derivation readDerivation(const StorePath & drvPath); + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly or indirectly reached from it. `out' is not cleared. If @@ -735,10 +730,6 @@ public: }; -/* Extract the hash part of the given store path. */ -string storePathToHash(const Path & path); - - /* Copy a path from one store to another. */ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); @@ -848,7 +839,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo( /* Compute the content-addressability assertion (ValidPathInfo::ca) for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(bool recursive, const Hash & hash); +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); /* Split URI into protocol+hierarchy part and its parameter set. */ diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 857d54d99..ac42457fc 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -36,7 +36,7 @@ typedef enum { wopClearFailedPaths = 25, wopQueryPathInfo = 26, wopImportPaths = 27, // obsolete - wopQueryDerivationOutputNames = 28, + wopQueryDerivationOutputNames = 28, // obsolete wopQueryPathFromHashPart = 29, wopQuerySubstitutablePathInfos = 30, wopQueryValidPaths = 31, |