diff options
Diffstat (limited to 'src/libstore')
43 files changed, 944 insertions, 830 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 583f9d8ac..9f52ddafa 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -40,14 +40,14 @@ void BinaryCacheStore::init() upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString<Strings>(*cacheInfo, "\n")) { - size_t colon = line.find(':'); - if (colon == std::string::npos) continue; + size_t colon= line.find(':'); + if (colon ==std::string::npos) continue; auto name = line.substr(0, colon); auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { if (value != storeDir) - throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") - % getUri() % value % storeDir); + throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", + getUri(), value, storeDir); } else if (name == "WantMassQuery") { wantMassQuery.setDefault(value == "1" ? "true" : "false"); } else if (name == "Priority") { @@ -93,7 +93,7 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) { - return storePathToHash(printStorePath(storePath)) + ".narinfo"; + return std::string(storePath.hashPart()) + ".narinfo"; } void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) @@ -102,7 +102,7 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - auto hashPart = storePathToHash(printStorePath(narInfo->path)); + std::string hashPart(narInfo->path.hashPart()); { auto state_(state.lock()); @@ -137,7 +137,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource auto narInfo = make_ref<NarInfo>(info); narInfo->narSize = nar->size(); - narInfo->narHash = hashString(HashType::SHA256, *nar); + narInfo->narHash = hashString(htSHA256, *nar); if (info.narHash && info.narHash != narInfo->narHash) throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); @@ -164,7 +164,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } } - upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json"); + upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json"); } /* Compress the NAR. */ @@ -172,16 +172,16 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource auto now1 = std::chrono::steady_clock::now(); auto narCompressed = compress(compression, *nar, parallelCompression); auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(HashType::SHA256, *narCompressed); + narInfo->fileHash = hashString(htSHA256, *narCompressed); narInfo->fileSize = narCompressed->size(); auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); - printMsg(Verbosity::Talkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", printStorePath(narInfo->path), narInfo->narSize, ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), duration); - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base::Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : @@ -209,7 +209,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource // to a GC'ed file, so overwriting might be useful... if (fileExists(key)) return; - printMsg(Verbosity::Talkative, "creating debuginfo link from '%s' to '%s'", key, target); + printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target); upsertFile(key, json.dump(), "application/json"); }; @@ -287,7 +287,7 @@ void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) try { getFile(info->url, *decompressor); } catch (NoSuchBinaryCacheFile & e) { - throw SubstituteGone(e.what()); + throw SubstituteGone(e.info()); } decompressor->finish(); @@ -302,7 +302,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, { auto uri = getUri(); auto storePathS = printStorePath(storePath); - auto act = std::make_shared<Activity>(*logger, Verbosity::Talkative, ActivityType::QueryPathInfo, + auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo, fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); PushActivity pact(act->id); @@ -360,7 +360,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s const StorePathSet & references, RepairFlag repair) { ValidPathInfo info(computeStorePathForText(name, s, references)); - info.references = cloneStorePathSet(references); + info.references = references; if (repair || !isValidPath(info.path)) { StringSink sink; @@ -388,21 +388,19 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe narInfo->sigs.insert(sigs.begin(), sigs.end()); - auto narInfoFile = narInfoFileFor(narInfo->path); - writeNarInfo(narInfo); } std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) { - auto drvPath = path.clone(); + auto drvPath = path; if (!path.isDerivation()) { try { auto info = queryPathInfo(path); // FIXME: add a "Log" field to .narinfo if (!info->deriver) return nullptr; - drvPath = info->deriver->clone(); + drvPath = *info->deriver; } catch (InvalidPath &) { return nullptr; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 3eb103e94..ecd447495 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -86,7 +86,7 @@ struct HookInstance; /* A pointer to a goal. */ -class Goal; +struct Goal; class DerivationGoal; typedef std::shared_ptr<Goal> GoalPtr; typedef std::weak_ptr<Goal> WeakGoalPtr; @@ -104,18 +104,9 @@ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; -class Goal : public std::enable_shared_from_this<Goal> +struct Goal : public std::enable_shared_from_this<Goal> { -public: - enum struct ExitCode { - Busy, - Success, - Failed, - NoSubstituters, - IncompleteClosure, - }; - -protected: + typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; /* Backlink to the worker. */ Worker & worker; @@ -144,10 +135,13 @@ protected: /* Whether the goal is finished. */ ExitCode exitCode; + /* Exception containing an error message, if any. */ + std::optional<Error> ex; + Goal(Worker & worker) : worker(worker) { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ExitCode::Busy; + exitCode = ecBusy; } virtual ~Goal() @@ -155,7 +149,6 @@ protected: trace("goal destroyed"); } -public: virtual void work() = 0; void addWaitee(GoalPtr waitee); @@ -179,21 +172,14 @@ public: return name; } - ExitCode getExitCode() - { - return exitCode; - } - /* Callback in case of a timeout. It should wake up its waiters, get rid of any running child processes that are being monitored by the worker (important!), etc. */ - virtual void timedOut() = 0; + virtual void timedOut(Error && ex) = 0; virtual string key() = 0; -protected: - - virtual void amDone(ExitCode result); + void amDone(ExitCode result, std::optional<Error> ex = {}); }; @@ -309,7 +295,7 @@ public: /* Make a goal (with caching). */ GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(StorePath && drvPath, + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); @@ -361,14 +347,14 @@ public: contents. */ bool pathContentsGood(const StorePath & path); - void markContentsGood(StorePath && path); + void markContentsGood(const StorePath & path); void updateProgress() { actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(ActivityType::Download, expectedDownloadSize + doneDownloadSize); - act.setExpected(ActivityType::CopyPath, expectedNarSize + doneNarSize); + act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); + act.setExpected(actCopyPath, expectedNarSize + doneNarSize); } }; @@ -398,16 +384,15 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) assert(waitees.find(waitee) != waitees.end()); waitees.erase(waitee); - trace(format("waitee '%1%' done; %2% left") % - waitee->name % waitees.size()); + trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - if (result == ExitCode::Failed || result == ExitCode::NoSubstituters || result == ExitCode::IncompleteClosure) ++nrFailed; + if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - if (result == ExitCode::NoSubstituters) ++nrNoSubstituters; + if (result == ecNoSubstituters) ++nrNoSubstituters; - if (result == ExitCode::IncompleteClosure) ++nrIncompleteClosure; + if (result == ecIncompleteClosure) ++nrIncompleteClosure; - if (waitees.empty() || (result == ExitCode::Failed && !settings.keepGoing)) { + if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { /* If we failed and keepGoing is not set, we remove all remaining waitees. */ @@ -424,12 +409,20 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) } -void Goal::amDone(ExitCode result) +void Goal::amDone(ExitCode result, std::optional<Error> ex) { trace("done"); - assert(exitCode == ExitCode::Busy); - assert(result == ExitCode::Success || result == ExitCode::Failed || result == ExitCode::NoSubstituters || result == ExitCode::IncompleteClosure); + assert(exitCode == ecBusy); + assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); exitCode = result; + + if (ex) { + if (!waiters.empty()) + logError(ex->info()); + else + this->ex = std::move(*ex); + } + for (auto & i : waiters) { GoalPtr goal = i.lock(); if (goal) goal->waiteeDone(shared_from_this(), result); @@ -459,7 +452,7 @@ static void commonChildInit(Pipe & logPipe) that e.g. ssh cannot open /dev/tty) and it doesn't receive terminal signals. */ if (setsid() == -1) - throw SysError(format("creating a new session")); + throw SysError("creating a new session"); /* Dup the write side of the logger pipe into stderr. */ if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) @@ -472,7 +465,7 @@ static void commonChildInit(Pipe & logPipe) /* Reroute stdin to /dev/null. */ int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); if (fdDevNull == -1) - throw SysError(format("cannot open '%1%'") % pathNullDevice); + throw SysError("cannot open '%1%'", pathNullDevice); if (dup2(fdDevNull, STDIN_FILENO) == -1) throw SysError("cannot dup null device into stdin"); close(fdDevNull); @@ -494,12 +487,18 @@ void handleDiffHook( auto diffRes = runProgram(diffHookOptions); if (!statusOk(diffRes.first)) - throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first))); + throw ExecError(diffRes.first, + "diff-hook program '%1%' %2%", + diffHook, + statusToString(diffRes.first)); if (diffRes.second != "") printError(chomp(diffRes.second)); } catch (Error & error) { - printError("diff hook execution failed: %s", error.what()); + ErrorInfo ei = error.info(); + ei.hint = hintfmt("diff hook execution failed: %s", + (error.info().hint.has_value() ? error.info().hint->str() : "")); + logError(ei); } } } @@ -548,37 +547,37 @@ bool UserLock::findFreeUser() { /* Get the members of the build-users-group. */ struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - throw Error(format("the group '%1%' specified in 'build-users-group' does not exist") - % settings.buildUsersGroup); + throw Error("the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup); gid = gr->gr_gid; /* Copy the result of getgrnam. */ Strings users; for (char * * p = gr->gr_mem; *p; ++p) { - debug(format("found build user '%1%'") % *p); + debug("found build user '%1%'", *p); users.push_back(*p); } if (users.empty()) - throw Error(format("the build users group '%1%' has no members") - % settings.buildUsersGroup); + throw Error("the build users group '%1%' has no members", + settings.buildUsersGroup); /* Find a user account that isn't currently in use for another build. */ for (auto & i : users) { - debug(format("trying user '%1%'") % i); + debug("trying user '%1%'", i); struct passwd * pw = getpwnam(i.c_str()); if (!pw) - throw Error(format("the user '%1%' in the group '%2%' does not exist") - % i % settings.buildUsersGroup); + throw Error("the user '%1%' in the group '%2%' does not exist", + i, settings.buildUsersGroup); fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fd) - throw SysError(format("opening user lock '%1%'") % fnUserLock); + throw SysError("opening user lock '%1%'", fnUserLock); if (lockFile(fd.get(), ltWrite, false)) { fdUserLock = std::move(fd); @@ -587,8 +586,8 @@ bool UserLock::findFreeUser() { /* Sanity check... */ if (uid == getuid() || uid == geteuid()) - throw Error(format("the Nix user should not be a member of '%1%'") - % settings.buildUsersGroup); + throw Error("the Nix user should not be a member of '%1%'", + settings.buildUsersGroup); #if __linux__ /* Get the list of supplementary groups of this build user. This @@ -598,7 +597,7 @@ bool UserLock::findFreeUser() { int err = getgrouplist(pw->pw_name, pw->pw_gid, supplementaryGIDs.data(), &ngroups); if (err == -1) - throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name); + throw Error("failed to get list of supplementary groups for '%1%'", pw->pw_name); supplementaryGIDs.resize(ngroups); #endif @@ -607,6 +606,7 @@ bool UserLock::findFreeUser() { return true; } } + return false; } @@ -678,7 +678,7 @@ HookInstance::HookInstance() Strings args = { std::string(baseNameOf(settings.buildHook.get())), - std::to_string((uint64_t)verbosity), + std::to_string(verbosity), }; execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); @@ -868,6 +868,9 @@ private: std::unique_ptr<Activity> act; + /* Activity that denotes waiting for a lock. */ + std::unique_ptr<Activity> actLock; + std::map<ActivityId, Activity> builderActivities; /* The remote machine on which we're building. */ @@ -897,16 +900,16 @@ private: friend struct RestrictedStore; public: - DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, + DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, + DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); /* Whether we need to perform hash rewriting if there are valid output paths. */ bool needsHashRewrite(); - void timedOut() override; + void timedOut(Error && ex) override; string key() override { @@ -921,7 +924,7 @@ public: StorePath getDrvPath() { - return drvPath.clone(); + return drvPath; } /* Add wanted outputs to an already existing derivation goal. */ @@ -1005,14 +1008,11 @@ private: void repairClosure(); - void amDone(ExitCode result) override - { - Goal::amDone(result); - } - void started(); - void done(BuildResult::Status status, const string & msg = ""); + void done( + BuildResult::Status status, + std::optional<Error> ex = {}); StorePathSet exportReferences(const StorePathSet & storePaths); }; @@ -1021,11 +1021,11 @@ private: const Path DerivationGoal::homeDir = "/homeless-shelter"; -DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, +DerivationGoal::DerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(true) - , drvPath(std::move(drvPath)) + , drvPath(drvPath) , wantedOutputs(wantedOutputs) , buildMode(buildMode) { @@ -1038,11 +1038,11 @@ DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOut } -DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, +DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(false) - , drvPath(std::move(drvPath)) + , drvPath(drvPath) , buildMode(buildMode) { this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv)); @@ -1106,10 +1106,10 @@ void DerivationGoal::killChild() } -void DerivationGoal::timedOut() +void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut); + done(BuildResult::TimedOut, ex); } @@ -1157,8 +1157,7 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - printError("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)); - done(BuildResult::MiscFailure); + done(BuildResult::MiscFailure, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); return; } @@ -1194,7 +1193,13 @@ void DerivationGoal::haveDerivation() return; } - parsedDrv = std::make_unique<ParsedDerivation>(drvPath.clone(), *drv); + parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + + if (parsedDrv->contentAddressed()) { + settings.requireExperimentalFeature("ca-derivations"); + throw Error("ca-derivations isn't implemented yet"); + } + /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build @@ -1302,14 +1307,18 @@ void DerivationGoal::repairClosure() if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv.insert_or_assign(j.second.path.clone(), i.clone()); + outputsToDrv.insert_or_assign(j.second.path, i); } /* Check each path (slow!). */ for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - printError("found corrupted or missing path '%s' in the output closure of '%s'", - worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); + logError({ + .name = "Corrupt path in closure", + .hint = hintfmt( + "found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)) + }); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) addWaitee(worker.makeSubstitutionGoal(i, Repair)); @@ -1343,9 +1352,9 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - printError("cannot build derivation '%s': %s dependencies couldn't be built", - worker.store.printStorePath(drvPath), nrFailed); - done(BuildResult::DependencyFailed); + done(BuildResult::DependencyFailed, Error( + "%s dependencies of derivation '%s' failed to build", + nrFailed, worker.store.printStorePath(drvPath))); return; } @@ -1407,7 +1416,7 @@ void DerivationGoal::started() { "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); fmt("building '%s'", worker.store.printStorePath(drvPath)); if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique<Activity>(*logger, Verbosity::Info, ActivityType::Build, msg, + act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); worker.updateProgress(); @@ -1427,10 +1436,15 @@ void DerivationGoal::tryToBuild() lockFiles.insert(worker.store.Store::toRealPath(outPath)); if (!outputLocks.lockPaths(lockFiles, "", false)) { + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); return; } + actLock.reset(); + /* Now check again whether the outputs are valid. This is because another process may have started building in parallel. After it has finished and released the locks, we can (and should) @@ -1446,7 +1460,7 @@ void DerivationGoal::tryToBuild() return; } - missingPaths = cloneStorePathSet(drv->outputPaths()); + missingPaths = drv->outputPaths(); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); @@ -1463,26 +1477,13 @@ void DerivationGoal::tryToBuild() supported for local builds. */ bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); - auto started = [&]() { - auto msg = fmt( - buildMode == bmRepair ? "repairing outputs of '%s'" : - buildMode == bmCheck ? "checking outputs of '%s'" : - nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); - fmt("building '%s'", worker.store.printStorePath(drvPath)); - if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique<Activity>(*logger, Verbosity::Info, ActivityType::Build, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); - mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); - worker.updateProgress(); - }; - /* Is the build hook willing to accept this job? */ if (!buildLocally) { switch (tryBuildHook()) { case rpAccept: /* Yes, it has started doing so. Wait until we get EOF from the hook. */ + actLock.reset(); result.startTime = time(0); // inexact state = &DerivationGoal::buildDone; started(); @@ -1490,6 +1491,9 @@ void DerivationGoal::tryToBuild() case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); return; @@ -1499,6 +1503,8 @@ void DerivationGoal::tryToBuild() } } + actLock.reset(); + /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ @@ -1526,7 +1532,9 @@ void DerivationGoal::tryLocalBuild() { uid. */ buildUser->kill(); } else { - debug("waiting for build users"); + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, + fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } @@ -1537,17 +1545,18 @@ void DerivationGoal::tryLocalBuild() { #endif } + actLock.reset(); + try { /* Okay, we have to build. */ startBuilder(); } catch (BuildError & e) { - printError(e.msg()); outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, e.msg()); + done(BuildResult::InputRejected, e); return; } @@ -1659,10 +1668,10 @@ void DerivationGoal::buildDone() } auto msg = fmt("builder for '%s' %s", - worker.store.printStorePath(drvPath), + yellowtxt(worker.store.printStorePath(drvPath)), statusToString(status)); - if (!settings.verboseBuild && !logTail.empty()) { + if (!logger->isVerbose() && !logTail.empty()) { msg += (format("; last %d log lines:") % logTail.size()).str(); for (auto & line : logTail) msg += "\n " + line; @@ -1679,7 +1688,7 @@ void DerivationGoal::buildDone() registerOutputs(); if (settings.postBuildHook != "") { - Activity act(*logger, Verbosity::Info, ActivityType::PostBuildHook, + Activity act(*logger, lvlInfo, actPostBuildHook, fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); @@ -1711,11 +1720,7 @@ void DerivationGoal::buildDone() } void flushLine() { - if (settings.verboseBuild) { - printError("post-build-hook: " + currentLine); - } else { - act.result(ResultType::PostBuildLogLine, currentLine); - } + act.result(resPostBuildLogLine, currentLine); currentLine.clear(); } @@ -1764,8 +1769,6 @@ void DerivationGoal::buildDone() outputLocks.unlock(); } catch (BuildError & e) { - printError(e.msg()); - outputLocks.unlock(); BuildResult::Status st = BuildResult::MiscFailure; @@ -1784,7 +1787,7 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, e.msg()); + done(st, e); return; } @@ -1827,7 +1830,7 @@ HookReply DerivationGoal::tryBuildHook() } } - debug(format("hook reply is '%1%'") % reply); + debug("hook reply is '%1%'", reply); if (reply == "decline") return rpDecline; @@ -1843,8 +1846,12 @@ HookReply DerivationGoal::tryBuildHook() } catch (SysError & e) { if (e.errNo == EPIPE) { - printError("build hook died unexpectedly: %s", - chomp(drainFD(worker.hook->fromHook.readSide.get()))); + logError({ + .name = "Build hook died", + .hint = hintfmt( + "build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))) + }); worker.hook = 0; return rpDecline; } else @@ -1900,14 +1907,14 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) if (!inputPaths.count(storePath)) throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - worker.store.computeFSClosure(singleton(storePath), paths); + worker.store.computeFSClosure({storePath}, paths); } /* If there are derivations in the graph, then include their outputs as well. This is useful if you want to do things like passing all build-time dependencies of some path to a derivation that builds a NixOS DVD image. */ - auto paths2 = cloneStorePathSet(paths); + auto paths2 = paths; for (auto & j : paths2) { if (j.isDerivation()) { @@ -2024,7 +2031,7 @@ void DerivationGoal::startBuilder() string s = get(drv->env, "exportReferencesGraph").value_or(""); Strings ss = tokenizeString<Strings>(s); if (ss.size() % 2 != 0) - throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); + throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s); for (Strings::iterator i = ss.begin(); i != ss.end(); ) { string fileName = *i++; static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*"); @@ -2036,7 +2043,7 @@ void DerivationGoal::startBuilder() /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences(singleton(storePath)), false, false)); + exportReferences({storePath}), false, false)); } } @@ -2073,7 +2080,7 @@ void DerivationGoal::startBuilder() worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); } catch (InvalidPath & e) { } catch (Error & e) { - throw Error(format("while processing 'sandbox-paths': %s") % e.what()); + throw Error("while processing 'sandbox-paths': %s", e.what()); } for (auto & i : closure) { auto p = worker.store.printStorePath(i); @@ -2117,13 +2124,13 @@ void DerivationGoal::startBuilder() /* Clean up the chroot directory automatically. */ autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir); - printMsg(Verbosity::Chatty, format("setting up chroot environment in '%1%'") % chrootRootDir); + printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); if (mkdir(chrootRootDir.c_str(), 0750) == -1) - throw SysError(format("cannot create '%1%'") % chrootRootDir); + throw SysError("cannot create '%1%'", chrootRootDir); if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % chrootRootDir); + throw SysError("cannot change ownership of '%1%'", chrootRootDir); /* Create a writable /tmp in the chroot. Many builders need this. (Of course they should really respect $TMPDIR @@ -2167,7 +2174,7 @@ void DerivationGoal::startBuilder() chmod_(chrootStoreDir, 01775); if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir); + throw SysError("cannot change ownership of '%1%'", chrootStoreDir); for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); @@ -2200,7 +2207,7 @@ void DerivationGoal::startBuilder() if (needsHashRewrite()) { if (pathExists(homeDir)) - throw Error(format("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing") % homeDir); + throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir); /* We're not doing a chroot build, but we have some valid output paths. Since we can't just overwrite or delete @@ -2221,12 +2228,12 @@ void DerivationGoal::startBuilder() for (auto & i : missingPaths) if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) { addHashRewrite(i); - redirectedBadOutputs.insert(i.clone()); + redirectedBadOutputs.insert(i); } } if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) { - printMsg(Verbosity::Chatty, format("executing pre-build hook '%1%'") + printMsg(lvlChatty, format("executing pre-build hook '%1%'") % settings.preBuildHook); auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : Strings({ worker.store.printStorePath(drvPath) }); @@ -2245,8 +2252,7 @@ void DerivationGoal::startBuilder() if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { state = stExtraChrootDirs; } else { - throw Error(format("unknown pre-build hook command '%1%'") - % line); + throw Error("unknown pre-build hook command '%1%'", line); } } else if (state == stExtraChrootDirs) { if (line == "") { @@ -2268,7 +2274,7 @@ void DerivationGoal::startBuilder() startDaemon(); /* Run the builder. */ - printMsg(Verbosity::Chatty, format("executing builder '%1%'") % drv->builder); + printMsg(lvlChatty, "executing builder '%1%'", drv->builder); /* Create the log file. */ Path logFile = openLogFile(); @@ -2504,8 +2510,8 @@ void DerivationGoal::initTmpDir() { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - auto hash = hashString(HashType::SHA256, i.first); - string fn = ".attr-" + hash.to_string(Base::Base32, false); + auto hash = hashString(htSHA256, i.first); + string fn = ".attr-" + hash.to_string(Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -2717,8 +2723,8 @@ struct RestrictedStore : public LocalFSStore StorePathSet queryAllValidPaths() override { StorePathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p.clone()); - for (auto & p : goal.addedPaths) paths.insert(p.clone()); + for (auto & p : goal.inputPaths) paths.insert(p); + for (auto & p : goal.addedPaths) paths.insert(p); return paths; } @@ -2747,14 +2753,11 @@ struct RestrictedStore : public LocalFSStore StorePathSet queryDerivationOutputs(const StorePath & path) override { throw Error("queryDerivationOutputs"); } - StringSet queryDerivationOutputNames(const StorePath & path) override - { throw Error("queryDerivationOutputNames"); } - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override { throw Error("addToStore"); } @@ -2767,7 +2770,7 @@ struct RestrictedStore : public LocalFSStore } StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); goal.addDependency(path); @@ -2809,7 +2812,7 @@ struct RestrictedStore : public LocalFSStore auto drv = derivationFromPath(path.path); for (auto & output : drv.outputs) if (wantOutput(output.first, path.outputs)) - newPaths.insert(output.second.path.clone()); + newPaths.insert(output.second.path); } else if (!goal.isAllowed(path.path)) throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } @@ -2854,7 +2857,7 @@ struct RestrictedStore : public LocalFSStore if (goal.isAllowed(path.path)) allowed.emplace_back(path); else - unknown.insert(path.path.clone()); + unknown.insert(path.path); } next->queryMissing(allowed, willBuild, willSubstitute, @@ -2949,7 +2952,7 @@ void DerivationGoal::addDependency(const StorePath & path) { if (isAllowed(path)) return; - addedPaths.insert(path.clone()); + addedPaths.insert(path); /* If we're doing a sandbox build, then we have to make the path appear in the sandbox. */ @@ -3009,7 +3012,7 @@ void DerivationGoal::chownToBuilder(const Path & path) { if (!buildUser) return; if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) - throw SysError(format("cannot change ownership of '%1%'") % path); + throw SysError("cannot change ownership of '%1%'", path); } @@ -3146,7 +3149,7 @@ void DerivationGoal::runChild() /* Bind-mount chroot directory to itself, to treat it as a different filesystem from /, as needed for pivot_root. */ if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) - throw SysError(format("unable to bind mount '%1%'") % chrootRootDir); + throw SysError("unable to bind mount '%1%'", chrootRootDir); /* Bind-mount the sandbox's Nix store onto itself so that we can mark it as a "shared" subtree, allowing bind @@ -3208,7 +3211,7 @@ void DerivationGoal::runChild() filesystem that we want in the chroot environment. */ auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - debug(format("bind mounting '%1%' to '%2%'") % source % target); + debug("bind mounting '%1%' to '%2%'", source, target); struct stat st; if (stat(source.c_str(), &st) == -1) { if (optional && errno == ENOENT) @@ -3280,16 +3283,16 @@ void DerivationGoal::runChild() /* Do the chroot(). */ if (chdir(chrootRootDir.c_str()) == -1) - throw SysError(format("cannot change directory to '%1%'") % chrootRootDir); + throw SysError("cannot change directory to '%1%'", chrootRootDir); if (mkdir("real-root", 0) == -1) throw SysError("cannot create real-root directory"); if (pivot_root(".", "real-root") == -1) - throw SysError(format("cannot pivot old root directory onto '%1%'") % (chrootRootDir + "/real-root")); + throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root")); if (chroot(".") == -1) - throw SysError(format("cannot change root directory to '%1%'") % chrootRootDir); + throw SysError("cannot change root directory to '%1%'", chrootRootDir); if (umount2("real-root", MNT_DETACH) == -1) throw SysError("cannot unmount real root filesystem"); @@ -3310,7 +3313,7 @@ void DerivationGoal::runChild() #endif if (chdir(tmpDirInSandbox.c_str()) == -1) - throw SysError(format("changing into '%1%'") % tmpDir); + throw SysError("changing into '%1%'", tmpDir); /* Close all other file descriptors. */ closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); @@ -3449,9 +3452,9 @@ void DerivationGoal::runChild() sandboxProfile += "(allow file-read* file-write* process-exec\n"; for (auto & i : dirsInChroot) { if (i.first != i.second.source) - throw Error(format( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") - % i.first % i.second.source); + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); string path = i.first; struct stat st; @@ -3542,7 +3545,7 @@ void DerivationGoal::runChild() else if (drv->builder == "builtin:unpack-channel") builtinUnpackChannel(drv2); else - throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); + throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); @@ -3552,7 +3555,7 @@ void DerivationGoal::runChild() execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); - throw SysError(format("executing '%1%'") % drv->builder); + throw SysError("executing '%1%'", drv->builder); } catch (std::exception & e) { writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); @@ -3571,7 +3574,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv if (store.isStorePath(i)) result.insert(store.parseStorePath(i)); else if (drv.outputs.count(i)) - result.insert(drv.outputs.find(i)->second.path.clone()); + result.insert(drv.outputs.find(i)->second.path); else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; @@ -3585,7 +3588,7 @@ static void moveCheckToStore(const Path & src, const Path & dst) directory's parent link ".."). */ struct stat st; if (lstat(src.c_str(), &st) == -1) { - throw SysError(format("getting attributes of path '%1%'") % src); + throw SysError("getting attributes of path '%1%'", src); } bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); @@ -3594,7 +3597,7 @@ static void moveCheckToStore(const Path & src, const Path & dst) chmod_(src, st.st_mode | S_IWUSR); if (rename(src.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % src % dst); + throw SysError("renaming '%1%' to '%2%'", src, dst); if (changePerm) chmod_(dst, st.st_mode); @@ -3629,9 +3632,9 @@ void DerivationGoal::registerOutputs() output paths, and any paths that have been built via recursive Nix calls. */ StorePathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p.clone()); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path.clone()); - for (auto & p : addedPaths) referenceablePaths.insert(p.clone()); + for (auto & p : inputPaths) referenceablePaths.insert(p); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); + for (auto & p : addedPaths) referenceablePaths.insert(p); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all @@ -3660,7 +3663,7 @@ void DerivationGoal::registerOutputs() replaceValidPath(path, actualPath); else if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1) - throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); + throw SysError("moving build output '%1%' from the sandbox to the Nix store", path); } if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); } @@ -3681,13 +3684,16 @@ void DerivationGoal::registerOutputs() user. */ if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || (buildUser && st.st_uid != buildUser->getUID())) - throw BuildError(format("suspicious ownership or permission on '%1%'; rejecting this build output") % path); + throw BuildError("suspicious ownership or permission on '%1%'; rejecting this build output", path); #endif /* Apply hash rewriting if necessary. */ bool rewritten = false; if (!outputRewrites.empty()) { - printError(format("warning: rewriting hashes in '%1%'; cross fingers") % path); + logWarning({ + .name = "Rewriting hashes", + .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path) + }); /* Canonicalise first. This ensures that the path we're rewriting doesn't contain a hard link to /etc/shadow or @@ -3716,8 +3722,9 @@ void DerivationGoal::registerOutputs() /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( - format("output path '%1%' should be a non-executable regular file " - "since recursive hashing is not enabled (outputHashMode=flat)") % path); + "output path '%1%' should be a non-executable regular file " + "since recursive hashing is not enabled (outputHashMode=flat)", + path); } /* Check the hash. In hash mode, move the path produced by @@ -3736,8 +3743,8 @@ void DerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", worker.store.printStorePath(dest), - i.second.hash->hash.to_string(Base::SRI), - h2.to_string(Base::SRI))); + i.second.hash->hash.to_string(SRI, true), + h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); @@ -3825,7 +3832,7 @@ void DerivationGoal::registerOutputs() info.narHash = hash.first; info.narSize = hash.second; info.references = std::move(references); - info.deriver = drvPath.clone(); + info.deriver = drvPath; info.ultimate = true; info.ca = ca; worker.store.signPathInfo(info); @@ -3852,10 +3859,10 @@ void DerivationGoal::registerOutputs() result.isNonDeterministic = true; Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); - auto msg = prevExists - ? fmt("output '%s' of '%s' differs from '%s' from previous round", + hintformat hint = prevExists + ? hintfmt("output '%s' of '%s' differs from '%s' from previous round", worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) - : fmt("output '%s' of '%s' differs from previous round", + : hintfmt("output '%s' of '%s' differs from previous round", worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); handleDiffHook( @@ -3865,9 +3872,13 @@ void DerivationGoal::registerOutputs() worker.store.printStorePath(drvPath), tmpDir); if (settings.enforceDeterminism) - throw NotDeterministic(msg); + throw NotDeterministic(hint); + + logError({ + .name = "Output determinism error", + .hint = hint + }); - printError(msg); curRound = nrRounds; // we know enough, bail out early } } @@ -3939,23 +3950,23 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) uint64_t closureSize = 0; StorePathSet pathsDone; std::queue<StorePath> pathsLeft; - pathsLeft.push(path.clone()); + pathsLeft.push(path); while (!pathsLeft.empty()) { - auto path = pathsLeft.front().clone(); + auto path = pathsLeft.front(); pathsLeft.pop(); - if (!pathsDone.insert(path.clone()).second) continue; + if (!pathsDone.insert(path).second) continue; auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; for (auto & ref : i->second.references) - pathsLeft.push(ref.clone()); + pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; for (auto & ref : info->references) - pathsLeft.push(ref.clone()); + pathsLeft.push(ref); } } @@ -3982,8 +3993,8 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); auto used = recursive - ? cloneStorePathSet(getClosure(info.path).first) - : cloneStorePathSet(info.references); + ? getClosure(info.path).first + : info.references; if (recursive && checks.ignoreSelfRefs) used.erase(info.path); @@ -3993,10 +4004,10 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) for (auto & i : used) if (allowed) { if (!spec.count(i)) - badPaths.insert(i.clone()); + badPaths.insert(i); } else { if (spec.count(i)) - badPaths.insert(i.clone()); + badPaths.insert(i); } if (!badPaths.empty()) { @@ -4085,7 +4096,7 @@ Path DerivationGoal::openLogFile() settings.compressLog ? ".bz2" : ""); fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); - if (!fdLogFile) throw SysError(format("creating log file '%1%'") % logFileName); + if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); logFileSink = std::make_shared<FdSink>(fdLogFile.get()); @@ -4131,11 +4142,11 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - printError( - format("%1% killed after writing more than %2% bytes of log output") - % getName() % settings.maxLogSize); killChild(); - done(BuildResult::LogLimitExceeded); + done( + BuildResult::LogLimitExceeded, + Error("%s killed after writing more than %d bytes of log output", + getName(), settings.maxLogSize)); return; } @@ -4177,15 +4188,10 @@ void DerivationGoal::flushLine() ; else { - if (settings.verboseBuild && - (settings.printRepeatedBuilds || curRound == 1)) - printError(currentLogLine); - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - } + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); - act->result(ResultType::BuildLogLine, currentLogLine); + act->result(resBuildLogLine, currentLogLine); } currentLogLine = ""; @@ -4201,7 +4207,7 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) bool good = worker.store.isValidPath(i.second.path) && (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path.clone()); + if (good == returnValid) result.insert(i.second.path); } return result; } @@ -4212,20 +4218,21 @@ void DerivationGoal::addHashRewrite(const StorePath & path) auto h1 = std::string(((std::string_view) path.to_string()).substr(0, 32)); auto p = worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(HashType::SHA256), path.name()); + Hash(htSHA256), path.name()); auto h2 = std::string(((std::string_view) p.to_string()).substr(0, 32)); deletePath(worker.store.printStorePath(p)); inputRewrites[h1] = h2; outputRewrites[h2] = h1; - redirectedOutputs.insert_or_assign(path.clone(), std::move(p)); + redirectedOutputs.insert_or_assign(path, std::move(p)); } -void DerivationGoal::done(BuildResult::Status status, const string & msg) +void DerivationGoal::done(BuildResult::Status status, std::optional<Error> ex) { result.status = status; - result.errorMsg = msg; - amDone(result.success() ? ExitCode::Success : ExitCode::Failed); + if (ex) + result.errorMsg = ex->what(); + amDone(result.success() ? ecSuccess : ecFailed, ex); if (result.status == BuildResult::TimedOut) worker.timedOut = true; if (result.status == BuildResult::PermanentFailure) @@ -4291,10 +4298,10 @@ private: GoalState state; public: - SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair); ~SubstitutionGoal(); - void timedOut() override { abort(); }; + void timedOut(Error && ex) override { abort(); }; string key() override { @@ -4317,18 +4324,13 @@ public: void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; - StorePath getStorePath() { return storePath.clone(); } - - void amDone(ExitCode result) override - { - Goal::amDone(result); - } + StorePath getStorePath() { return storePath; } }; -SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair) : Goal(worker) - , storePath(std::move(storePath)) + , storePath(storePath) , repair(repair) { state = &SubstitutionGoal::init; @@ -4366,7 +4368,7 @@ void SubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - amDone(ExitCode::Success); + amDone(ecSuccess); return; } @@ -4391,7 +4393,7 @@ void SubstitutionGoal::tryNext() /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - amDone(substituterFailed ? ExitCode::Failed : ExitCode::NoSubstituters); + amDone(substituterFailed ? ecFailed : ecNoSubstituters); if (substituterFailed) { worker.failedSubstitutions++; @@ -4423,7 +4425,7 @@ void SubstitutionGoal::tryNext() throw; } catch (Error & e) { if (settings.tryFallback) { - printError(e.what()); + logError(e.info()); tryNext(); return; } @@ -4449,8 +4451,11 @@ void SubstitutionGoal::tryNext() && !sub->isTrusted && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { - printError("warning: substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), worker.store.printStorePath(storePath)); + logWarning({ + .name = "Invalid path signature", + .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), worker.store.printStorePath(storePath)) + }); tryNext(); return; } @@ -4474,7 +4479,7 @@ void SubstitutionGoal::referencesValid() if (nrFailed > 0) { debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ExitCode::IncompleteClosure : ExitCode::Failed); + amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); return; } @@ -4512,7 +4517,7 @@ void SubstitutionGoal::tryToRun() /* Wake up the worker loop when we're done. */ Finally updateStats([this]() { outPipe.writeSide = -1; }); - Activity act(*logger, ActivityType::Substitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); + Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), @@ -4559,9 +4564,9 @@ void SubstitutionGoal::finished() return; } - worker.markContentsGood(storePath.clone()); + worker.markContentsGood(storePath); - printMsg(Verbosity::Chatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); + printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); maintainRunningSubstitutions.reset(); @@ -4579,7 +4584,7 @@ void SubstitutionGoal::finished() worker.updateProgress(); - amDone(ExitCode::Success); + amDone(ecSuccess); } @@ -4593,14 +4598,13 @@ void SubstitutionGoal::handleEOF(int fd) if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); } - ////////////////////////////////////////////////////////////////////// Worker::Worker(LocalStore & store) - : act(*logger, ActivityType::Realise) - , actDerivations(*logger, ActivityType::Builds) - , actSubstitutions(*logger, ActivityType::CopyPaths) + : act(*logger, actRealise) + , actDerivations(*logger, actBuilds) + , actSubstitutions(*logger, actCopyPaths) , store(store) { /* Debugging: prevent recursive workers. */ @@ -4630,10 +4634,10 @@ Worker::~Worker() GoalPtr Worker::makeDerivationGoal(const StorePath & path, const StringSet & wantedOutputs, BuildMode buildMode) { - GoalPtr goal = derivationGoals[path.clone()].lock(); // FIXME + GoalPtr goal = derivationGoals[path].lock(); // FIXME if (!goal) { - goal = std::make_shared<DerivationGoal>(path.clone(), wantedOutputs, *this, buildMode); - derivationGoals.insert_or_assign(path.clone(), goal); + goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); + derivationGoals.insert_or_assign(path, goal); wakeUp(goal); } else (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); @@ -4641,10 +4645,10 @@ GoalPtr Worker::makeDerivationGoal(const StorePath & path, } -std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(StorePath && drvPath, +std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto goal = std::make_shared<DerivationGoal>(std::move(drvPath), drv, *this, buildMode); + auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); wakeUp(goal); return goal; } @@ -4652,10 +4656,10 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(StorePath && drv GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) { - GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME + GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { - goal = std::make_shared<SubstitutionGoal>(path.clone(), *this, repair); - substitutionGoals.insert_or_assign(path.clone(), goal); + goal = std::make_shared<SubstitutionGoal>(path, *this, repair); + substitutionGoals.insert_or_assign(path, goal); wakeUp(goal); } return goal; @@ -4684,7 +4688,7 @@ void Worker::removeGoal(GoalPtr goal) topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals (unless keepGoing was set). */ - if (goal->getExitCode() == Goal::ExitCode::Failed && !settings.keepGoing) + if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) topGoals.clear(); } @@ -4810,9 +4814,9 @@ void Worker::run(const Goals & _topGoals) if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); else { - if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( - "unable to start any build; either increase '--max-jobs' " - "or enable remote builds"); + if (awake.empty() && 0 == settings.maxBuildJobs) + throw Error("unable to start any build; either increase '--max-jobs' " + "or enable remote builds"); assert(!awake.empty()); } } @@ -4825,10 +4829,9 @@ void Worker::run(const Goals & _topGoals) assert(!settings.keepGoing || children.empty()); } - void Worker::waitForInput() { - printMsg(Verbosity::Vomit, "waiting for children"); + printMsg(lvlVomit, "waiting for children"); /* Process output from the file descriptors attached to the children, namely log output and output path creation commands. @@ -4863,8 +4866,6 @@ void Worker::waitForInput() up after a few seconds at most. */ if (!waitingForAWhile.empty()) { useTimeout = true; - if (lastWokenUp == steady_time_point::min()) - printError("waiting for locks, build slots or build users..."); if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>( @@ -4913,15 +4914,15 @@ void Worker::waitForInput() // FIXME: is there a cleaner way to handle pt close // than EIO? Is this even standard? if (rd == 0 || (rd == -1 && errno == EIO)) { - debug(format("%1%: got EOF") % goal->getName()); + debug("%1%: got EOF", goal->getName()); goal->handleEOF(k); j->fds.erase(k); } else if (rd == -1) { if (errno != EINTR) throw SysError("%s: read failed", goal->getName()); } else { - printMsg(Verbosity::Vomit, format("%1%: read %2% bytes") - % goal->getName() % rd); + printMsg(lvlVomit, "%1%: read %2% bytes", + goal->getName(), rd); string data((char *) buffer.data(), rd); j->lastOutput = after; goal->handleChildOutput(k, data); @@ -4929,26 +4930,24 @@ void Worker::waitForInput() } } - if (goal->getExitCode() == Goal::ExitCode::Busy && + if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { - printError( - format("%1% timed out after %2% seconds of silence") - % goal->getName() % settings.maxSilentTime); - goal->timedOut(); + goal->timedOut(Error( + "%1% timed out after %2% seconds of silence", + goal->getName(), settings.maxSilentTime)); } - else if (goal->getExitCode() == Goal::ExitCode::Busy && + else if (goal->exitCode == Goal::ecBusy && 0 != settings.buildTimeout && j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) { - printError( - format("%1% timed out after %2% seconds") - % goal->getName() % settings.buildTimeout); - goal->timedOut(); + goal->timedOut(Error( + "%1% timed out after %2% seconds", + goal->getName(), settings.buildTimeout)); } } @@ -5002,18 +5001,22 @@ bool Worker::pathContentsGood(const StorePath & path) res = false; else { HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); - Hash nullHash(HashType::SHA256); + Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } - pathContentsGoodCache.insert_or_assign(path.clone(), res); - if (!res) printError("path '%s' is corrupted or missing!", store.printStorePath(path)); + pathContentsGoodCache.insert_or_assign(path, res); + if (!res) + logError({ + .name = "Corrupted path", + .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path)) + }); return res; } -void Worker::markContentsGood(StorePath && path) +void Worker::markContentsGood(const StorePath & path) { - pathContentsGoodCache.insert_or_assign(std::move(path), true); + pathContentsGoodCache.insert_or_assign(path, true); } @@ -5050,24 +5053,35 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, worker.run(goals); StorePathSet failed; + std::optional<Error> ex; for (auto & i : goals) { - if (i->getExitCode() != Goal::ExitCode::Success) { + if (i->ex) { + if (ex) + logError(i->ex->info()); + else + ex = i->ex; + } + if (i->exitCode != Goal::ecSuccess) { DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); if (i2) failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath()); } } - if (!failed.empty()) + if (failed.size() == 1 && ex) { + ex->status = worker.exitStatus(); + throw *ex; + } else if (!failed.empty()) { + if (ex) logError(ex->info()); throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + } } - BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath.clone(), drv, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); BuildResult result; @@ -5088,7 +5102,7 @@ void LocalStore::ensurePath(const StorePath & path) /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {StorePathWithOutputs(path)}); + primeCache(*this, {{path}}); Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); @@ -5096,8 +5110,13 @@ void LocalStore::ensurePath(const StorePath & path) worker.run(goals); - if (goal->getExitCode() != Goal::ExitCode::Success) - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + if (goal->exitCode != Goal::ecSuccess) { + if (goal->ex) { + goal->ex->status = worker.exitStatus(); + throw *goal->ex; + } else + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + } } @@ -5109,7 +5128,7 @@ void LocalStore::repairPath(const StorePath & path) worker.run(goals); - if (goal->getExitCode() != Goal::ExitCode::Success) { + if (goal->exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 1b802d908..802fb87bc 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -22,7 +22,10 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, srcFiles = readDirectory(srcDir); } catch (SysError & e) { if (e.errNo == ENOTDIR) { - printError("warning: not including '%s' in the user environment because it's not a directory", srcDir); + logWarning({ + .name = "Create links - directory", + .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir) + }); return; } throw; @@ -41,7 +44,10 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw SysError("getting status of '%1%'", srcFile); } catch (SysError & e) { if (e.errNo == ENOENT || e.errNo == ENOTDIR) { - printError("warning: skipping dangling symlink '%s'", dstFile); + logWarning({ + .name = "Create links - skipping symlink", + .hint = hintfmt("skipping dangling symlink '%s'", dstFile) + }); continue; } throw; @@ -72,15 +78,15 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, if (!S_ISDIR(lstat(target).st_mode)) throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); + throw SysError("unlinking '%1%'", dstFile); if (mkdir(dstFile.c_str(), 0755) == -1) - throw SysError(format("creating directory '%1%'")); + throw SysError("creating directory '%1%'", dstFile); createLinks(state, target, dstFile, state.priorities[dstFile]); createLinks(state, srcFile, dstFile, priority); continue; } } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); + throw SysError("getting status of '%1%'", dstFile); } else { @@ -99,11 +105,11 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, if (prevPriority < priority) continue; if (unlink(dstFile.c_str()) == -1) - throw SysError(format("unlinking '%1%'") % dstFile); + throw SysError("unlinking '%1%'", dstFile); } else if (S_ISDIR(dstSt.st_mode)) throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); } else if (errno != ENOENT) - throw SysError(format("getting status of '%1%'") % dstFile); + throw SysError("getting status of '%1%'", dstFile); } createSymlink(srcFile, dstFile); diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 770df2927..1cfe4a46a 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -18,7 +18,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto getAttr = [&](const string & name) { auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); return i->second; }; @@ -54,7 +54,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) auto executable = drv.env.find("executable"); if (executable != drv.env.end() && executable->second == "1") { if (chmod(storePath.c_str(), 0755) == -1) - throw SysError(format("making '%1%' executable") % storePath); + throw SysError("making '%1%' executable", storePath); } }; @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashTypeOpt(getAttr("outputHashAlgo")); auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base::Base16, false)); + fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index f30f53ff8..5bf51cb97 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -21,7 +21,7 @@ std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) { return "fixed:" + makeFileIngestionPrefix(method) - + hash.to_string(); + + hash.to_string(Base32, true); } // FIXME Put this somewhere? @@ -31,7 +31,7 @@ template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { - return "text:" + th.hash.to_string(); + return "text:" + th.hash.to_string(Base::Base32, true); }, [](FileSystemHash fsh) { return makeFixedOutputCA(fsh.method, fsh.hash); @@ -46,7 +46,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) { if (prefix == "text") { auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); Hash hash = Hash(string(hashTypeAndHash)); - if (*hash.type != HashType::SHA256) { + if (*hash.type != htSHA256) { throw Error("parseContentAddress: the text hash should have type SHA256"); } return TextHash { hash }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 63b8d4bfe..842aef20c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -73,6 +73,18 @@ struct TunnelLogger : public Logger enqueueMsg(*buf.s); } + void logEI(const ErrorInfo & ei) override + { + if (ei.level > verbosity) return; + + std::stringstream oss; + oss << ei; + + StringSink buf; + buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n"); + enqueueMsg(*buf.s); + } + /* startWork() means that we're starting an operation for which we want to send out stderr to the client. */ void startWork() @@ -114,13 +126,7 @@ struct TunnelLogger : public Logger } StringSink buf; - buf << STDERR_START_ACTIVITY - << act - << (uint64_t) lvl - << (uint64_t) type - << s - << fields - << parent; + buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; enqueueMsg(*buf.s); } @@ -136,10 +142,7 @@ struct TunnelLogger : public Logger { if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; StringSink buf; - buf << STDERR_RESULT - << act - << (uint64_t) type - << fields; + buf << STDERR_RESULT << act << type << fields; enqueueMsg(*buf.s); } }; @@ -290,7 +293,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto path = store->parseStorePath(readString(from)); logger->startWork(); StorePathSet paths; // FIXME - paths.insert(path.clone()); + paths.insert(path); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); to << (res.count(path) != 0); @@ -311,7 +314,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base::Base16, false); + to << hash.to_string(Base16, false); break; } @@ -324,7 +327,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, StorePathSet paths; if (op == wopQueryReferences) for (auto & i : store->queryPathInfo(path)->references) - paths.insert(i.clone()); + paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); else if (op == wopQueryValidDerivers) @@ -338,8 +341,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopQueryDerivationOutputNames: { auto path = store->parseStorePath(readString(from)); logger->startWork(); - StringSet names; - names = store->queryDerivationOutputNames(path); + auto names = store->readDerivation(path).outputNames(); logger->stopWork(); to << names; break; @@ -367,8 +369,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store, std::string s, baseName; FileIngestionMethod method; { - bool fixed, recursive; + bool fixed; uint8_t recursive; from >> baseName >> fixed /* obsolete */ >> recursive >> s; + if (recursive > (uint8_t) FileIngestionMethod::Recursive) + throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); method = FileIngestionMethod { recursive }; /* Compatibility hack. */ if (!fixed) { @@ -559,7 +563,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, clientSettings.maxBuildJobs = readInt(from); clientSettings.maxSilentTime = readInt(from); readInt(from); // obsolete useBuildHook - clientSettings.verboseBuild = Verbosity::Error == (Verbosity) readInt(from); + clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from); readInt(from); // obsolete logType readInt(from); // obsolete printBuildTrace clientSettings.buildCores = readInt(from); @@ -589,9 +593,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - StorePathSet paths; - paths.insert(path.clone()); // FIXME - store->querySubstitutablePathInfos(paths, infos); + store->querySubstitutablePathInfos({path}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) @@ -644,7 +646,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base::Base16, false); + << info->narHash.to_string(Base16, false); writeStorePaths(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { @@ -704,7 +706,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto deriver = readString(from); if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash(readString(from), HashType::SHA256); + info.narHash = Hash(readString(from), htSHA256); info.references = readStorePaths<StorePathSet>(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); @@ -753,7 +755,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } default: - throw Error(format("invalid operation %1%") % op); + throw Error("invalid operation %1%", op); } } @@ -788,7 +790,7 @@ void processConnection( Finally finally([&]() { _isInterrupted = false; - prevLogger->log(Verbosity::Debug, fmt("%d operations", opCount)); + prevLogger->log(lvlDebug, fmt("%d operations", opCount)); }); if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a522eb950..1ba957c4a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -8,28 +8,6 @@ namespace nix { -BasicDerivation::BasicDerivation(const BasicDerivation & other) - : platform(other.platform) - , builder(other.builder) - , args(other.args) - , env(other.env) -{ - for (auto & i : other.outputs) - outputs.insert_or_assign(i.first, - DerivationOutput(i.second.path.clone(), std::optional<FileSystemHash>(i.second.hash))); - for (auto & i : other.inputSrcs) - inputSrcs.insert(i.clone()); -} - - -Derivation::Derivation(const Derivation & other) - : BasicDerivation(other) -{ - for (auto & i : other.inputDrvs) - inputDrvs.insert_or_assign(i.first.clone(), i.second); -} - - const StorePath & BasicDerivation::findOutput(const string & id) const { auto i = outputs.find(id); @@ -48,9 +26,9 @@ bool BasicDerivation::isBuiltin() const StorePath writeDerivation(ref<Store> store, const Derivation & drv, std::string_view name, RepairFlag repair) { - auto references = cloneStorePathSet(drv.inputSrcs); + auto references = drv.inputSrcs; for (auto & i : drv.inputDrvs) - references.insert(i.first.clone()); + references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ @@ -68,7 +46,7 @@ static void expect(std::istream & str, const string & s) char s2[s.size()]; str.read(s2, s.size()); if (string(s2, s.size()) != s) - throw FormatError(format("expected string '%1%'") % s); + throw FormatError("expected string '%1%'", s); } @@ -95,7 +73,7 @@ static Path parsePath(std::istream & str) { string s = parseString(str); if (s.size() == 0 || s[0] != '/') - throw FormatError(format("bad path '%1%' in derivation") % s); + throw FormatError("bad path '%1%' in derivation", s); return s; } @@ -130,9 +108,9 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - auto method = FileIngestionMethod::Flat; std::optional<FileSystemHash> fsh; if (hashAlgo != "") { + auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { method = FileIngestionMethod::Recursive; hashAlgo = string(hashAlgo, 2); @@ -144,7 +122,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream }; } - return DerivationOutput(std::move(path), std::move(fsh)); + return DerivationOutput { + .path = std::move(path), + .hash = std::move(fsh), + }; } @@ -157,7 +138,8 @@ static Derivation parseDerivation(const Store & store, const string & s) /* Parse the list of outputs. */ while (!endOfList(str)) { expect(str, "("); std::string id = parseString(str); - drv.outputs.emplace(id, parseDerivationOutput(store, str)); + auto output = parseDerivationOutput(store, str); + drv.outputs.emplace(std::move(id), std::move(output)); } /* Parse the list of input derivations. */ @@ -198,7 +180,7 @@ Derivation readDerivation(const Store & store, const Path & drvPath) try { return parseDerivation(store, readFile(drvPath)); } catch (FormatError & e) { - throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); + throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg()); } } @@ -206,6 +188,12 @@ Derivation readDerivation(const Store & store, const Path & drvPath) Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); + return readDerivation(drvPath); +} + + +Derivation Store::readDerivation(const StorePath & drvPath) +{ auto accessor = getFSAccessor(); try { return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); @@ -367,9 +355,9 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin(); - return hashString(HashType::SHA256, "fixed:out:" + return hashString(htSHA256, "fixed:out:" + i->second.hash->printMethodAlgo() + ":" - + i->second.hash->hash.to_string(Base::Base16, false) + ":" + + i->second.hash->hash.to_string(Base16, false) + ":" + store.printStorePath(i->second.path)); } @@ -380,13 +368,13 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput auto h = drvHashes.find(i.first); if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); - h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, - readDerivation(store, store.toRealPath(i.first)), false)).first; + h = drvHashes.insert_or_assign(i.first, hashDerivationModulo(store, + store.readDerivation(i.first), false)).first; } - inputs2.insert_or_assign(h->second.to_string(Base::Base16, false), i.second); + inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } - return hashString(HashType::SHA256, drv.unparse(store, maskOutputs, &inputs2)); + return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); } @@ -408,7 +396,7 @@ StorePathSet BasicDerivation::outputPaths() const { StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path.clone()); + paths.insert(i.second.path); return paths; } @@ -418,23 +406,35 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) auto hashAlgo = readString(in); const auto hash = readString(in); - auto method = FileIngestionMethod::Flat; std::optional<FileSystemHash> fsh; if (hashAlgo != "") { + auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { method = FileIngestionMethod::Recursive; hashAlgo = string(hashAlgo, 2); } - HashType hashType = parseHashType(hashAlgo); + const HashType hashType = parseHashType(hashAlgo); fsh = FileSystemHash { std::move(method), Hash(hash, hashType), }; } - return DerivationOutput(std::move(path), std::move(fsh)); + return DerivationOutput { + .path = std::move(path), + .hash = std::move(fsh), + }; } +StringSet BasicDerivation::outputNames() const +{ + StringSet names; + for (auto & i : outputs) + names.insert(i.first); + return names; +} + + Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) { drv.outputs.clear(); @@ -442,7 +442,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) for (size_t n = 0; n < nr; n++) { auto name = readString(in); auto output = readDerivationOutput(in, store); - drv.outputs.emplace(name, output); + drv.outputs.emplace(std::move(name), std::move(output)); } drv.inputSrcs = readStorePaths<StorePathSet>(store, in); @@ -479,7 +479,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr std::string hashPlaceholder(const std::string & outputName) { // FIXME: memoize? - return "/" + hashString(HashType::SHA256, "nix-output:" + outputName).to_string(Base::Base32, false); + return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 838b63358..b2f3ff03b 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -17,13 +17,6 @@ struct DerivationOutput { StorePath path; std::optional<FileSystemHash> hash; /* hash used for expected hash computation */ - DerivationOutput(StorePath && path, std::optional<FileSystemHash> && hash) - : path(std::move(path)) - , hash(std::move(hash)) - { } - DerivationOutput(const DerivationOutput &) = default; - DerivationOutput(DerivationOutput &&) = default; - DerivationOutput & operator = (const DerivationOutput &) = default; }; typedef std::map<string, DerivationOutput> DerivationOutputs; @@ -44,7 +37,6 @@ struct BasicDerivation StringPairs env; BasicDerivation() { } - explicit BasicDerivation(const BasicDerivation & other); virtual ~BasicDerivation() { }; /* Return the path corresponding to the output identifier `id' in @@ -59,6 +51,8 @@ struct BasicDerivation /* Return the output paths of a derivation. */ StorePathSet outputPaths() const; + /* Return the output names of a derivation. */ + StringSet outputNames() const; }; struct Derivation : BasicDerivation @@ -70,8 +64,6 @@ struct Derivation : BasicDerivation std::map<std::string, StringSet> * actualInputs = nullptr) const; Derivation() { } - Derivation(Derivation && other) = default; - explicit Derivation(const Derivation & other); }; diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index aef5cc6da..57b7e9590 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -11,7 +11,7 @@ struct HashAndWriteSink : Sink { Sink & writeSink; HashSink hashSink; - HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(HashType::SHA256) + HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) { } virtual void operator () (const unsigned char * data, size_t len) @@ -34,7 +34,7 @@ void Store::exportPaths(const StorePathSet & paths, Sink & sink) //logger->incExpected(doneLabel, sorted.size()); for (auto & path : sorted) { - //Activity act(*logger, Verbosity::Info, format("exporting path '%s'") % path); + //Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); sink << 1; exportPath(path, sink); //logger->incProgress(doneLabel); @@ -57,7 +57,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashAndWriteSink.currentHash(); if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(), hash.to_string()); + printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic @@ -86,7 +86,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces ValidPathInfo info(parseStorePath(readString(source))); - //Activity act(*logger, Verbosity::Info, format("importing path '%s'") % info.path); + //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); info.references = readStorePaths<StorePathSet>(*this, source); @@ -94,7 +94,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(HashType::SHA256, *tee.source.data); + info.narHash = hashString(htSHA256, *tee.source.data); info.narSize = tee.source.data->size(); // Ignore optional legacy signature. @@ -105,7 +105,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces auto source = StringSource { *tee.source.data }; addToStore(info, source, NoRepair, checkSigs, accessor); - res.push_back(info.path.clone()); + res.push_back(info.path); } return res; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 933c8038d..531b85af8 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -72,23 +72,34 @@ struct curlFileTransfer : public FileTransfer curl_off_t writtenToSink = 0; + inline static const std::set<long> successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */}; + /* Get the HTTP status code, or 0 for other protocols. */ + long getHTTPStatus() + { + long httpStatus = 0; + long protocol = 0; + curl_easy_getinfo(req, CURLINFO_PROTOCOL, &protocol); + if (protocol == CURLPROTO_HTTP || protocol == CURLPROTO_HTTPS) + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + return httpStatus; + } + TransferItem(curlFileTransfer & fileTransfer, const FileTransferRequest & request, Callback<FileTransferResult> && callback) : fileTransfer(fileTransfer) , request(request) - , act(*logger, Verbosity::Talkative, ActivityType::Download, + , act(*logger, lvlTalkative, actFileTransfer, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) , finalSink([this](const unsigned char * data, size_t len) { if (this->request.dataCallback) { - long httpStatus = 0; - curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + auto httpStatus = getHTTPStatus(); /* Only write data to the sink if this is a successful response. */ - if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { + if (successfulStatuses.count(httpStatus)) { writtenToSink += len; this->request.dataCallback((char *) data, len); } @@ -112,7 +123,7 @@ struct curlFileTransfer : public FileTransfer if (requestHeaders) curl_slist_free_all(requestHeaders); try { if (!done) - fail(FileTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri)); + fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri)); } catch (...) { ignoreException(); } @@ -163,7 +174,7 @@ struct curlFileTransfer : public FileTransfer { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(Verbosity::Vomit, format("got header for '%s': %s") % request.uri % trim(line)); + printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); if (line.compare(0, 5, "HTTP/") == 0) { // new response starts result.etag = ""; auto ss = tokenizeString<vector<string>>(line, " "); @@ -246,7 +257,7 @@ struct curlFileTransfer : public FileTransfer curl_easy_reset(req); - if (verbosity >= Verbosity::Vomit) { + if (verbosity >= lvlVomit) { curl_easy_setopt(req, CURLOPT_VERBOSE, 1); curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); } @@ -316,8 +327,7 @@ struct curlFileTransfer : public FileTransfer void finish(CURLcode code) { - long httpStatus = 0; - curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + auto httpStatus = getHTTPStatus(); char * effectiveUriCStr; curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); @@ -343,8 +353,7 @@ struct curlFileTransfer : public FileTransfer if (writeException) failEx(writeException); - else if (code == CURLE_OK && - (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + else if (code == CURLE_OK && successfulStatuses.count(httpStatus)) { result.cached = httpStatus == 304; act.progress(result.bodySize, result.bodySize); @@ -517,7 +526,7 @@ struct curlFileTransfer : public FileTransfer int running; CURLMcode mc = curl_multi_perform(curlm, &running); if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc)); + throw nix::Error("unexpected error from curl_multi_perform(): %s", curl_multi_strerror(mc)); /* Set the promises of any finished requests. */ CURLMsg * msg; @@ -547,7 +556,7 @@ struct curlFileTransfer : public FileTransfer vomit("download thread waiting for %d ms", sleepTimeMs); mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); if (mc != CURLM_OK) - throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); + throw nix::Error("unexpected error from curl_multi_wait(): %s", curl_multi_strerror(mc)); nextWakeup = std::chrono::steady_clock::time_point(); @@ -599,7 +608,11 @@ struct curlFileTransfer : public FileTransfer workerThreadMain(); } catch (nix::Interrupted & e) { } catch (std::exception & e) { - printError("unexpected error in download thread: %s", e.what()); + logError({ + .name = "File transfer", + .hint = hintfmt("unexpected error in download thread: %s", + e.what()) + }); } { diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 2347f363d..11dca2fe0 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -103,8 +103,9 @@ class FileTransferError : public Error { public: FileTransfer::Error error; - FileTransferError(FileTransfer::Error error, const FormatOrString & fs) - : Error(fs), error(error) + template<typename... Args> + FileTransferError(FileTransfer::Error error, const Args & ... args) + : Error(args...), error(error) { } }; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f2646f6b3..57fb20845 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -38,10 +38,10 @@ AutoCloseFD LocalStore::openGCLock(LockType lockType) AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); if (!fdGCLock) - throw SysError(format("opening global GC lock '%1%'") % fnGCLock); + throw SysError("opening global GC lock '%1%'", fnGCLock); if (!lockFile(fdGCLock.get(), lockType, false)) { - printError(format("waiting for the big garbage collector lock...")); + printInfo("waiting for the big garbage collector lock..."); lockFile(fdGCLock.get(), lockType, true); } @@ -65,8 +65,8 @@ static void makeSymlink(const Path & link, const Path & target) /* Atomically replace the old one. */ if (rename(tempLink.c_str(), link.c_str()) == -1) - throw SysError(format("cannot rename '%1%' to '%2%'") - % tempLink % link); + throw SysError("cannot rename '%1%' to '%2%'", + tempLink , link); } @@ -78,7 +78,7 @@ void LocalStore::syncWithGC() void LocalStore::addIndirectRoot(const Path & path) { - string hash = hashString(HashType::SHA1, path).to_string(Base::Base32, false); + string hash = hashString(htSHA1, path).to_string(Base32, false); Path realRoot = canonPath((format("%1%/%2%/auto/%3%") % stateDir % gcRootsDir % hash).str()); makeSymlink(realRoot, path); @@ -91,15 +91,15 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, Path gcRoot(canonPath(_gcRoot)); if (isInStore(gcRoot)) - throw Error(format( + throw Error( "creating a garbage collector root (%1%) in the Nix store is forbidden " - "(are you running nix-build inside the store?)") % gcRoot); + "(are you running nix-build inside the store?)", gcRoot); if (indirect) { /* Don't clobber the link if it already exists and doesn't point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) - throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot); + throw Error("cannot create symlink '%1%'; already exists", gcRoot); makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); } @@ -109,10 +109,10 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") - throw Error(format( + throw Error( "path '%1%' is not a valid garbage collector root; " - "it's not in the directory '%2%'") - % gcRoot % rootsDir); + "it's not in the directory '%2%'", + gcRoot, rootsDir); } if (baseNameOf(gcRoot) == std::string(storePath.to_string())) @@ -128,11 +128,13 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, gcroots directory. */ if (settings.checkRootReachability) { auto roots = findRoots(false); - if (roots[storePath.clone()].count(gcRoot) == 0) - printError( - "warning: '%1%' is not in a directory where the garbage collector looks for roots; " - "therefore, '%2%' might be removed by the garbage collector", - gcRoot, printStorePath(storePath)); + if (roots[storePath].count(gcRoot) == 0) + logWarning({ + .name = "GC root", + .hint = hintfmt("warning: '%1%' is not in a directory where the garbage collector looks for roots; " + "therefore, '%2%' might be removed by the garbage collector", + gcRoot, printStorePath(storePath)) + }); } /* Grab the global GC root, causing us to block while a GC is in @@ -170,7 +172,7 @@ void LocalStore::addTempRoot(const StorePath & path) way. */ struct stat st; if (fstat(state->fdTempRoots.get(), &st) == -1) - throw SysError(format("statting '%1%'") % fnTempRoots); + throw SysError("statting '%1%'", fnTempRoots); if (st.st_size == 0) break; /* The garbage collector deleted this file before we could @@ -216,7 +218,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) if (!*fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; - throw SysError(format("opening temporary roots file '%1%'") % path); + throw SysError("opening temporary roots file '%1%'", path); } /* This should work, but doesn't, for some reason. */ @@ -227,7 +229,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) only succeed if the owning process has died. In that case we don't care about its temporary roots. */ if (lockFile(fd->get(), ltWrite, false)) { - printError(format("removing stale temporary roots file '%1%'") % path); + printInfo("removing stale temporary roots file '%1%'", path); unlink(path.c_str()); writeFull(fd->get(), "d"); continue; @@ -403,7 +405,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (!fdDir) { if (errno == ENOENT || errno == EACCES) continue; - throw SysError(format("opening %1%") % fdStr); + throw SysError("opening %1%", fdStr); } struct dirent * fd_ent; while (errno = 0, fd_ent = readdir(fdDir.get())) { @@ -413,7 +415,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (errno) { if (errno == ESRCH) continue; - throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); + throw SysError("iterating /proc/%1%/fd", ent->d_name); } fdDir.reset(); @@ -476,9 +478,9 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) if (!isValidPath(path)) continue; debug("got additional root '%1%'", pathS); if (censor) - roots[path.clone()].insert(censored); + roots[path].insert(censored); else - roots[path.clone()].insert(links.begin(), links.end()); + roots[path].insert(links.begin(), links.end()); } } @@ -541,7 +543,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) struct stat st; if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT) return; - throw SysError(format("getting status of %1%") % realPath); + throw SysError("getting status of %1%", realPath); } printInfo(format("deleting '%1%'") % path); @@ -559,10 +561,10 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) // size. try { if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("making '%1%' writable") % realPath); + throw SysError("making '%1%' writable", realPath); Path tmp = trashDir + "/" + std::string(baseNameOf(path)); if (rename(realPath.c_str(), tmp.c_str())) - throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp); + throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp); state.bytesInvalidated += size; } catch (SysError & e) { if (e.errNo == ENOSPC) { @@ -590,11 +592,11 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.roots.count(path)) { debug("cannot delete '%1%' because it's a root", printStorePath(path)); - state.alive.insert(path.clone()); + state.alive.insert(path); return true; } - visited.insert(path.clone()); + visited.insert(path); if (!isValidPath(path)) return false; @@ -608,7 +610,7 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.gcKeepDerivations && path.isDerivation()) { for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i.clone()); + incoming.insert(i); } /* If keep-outputs is set, then don't delete this path if there @@ -616,13 +618,13 @@ bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const Sto if (state.gcKeepOutputs) { auto derivers = queryValidDerivers(path); for (auto & i : derivers) - incoming.insert(i.clone()); + incoming.insert(i); } for (auto & i : incoming) if (i != path) if (canReachRoot(state, visited, i)) { - state.alive.insert(path.clone()); + state.alive.insert(path); return true; } @@ -637,7 +639,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) auto realPath = realStoreDir + "/" + std::string(baseNameOf(path)); if (realPath == linksDir || realPath == trashDir) return; - //Activity act(*logger, Verbosity::Debug, format("considering whether to delete '%1%'") % path); + //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); auto storePath = maybeParseStorePath(path); @@ -666,7 +668,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) ‘nix-store --delete’ doesn't have the unexpected effect of recursing into derivations and outputs. */ for (auto & i : visited) - state.dead.insert(i.clone()); + state.dead.insert(i); if (state.shouldDelete) deletePathRecursive(state, path); } @@ -681,7 +683,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) void LocalStore::removeUnusedLinks(const GCState & state) { AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + if (!dir) throw SysError("opening directory '%1%'", linksDir); long long actualSize = 0, unsharedSize = 0; @@ -694,7 +696,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) struct stat st; if (lstat(path.c_str(), &st) == -1) - throw SysError(format("statting '%1%'") % path); + throw SysError("statting '%1%'", path); if (st.st_nlink != 1) { actualSize += st.st_size; @@ -702,17 +704,17 @@ void LocalStore::removeUnusedLinks(const GCState & state) continue; } - printMsg(Verbosity::Talkative, format("deleting unused link '%1%'") % path); + printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); if (unlink(path.c_str()) == -1) - throw SysError(format("deleting '%1%'") % path); + throw SysError("deleting '%1%'", path); state.results.bytesFreed += st.st_size; } struct stat st; if (stat(linksDir.c_str(), &st) == -1) - throw SysError(format("statting '%1%'") % linksDir); + throw SysError("statting '%1%'", linksDir); long long overhead = st.st_blocks * 512ULL; printInfo(format("note: currently hard linking saves %.2f MiB") @@ -747,12 +749,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ - printError("finding garbage collector roots..."); + printInfo("finding garbage collector roots..."); Roots rootMap; if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true); - for (auto & i : rootMap) state.roots.insert(i.first.clone()); + for (auto & i : rootMap) state.roots.insert(i.first); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths @@ -761,8 +763,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) Roots tempRoots; findTempRoots(fds, tempRoots, true); for (auto & root : tempRoots) { - state.tempRoots.insert(root.first.clone()); - state.roots.insert(root.first.clone()); + state.tempRoots.insert(root.first); + state.roots.insert(root.first); } /* After this point the set of roots or temporary roots cannot @@ -799,14 +801,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } else if (options.maxFreed > 0) { if (state.shouldDelete) - printError("deleting garbage..."); + printInfo("deleting garbage..."); else - printError("determining live/dead paths..."); + printInfo("determining live/dead paths..."); try { AutoCloseDir dir(opendir(realStoreDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir); + if (!dir) throw SysError("opening directory '%1%'", realStoreDir); /* Read the store and immediately delete all paths that aren't valid. When using --max-freed etc., deleting @@ -868,7 +870,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Clean up the links directory. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { - printError("deleting unused links..."); + printInfo("deleting unused links..."); removeUnusedLinks(state); } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index da95fd3ae..2fbcafff8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -271,7 +271,7 @@ public: "listed in 'trusted-public-keys'."}; Setting<StringSet> extraPlatforms{this, - std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{}, + std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, "extra-platforms", "Additional platforms that can be built on the local system. " "These may be supported natively (e.g. armv7 on some aarch64 CPUs " diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index b43e34484..5657aa593 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -139,7 +139,7 @@ struct LegacySSHStore : public Store << cmdAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base::Base16, false); + << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime @@ -256,7 +256,7 @@ struct LegacySSHStore : public Store conn->to.flush(); for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) - out.insert(i.clone()); + out.insert(i); } StorePathSet queryValidPaths(const StorePathSet & paths, diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 363be1443..48aca478c 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -74,7 +74,7 @@ static void atomicWrite(const Path & path, const std::string & s) AutoDelete del(tmp, false); writeFile(tmp, s); if (rename(tmp.c_str(), path.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % tmp % path); + throw SysError("renaming '%1%' to '%2%'", tmp, path); del.cancel(); } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index aa5abd835..dd96d2578 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -22,7 +22,7 @@ struct LocalStoreAccessor : public FSAccessor { Path storePath = store->toStorePath(path); if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + throw InvalidPath("path '%1%' is not a valid store path", storePath); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -33,11 +33,11 @@ struct LocalStoreAccessor : public FSAccessor struct stat st; if (lstat(realPath.c_str(), &st)) { if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); } if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error(format("file '%1%' has unsupported type") % path); + throw Error("file '%1%' has unsupported type", path); return { S_ISREG(st.st_mode) ? Type::tRegular : @@ -90,13 +90,13 @@ const string LocalFSStore::drvsLogDir = "drvs"; std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_) { - auto path = path_.clone(); + auto path = path_; if (!path.isDerivation()) { try { auto info = queryPathInfo(path); if (!info->deriver) return nullptr; - path = info->deriver->clone(); + path = *info->deriver; } catch (InvalidPath &) { return nullptr; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 37c9a2bc3..17d678b9f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -87,18 +87,22 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) - printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist") - % settings.buildUsersGroup); + logError({ + .name = "'build-users-group' not found", + .hint = hintfmt( + "warning: the group '%1%' specified in 'build-users-group' does not exist", + settings.buildUsersGroup) + }); else { struct stat st; if (stat(realStoreDir.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % realStoreDir); + throw SysError("getting attributes of path '%1%'", realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) - throw SysError(format("changing ownership of path '%1%'") % realStoreDir); + throw SysError("changing ownership of path '%1%'", realStoreDir); if (chmod(realStoreDir.c_str(), perm) == -1) - throw SysError(format("changing permissions on path '%1%'") % realStoreDir); + throw SysError("changing permissions on path '%1%'", realStoreDir); } } } @@ -109,12 +113,12 @@ LocalStore::LocalStore(const Params & params) struct stat st; while (path != "/") { if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of '%1%'") % path); + throw SysError("getting status of '%1%'", path); if (S_ISLNK(st.st_mode)) - throw Error(format( + throw Error( "the path '%1%' is a symlink; " - "this is not allowed for the Nix store and its parent directories") - % path); + "this is not allowed for the Nix store and its parent directories", + path); path = dirOf(path); } } @@ -147,7 +151,7 @@ LocalStore::LocalStore(const Params & params) globalLock = openLockFile(globalLockPath.c_str(), true); if (!lockFile(globalLock.get(), ltRead, false)) { - printError("waiting for the big Nix store lock..."); + printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -155,8 +159,8 @@ LocalStore::LocalStore(const Params & params) upgrade. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) - throw Error(format("current Nix store schema is version %1%, but I only support %2%") - % curSchema % nixSchemaVersion); + throw Error("current Nix store schema is version %1%, but I only support %2%", + curSchema, nixSchemaVersion); else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; @@ -178,7 +182,7 @@ LocalStore::LocalStore(const Params & params) "please upgrade Nix to version 1.11 first."); if (!lockFile(globalLock.get(), ltWrite, false)) { - printError("waiting for exclusive access to the Nix store..."); + printInfo("waiting for exclusive access to the Nix store..."); lockFile(globalLock.get(), ltWrite, true); } @@ -256,7 +260,7 @@ LocalStore::~LocalStore() } if (future.valid()) { - printError("waiting for auto-GC to finish on exit..."); + printInfo("waiting for auto-GC to finish on exit..."); future.get(); } @@ -284,7 +288,7 @@ int LocalStore::getSchema() if (pathExists(schemaPath)) { string s = readFile(schemaPath); if (!string2Int(s, curSchema)) - throw Error(format("'%1%' is corrupt") % schemaPath); + throw Error("'%1%' is corrupt", schemaPath); } return curSchema; } @@ -293,7 +297,7 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { if (access(dbDir.c_str(), R_OK | W_OK)) - throw SysError(format("Nix database directory '%1%' is not writable") % dbDir); + throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ string dbPath = dbDir + "/db.sqlite"; @@ -367,7 +371,7 @@ void LocalStore::makeStoreWritable() throw SysError("setting up a private mount namespace"); if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError(format("remounting %1% writable") % realStoreDir); + throw SysError("remounting %1% writable", realStoreDir); } #endif } @@ -388,7 +392,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct | 0444 | (st.st_mode & S_IXUSR ? 0111 : 0); if (chmod(path.c_str(), mode) == -1) - throw SysError(format("changing mode of '%1%' to %2$o") % path % mode); + throw SysError("changing mode of '%1%' to %2$o", path, mode); } } @@ -406,7 +410,7 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct #else if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) #endif - throw SysError(format("changing modification time of '%1%'") % path); + throw SysError("changing modification time of '%1%'", path); } } @@ -415,7 +419,7 @@ void canonicaliseTimestampAndPermissions(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); canonicaliseTimestampAndPermissions(path, st); } @@ -430,17 +434,17 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe setattrlist() to remove other attributes as well. */ if (lchflags(path.c_str(), 0)) { if (errno != ENOTSUP) - throw SysError(format("clearing flags of path '%1%'") % path); + throw SysError("clearing flags of path '%1%'", path); } #endif struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); /* Really make sure that the path is of a supported type. */ if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error(format("file '%1%' has an unsupported type") % path); + throw Error("file '%1%' has an unsupported type", path); #if __linux__ /* Remove extended attributes / ACLs. */ @@ -474,7 +478,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { assert(!S_ISDIR(st.st_mode)); if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) - throw BuildError(format("invalid ownership on file '%1%'") % path); + throw BuildError("invalid ownership on file '%1%'", path); mode_t mode = st.st_mode & ~S_IFMT; assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); return; @@ -498,8 +502,8 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe if (!S_ISLNK(st.st_mode) && chown(path.c_str(), geteuid(), getegid()) == -1) #endif - throw SysError(format("changing owner of '%1%' to %2%") - % path % geteuid()); + throw SysError("changing owner of '%1%' to %2%", + path, geteuid()); } if (S_ISDIR(st.st_mode)) { @@ -518,11 +522,11 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & ino be a symlink, since we can't change its ownership. */ struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); if (st.st_uid != geteuid()) { assert(S_ISLNK(st.st_mode)); - throw Error(format("wrong ownership of top-level store path '%1%'") % path); + throw Error("wrong ownership of top-level store path '%1%'", path); } } @@ -582,7 +586,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base::Base16)) + (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -597,7 +601,7 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string())); + auto drv = readDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -617,7 +621,7 @@ uint64_t LocalStore::addValidPath(State & state, { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), + state_->pathInfoCache.upsert(std::string(info.path.hashPart()), PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) }); } @@ -629,7 +633,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { - auto info = std::make_shared<ValidPathInfo>(path.clone()); + auto info = std::make_shared<ValidPathInfo>(path); callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { auto state(_state.lock()); @@ -682,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base::Base16)) + (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -719,7 +723,7 @@ StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteF { StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i.clone()); + if (isValidPath(i)) res.insert(i); return res; } @@ -787,26 +791,9 @@ StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) } -StringSet LocalStore::queryDerivationOutputNames(const StorePath & path) -{ - return retrySQLite<StringSet>([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); - - StringSet outputNames; - while (useQueryDerivationOutputs.next()) - outputNames.insert(useQueryDerivationOutputs.getStr(0)); - - return outputNames; - }); -} - - std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart) { - if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); + if (hashPart.size() != StorePath::HashLen) throw Error("invalid hash part"); Path prefix = storeDir + "/" + hashPart; @@ -831,7 +818,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) StorePathSet remaining; for (auto & i : paths) - remaining.insert(i.clone()); + remaining.insert(i); StorePathSet res; @@ -845,9 +832,9 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) StorePathSet remaining2; for (auto & path : remaining) if (valid.count(path)) - res.insert(path.clone()); + res.insert(path); else - remaining2.insert(path.clone()); + remaining2.insert(path); std::swap(remaining, remaining2); } @@ -869,16 +856,16 @@ void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, auto info = sub->queryPathInfo(path); auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); - infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ - info->deriver ? info->deriver->clone() : std::optional<StorePath>(), - cloneStorePathSet(info->references), + infos.insert_or_assign(path, SubstitutablePathInfo{ + info->deriver, + info->references, narInfo ? narInfo->fileSize : 0, info->narSize}); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { if (settings.tryFallback) - printError(e.what()); + logError(e.info()); else throw; } @@ -910,12 +897,12 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == HashType::SHA256); + assert(i.narHash.type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else addValidPath(*state, i, false); - paths.insert(i.path.clone()); + paths.insert(i.path); } for (auto & i : infos) { @@ -930,8 +917,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string()))); + checkDerivationOutputs(i.path, readDerivation(i.path)); } /* Do a topological sort of the paths. This will throw an @@ -958,7 +944,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(storePathToHash(printStorePath(path))); + state_->pathInfoCache.erase(std::string(path.hashPart())); } } @@ -1008,9 +994,9 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, of the NAR. */ std::unique_ptr<AbstractHashSink> hashSink; if (!info.ca.has_value() || !info.references.count(info.path)) - hashSink = std::make_unique<HashSink>(HashType::SHA256); + hashSink = std::make_unique<HashSink>(htSHA256); else - hashSink = std::make_unique<HashModuloSink>(HashType::SHA256, storePathToHash(printStorePath(info.path))); + hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart())); LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { size_t n = source.read(data, len); @@ -1024,7 +1010,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string()); + printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1083,14 +1069,14 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam sha256); otherwise, compute it here. */ HashResult hash; if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == HashType::SHA256 ? h : hashString(HashType::SHA256, dump); + hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); hash.second = dump.size(); } else - hash = hashPath(HashType::SHA256, realPath); + hash = hashPath(htSHA256, realPath); optimisePath(realPath); // FIXME: combine with hashPath() - ValidPathInfo info(dstPath.clone()); + ValidPathInfo info(dstPath); info.narHash = hash.first; info.narSize = hash.second; info.ca = FileSystemHash { method, h }; @@ -1125,7 +1111,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - auto hash = hashString(HashType::SHA256, s); + auto hash = hashString(htSHA256, s); auto dstPath = makeTextPath(name, hash, references); addTempRoot(dstPath); @@ -1149,14 +1135,14 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, StringSink sink; dumpString(s, sink); - auto narHash = hashString(HashType::SHA256, *sink.s); + auto narHash = hashString(htSHA256, *sink.s); optimisePath(realPath); - ValidPathInfo info(dstPath.clone()); + ValidPathInfo info(dstPath); info.narHash = narHash; info.narSize = sink.s->size(); - info.references = cloneStorePathSet(references); + info.references = references; info.ca = TextHash { .hash = hash }; registerValidPath(info); } @@ -1207,7 +1193,7 @@ void LocalStore::invalidatePathChecked(const StorePath & path) bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { - printError(format("reading the Nix store...")); + printInfo(format("reading the Nix store...")); bool errors = false; @@ -1235,16 +1221,19 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking link hashes..."); for (auto & link : readDirectory(linksDir)) { - printMsg(Verbosity::Talkative, "checking contents of '%s'", link.name); + printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - string hash = hashPath(HashType::SHA256, linkPath).first.to_string(Base::Base32, false); + string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); if (hash != link.name) { - printError( - "link '%s' was modified! expected hash '%s', got '%s'", - linkPath, link.name, hash); + logError({ + .name = "Invalid hash", + .hint = hintfmt( + "link '%s' was modified! expected hash '%s', got '%s'", + linkPath, link.name, hash) + }); if (repair) { if (unlink(linkPath.c_str()) == 0) - printError("removed link '%s'", linkPath); + printInfo("removed link '%s'", linkPath); else throw SysError("removing corrupt link '%s'", linkPath); } else { @@ -1255,27 +1244,30 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking store hashes..."); - Hash nullHash(HashType::SHA256); + Hash nullHash(htSHA256); for (auto & i : validPaths) { try { auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); /* Check the content hash (optionally - slow). */ - printMsg(Verbosity::Talkative, "checking contents of '%s'", printStorePath(i)); + printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); std::unique_ptr<AbstractHashSink> hashSink; if (!info->ca || !info->references.count(info->path)) hashSink = std::make_unique<HashSink>(*info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, storePathToHash(printStorePath(info->path))); + hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(), current.first.to_string()); + logError({ + .name = "Invalid hash - path modified", + .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) + }); if (repair) repairPath(i); else errors = true; } else { @@ -1283,14 +1275,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Fill in missing hashes. */ if (info->narHash == nullHash) { - printError("fixing missing hash on '%s'", printStorePath(i)); + printInfo("fixing missing hash on '%s'", printStorePath(i)); info->narHash = current.first; update = true; } /* Fill in missing narSize fields (from old stores). */ if (info->narSize == 0) { - printError("updating size field on '%s' to %s", printStorePath(i), current.second); + printInfo("updating size field on '%s' to %s", printStorePath(i), current.second); info->narSize = current.second; update = true; } @@ -1306,7 +1298,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* It's possible that the path got GC'ed, so ignore errors on invalid paths. */ if (isValidPath(i)) - printError("error: %s", e.msg()); + logError(e.info()); else warn(e.msg()); errors = true; @@ -1326,7 +1318,10 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, if (!done.insert(pathS).second) return; if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); + logError({ + .name = "Nix path not found", + .hint = hintfmt("path '%s' is not in the Nix store", pathS) + }); return; } @@ -1345,16 +1340,19 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store, } if (canInvalidate) { - printError("path '%s' disappeared, removing from database...", pathS); + printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); invalidatePath(*state, path); } else { - printError("path '%s' disappeared, but it still has valid referrers!", pathS); + logError({ + .name = "Missing path with referrers", + .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS) + }); if (repair) try { repairPath(path); } catch (Error & e) { - warn(e.msg()); + logWarning(e.info()); errors = true; } else errors = true; @@ -1394,7 +1392,7 @@ static void makeMutable(const Path & path) AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); if (fd == -1) { if (errno == ELOOP) return; // it's a symlink - throw SysError(format("opening file '%1%'") % path); + throw SysError("opening file '%1%'", path); } unsigned int flags = 0, old; @@ -1412,7 +1410,7 @@ static void makeMutable(const Path & path) void LocalStore::upgradeStore7() { if (getuid() != 0) return; - printError("removing immutable bits from the Nix store (this may take a while)..."); + printInfo("removing immutable bits from the Nix store (this may take a while)..."); makeMutable(realStoreDir); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index bc9a415b3..e17cc45ae 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -135,8 +135,6 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const StorePath & path) override; - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -157,7 +155,7 @@ public: true) or simply the contents of a regular file (if recursive == false). */ StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 91acef368..aec4ed493 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil libnixrust +libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index fb538a1c5..e68edb38c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -103,7 +103,7 @@ void Store::computeFSClosure(const StorePath & startPath, StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { StorePathSet paths; - paths.insert(startPath.clone()); + paths.insert(startPath); computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers); } @@ -112,7 +112,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, unsigned long long & downloadSize_, unsigned long long & narSize_) { - Activity act(*logger, Verbosity::Debug, ActivityType::Unknown, "querying info about missing paths"); + Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); downloadSize_ = narSize_ = 0; @@ -141,11 +141,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); - state->willBuild.insert(drvPath.clone()); + state->willBuild.insert(drvPath); } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second })); }; auto checkOutput = [&]( @@ -157,9 +157,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, auto outPath = parseStorePath(outPathS); SubstitutablePathInfos infos; - StorePathSet paths; // FIXME - paths.insert(outPath.clone()); - querySubstitutablePathInfos(paths, infos); + querySubstitutablePathInfos({outPath}, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -170,10 +168,10 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (drvState->done) return; assert(drvState->left); drvState->left--; - drvState->outPaths.insert(outPath.clone()); + drvState->outPaths.insert(outPath); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone()))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } )); } } } @@ -190,12 +188,12 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (!isValidPath(path.path)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path.path.clone()); + state->unknown.insert(path.path); return; } auto drv = make_ref<Derivation>(derivationFromPath(path.path)); - ParsedDerivation parsedDrv(path.path.clone(), *drv); + ParsedDerivation parsedDrv(StorePath(path.path), *drv); PathSet invalid; for (auto & j : drv->outputs) @@ -216,13 +214,11 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - StorePathSet paths; // FIXME - paths.insert(path.path.clone()); - querySubstitutablePathInfos(paths, infos); + querySubstitutablePathInfos({path.path}, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path.path.clone()); + state->unknown.insert(path.path); return; } @@ -231,13 +227,13 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, { auto state(state_.lock()); - state->willSubstitute.insert(path.path.clone()); + state->willSubstitute.insert(path.path); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref })); } }; @@ -260,12 +256,12 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) throw BuildError("cycle detected in the references of '%s' from '%s'", printStorePath(path), printStorePath(*parent)); - if (!visited.insert(path.clone()).second) return; - parents.insert(path.clone()); + if (!visited.insert(path).second) return; + parents.insert(path); StorePathSet references; try { - references = cloneStorePathSet(queryPathInfo(path)->references); + references = queryPathInfo(path)->references; } catch (InvalidPath &) { } @@ -275,7 +271,7 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) if (i != path && paths.count(i)) dfsVisit(i, &path); - sorted.push_back(path.clone()); + sorted.push_back(path); parents.erase(path); }; diff --git a/src/libstore/names.cc b/src/libstore/names.cc new file mode 100644 index 000000000..d1c8a6101 --- /dev/null +++ b/src/libstore/names.cc @@ -0,0 +1,107 @@ +#include "names.hh" +#include "util.hh" + + +namespace nix { + + +DrvName::DrvName() +{ + name = ""; +} + + +/* Parse a derivation name. The `name' part of a derivation name is + everything up to but not including the first dash *not* followed by + a letter. The `version' part is the rest (excluding the separating + dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', + '2.0.48'). */ +DrvName::DrvName(std::string_view s) : hits(0) +{ + name = fullName = std::string(s); + for (unsigned int i = 0; i < s.size(); ++i) { + /* !!! isalpha/isdigit are affected by the locale. */ + if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) { + name = s.substr(0, i); + version = s.substr(i + 1); + break; + } + } +} + + +bool DrvName::matches(DrvName & n) +{ + if (name != "*") { + if (!regex) regex = std::unique_ptr<std::regex>(new std::regex(name, std::regex::extended)); + if (!std::regex_match(n.name, *regex)) return false; + } + if (version != "" && version != n.version) return false; + return true; +} + + +string nextComponent(string::const_iterator & p, + const string::const_iterator end) +{ + /* Skip any dots and dashes (component separators). */ + while (p != end && (*p == '.' || *p == '-')) ++p; + + if (p == end) return ""; + + /* If the first character is a digit, consume the longest sequence + of digits. Otherwise, consume the longest sequence of + non-digit, non-separator characters. */ + string s; + if (isdigit(*p)) + while (p != end && isdigit(*p)) s += *p++; + else + while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) + s += *p++; + + return s; +} + + +static bool componentsLT(const string & c1, const string & c2) +{ + int n1, n2; + bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); + + if (c1Num && c2Num) return n1 < n2; + else if (c1 == "" && c2Num) return true; + else if (c1 == "pre" && c2 != "pre") return true; + else if (c2 == "pre") return false; + /* Assume that `2.3a' < `2.3.1'. */ + else if (c2Num) return true; + else if (c1Num) return false; + else return c1 < c2; +} + + +int compareVersions(const string & v1, const string & v2) +{ + string::const_iterator p1 = v1.begin(); + string::const_iterator p2 = v2.begin(); + + while (p1 != v1.end() || p2 != v2.end()) { + string c1 = nextComponent(p1, v1.end()); + string c2 = nextComponent(p2, v2.end()); + if (componentsLT(c1, c2)) return -1; + else if (componentsLT(c2, c1)) return 1; + } + + return 0; +} + + +DrvNames drvNamesFromArgs(const Strings & opArgs) +{ + DrvNames result; + for (auto & i : opArgs) + result.push_back(DrvName(i)); + return result; +} + + +} diff --git a/src/libstore/names.hh b/src/libstore/names.hh new file mode 100644 index 000000000..00e14b8c7 --- /dev/null +++ b/src/libstore/names.hh @@ -0,0 +1,32 @@ +#pragma once + +#include <memory> + +#include "types.hh" +#include <regex> + +namespace nix { + +struct DrvName +{ + string fullName; + string name; + string version; + unsigned int hits; + + DrvName(); + DrvName(std::string_view s); + bool matches(DrvName & n); + +private: + std::unique_ptr<std::regex> regex; +}; + +typedef list<DrvName> DrvNames; + +string nextComponent(string::const_iterator & p, + const string::const_iterator end); +int compareVersions(const string & v1, const string & v2); +DrvNames drvNamesFromArgs(const Strings & opArgs); + +} diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index b74480684..ca663d837 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -184,7 +184,7 @@ struct NarAccessor : public FSAccessor auto i = get(path); if (i.type != FSAccessor::Type::tDirectory) - throw Error(format("path '%1%' inside NAR file is not a directory") % path); + throw Error("path '%1%' inside NAR file is not a directory", path); StringSet res; for (auto & child : i.children) @@ -197,7 +197,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); if (i.type != FSAccessor::Type::tRegular) - throw Error(format("path '%1%' inside NAR file is not a regular file") % path); + throw Error("path '%1%' inside NAR file is not a regular file", path); if (getNarBytes) return getNarBytes(i.start, i.size); @@ -209,7 +209,7 @@ struct NarAccessor : public FSAccessor { auto i = get(path); if (i.type != FSAccessor::Type::tSymlink) - throw Error(format("path '%1%' inside NAR file is not a symlink") % path); + throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } }; diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index def514840..012dea6ea 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -189,7 +189,7 @@ public: return {oInvalid, 0}; auto namePart = queryNAR.getStr(1); - auto narInfo = make_ref<NarInfo>(StorePath::fromBaseName(hashPart + "-" + namePart)); + auto narInfo = make_ref<NarInfo>(StorePath(hashPart + "-" + namePart)); narInfo->url = queryNAR.getStr(2); narInfo->compression = queryNAR.getStr(3); if (!queryNAR.isNull(4)) @@ -198,9 +198,9 @@ public: narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath::fromBaseName(r)); + narInfo->references.insert(StorePath(r)); if (!queryNAR.isNull(9)) - narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9)); + narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); @@ -230,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string()) + (info->narHash.to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index fe37d67ec..04550ed97 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,10 +4,10 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath::dummy) // FIXME: hack + : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack { auto corrupt = [&]() { - throw Error(format("NAR info file '%1%' is corrupt") % whence); + throw Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { @@ -56,11 +56,11 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString<Strings>(value, " "); if (!references.empty()) corrupt(); for (auto & r : refs) - references.insert(StorePath::fromBaseName(r)); + references.insert(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") - deriver = StorePath::fromBaseName(value); + deriver = StorePath(value); } else if (name == "System") system = value; @@ -87,11 +87,11 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash.type == HashType::SHA256); - res += "FileHash: " + fileHash.to_string(Base::Base32) + "\n"; + assert(fileHash.type == htSHA256); + res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == HashType::SHA256); - res += "NarHash: " + narHash.to_string(Base::Base32) + "\n"; + assert(narHash.type == htSHA256); + res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 5c01e1b3b..b2b2412a3 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -19,9 +19,9 @@ static void makeWritable(const Path & path) { struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError(format("changing writability of '%1%'") % path); + throw SysError("changing writability of '%1%'", path); } @@ -47,7 +47,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() InodeHash inodeHash; AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + if (!dir) throw SysError("opening directory '%1%'", linksDir); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -55,9 +55,9 @@ LocalStore::InodeHash LocalStore::loadInodeHash() // We don't care if we hit non-hash files, anything goes inodeHash.insert(dirent->d_ino); } - if (errno) throw SysError(format("reading directory '%1%'") % linksDir); + if (errno) throw SysError("reading directory '%1%'", linksDir); - printMsg(Verbosity::Talkative, format("loaded %1% hash inodes") % inodeHash.size()); + printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); return inodeHash; } @@ -68,7 +68,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa Strings names; AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError(format("opening directory '%1%'") % path); + if (!dir) throw SysError("opening directory '%1%'", path); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -83,7 +83,7 @@ Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHa if (name == "." || name == "..") continue; names.push_back(name); } - if (errno) throw SysError(format("reading directory '%1%'") % path); + if (errno) throw SysError("reading directory '%1%'", path); return names; } @@ -96,7 +96,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, struct stat st; if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%1%'", path); #if __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for @@ -130,7 +130,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, NixOS (example: $fontconfig/var/cache being modified). Skip those files. FIXME: check the modification time. */ if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { - printError(format("skipping suspicious writable file '%1%'") % path); + logWarning({ + .name = "Suspicious file", + .hint = hintfmt("skipping suspicious writable file '%1%'", path) + }); return; } @@ -149,11 +152,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(HashType::SHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); + Hash hash = hashPath(htSHA256, path).first; + debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base::Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(Base32, false); retry: if (!pathExists(linkPath)) { @@ -186,7 +189,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, current file with a hard link to that file. */ struct stat stLink; if (lstat(linkPath.c_str(), &stLink)) - throw SysError(format("getting attributes of path '%1%'") % linkPath); + throw SysError("getting attributes of path '%1%'", linkPath); if (st.st_ino == stLink.st_ino) { debug(format("'%1%' is already linked to '%2%'") % path % linkPath); @@ -194,12 +197,15 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, } if (st.st_size != stLink.st_size) { - printError(format("removing corrupted link '%1%'") % linkPath); + logWarning({ + .name = "Corrupted link", + .hint = hintfmt("removing corrupted link '%1%'", linkPath) + }); unlink(linkPath.c_str()); goto retry; } - printMsg(Verbosity::Talkative, format("linking '%1%' to '%2%'") % path % linkPath); + printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its @@ -229,7 +235,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* Atomically replace the old file with the new hard link. */ if (rename(tempLink.c_str(), path.c_str()) == -1) { if (unlink(tempLink.c_str()) == -1) - printError(format("unable to unlink '%1%'") % tempLink); + logError({ + .name = "Unlink error", + .hint = hintfmt("unable to unlink '%1%'", tempLink) + }); if (errno == EMLINK) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it @@ -238,7 +247,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, debug("'%s' has reached maximum number of links", linkPath); return; } - throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); + throw SysError("cannot rename '%1%' to '%2%'", tempLink, path); } stats.filesLinked++; @@ -246,13 +255,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, stats.blocksFreed += st.st_blocks; if (act) - act->result(ResultType::FileLinked, st.st_size, st.st_blocks); + act->result(resFileLinked, st.st_size, st.st_blocks); } void LocalStore::optimiseStore(OptimiseStats & stats) { - Activity act(*logger, ActivityType::OptimiseStore); + Activity act(*logger, actOptimiseStore); auto paths = queryAllValidPaths(); InodeHash inodeHash = loadInodeHash(); @@ -265,7 +274,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) addTempRoot(i); if (!isValidPath(i)) continue; /* path was GC'ed, probably */ { - Activity act(*logger, Verbosity::Talkative, ActivityType::Unknown, fmt("optimising path '%s'", printStorePath(i))); + Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash); } done++; diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 45c033c66..c7797b730 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -4,8 +4,8 @@ namespace nix { -ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) - : drvPath(std::move(drvPath)), drv(drv) +ParsedDerivation::ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv) + : drvPath(drvPath), drv(drv) { /* Parse the __json attribute, if any. */ auto jsonAttr = drv.env.find("__json"); @@ -117,4 +117,9 @@ bool ParsedDerivation::substitutesAllowed() const return getBoolAttr("allowSubstitutes", true); } +bool ParsedDerivation::contentAddressed() const +{ + return getBoolAttr("__contentAddressed", false); +} + } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index d0cc70f9c..0b8e8d031 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -12,7 +12,7 @@ class ParsedDerivation public: - ParsedDerivation(StorePath && drvPath, BasicDerivation & drv); + ParsedDerivation(const StorePath & drvPath, BasicDerivation & drv); ~ParsedDerivation(); @@ -34,6 +34,8 @@ public: bool willBuildLocally() const; bool substitutesAllowed() const; + + bool contentAddressed() const; }; } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 9a28aa96a..b3d8ce95c 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,38 +2,38 @@ namespace nix { -extern "C" { - rust::Result<StorePath> ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir); - rust::Result<StorePath> ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir); - rust::Result<StorePath> ffi_StorePath_fromBaseName(rust::StringSlice baseName); - rust::String ffi_StorePath_to_string(const StorePath & _this); - StorePath ffi_StorePath_clone(const StorePath & _this); - rust::StringSlice ffi_StorePath_name(const StorePath & _this); -} - -StorePath StorePath::make(std::string_view path, std::string_view storeDir) -{ - return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap(); -} - -StorePath StorePath::make(unsigned char hash[20], std::string_view name) -{ - return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap(); -} +MakeError(BadStorePath, Error); -StorePath StorePath::fromBaseName(std::string_view baseName) +static void checkName(std::string_view path, std::string_view name) { - return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap(); + if (name.empty()) + throw BadStorePath("store path '%s' has an empty name", path); + if (name.size() > 211) + throw BadStorePath("store path '%s' has a name longer than 211 characters", path); + for (auto c : name) + if (!((c >= '0' && c <= '9') + || (c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '=')) + throw BadStorePath("store path '%s' contains illegal character '%s'", path, c); } -rust::String StorePath::to_string() const +StorePath::StorePath(std::string_view _baseName) + : baseName(_baseName) { - return ffi_StorePath_to_string(*this); + if (baseName.size() < HashLen + 1) + throw BadStorePath("'%s' is too short to be a valid store path", baseName); + for (auto c : hashPart()) + if (c == 'e' || c == 'o' || c == 'u' || c == 't' + || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'))) + throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c); + checkName(baseName, name()); } -StorePath StorePath::clone() const +StorePath::StorePath(const Hash & hash, std::string_view _name) + : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) { - return ffi_StorePath_clone(*this); + checkName(baseName, name()); } bool StorePath::isDerivation() const @@ -41,18 +41,14 @@ bool StorePath::isDerivation() const return hasSuffix(name(), drvExtension); } -std::string_view StorePath::name() const -{ - return ffi_StorePath_name(*this); -} - -StorePath StorePath::dummy( - StorePath::make( - (unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")); +StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath Store::parseStorePath(std::string_view path) const { - return StorePath::make(path, storeDir); + auto p = canonPath(std::string(path)); + if (dirOf(p) != storeDir) + throw BadStorePath("path '%s' is not in the Nix store", p); + return StorePath(baseNameOf(p)); } std::optional<StorePath> Store::maybeParseStorePath(std::string_view path) const @@ -78,9 +74,7 @@ StorePathSet Store::parseStorePathSet(const PathSet & paths) const std::string Store::printStorePath(const StorePath & path) const { - auto s = storeDir + "/"; - s += (std::string_view) path.to_string(); - return s; + return (storeDir + "/").append(path.to_string()); } PathSet Store::printStorePathSet(const StorePathSet & paths) const @@ -90,29 +84,6 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const return res; } -StorePathSet cloneStorePathSet(const StorePathSet & paths) -{ - StorePathSet res; - for (auto & p : paths) - res.insert(p.clone()); - return res; -} - -StorePathSet storePathsToSet(const StorePaths & paths) -{ - StorePathSet res; - for (auto & p : paths) - res.insert(p.clone()); - return res; -} - -StorePathSet singleton(const StorePath & path) -{ - StorePathSet res; - res.insert(path.clone()); - return res; -} - std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s) { size_t n = s.find("!"); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index fdbc906ef..4f79843fe 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,74 +1,60 @@ #pragma once -#include "rust-ffi.hh" #include "content-address.hh" +#include "types.hh" namespace nix { -/* See path.rs. */ -struct StorePath; - class Store; +struct Hash; -extern "C" { - void ffi_StorePath_drop(void *); - bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b); - bool ffi_StorePath_eq(const StorePath & a, const StorePath & b); - void ffi_StorePath_clone_to(const StorePath & _other, StorePath & _this); - unsigned char * ffi_StorePath_hash_data(const StorePath & p); -} - -struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> +class StorePath { - StorePath() = delete; + std::string baseName; - static StorePath make(std::string_view path, std::string_view storeDir); +public: - static StorePath make(unsigned char hash[20], std::string_view name); + /* Size of the hash part of store paths, in base-32 characters. */ + constexpr static size_t HashLen = 32; // i.e. 160 bits - static StorePath fromBaseName(std::string_view baseName); + StorePath() = delete; - rust::String to_string() const; + StorePath(std::string_view baseName); - bool operator < (const StorePath & other) const - { - return ffi_StorePath_less_than(*this, other); - } + StorePath(const Hash & hash, std::string_view name); - bool operator == (const StorePath & other) const + std::string_view to_string() const { - return ffi_StorePath_eq(*this, other); + return baseName; } - bool operator != (const StorePath & other) const + bool operator < (const StorePath & other) const { - return !(*this == other); + return baseName < other.baseName; } - StorePath(StorePath && that) = default; - - StorePath(const StorePath & that) + bool operator == (const StorePath & other) const { - ffi_StorePath_clone_to(that, *this); + return baseName == other.baseName; } - void operator = (const StorePath & that) + bool operator != (const StorePath & other) const { - (rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>::operator = (that)); - ffi_StorePath_clone_to(that, *this); + return baseName != other.baseName; } - StorePath clone() const; - /* Check whether a file name ends with the extension for derivations. */ bool isDerivation() const; - std::string_view name() const; + std::string_view name() const + { + return std::string_view(baseName).substr(HashLen + 1); + } - unsigned char * hashData() const + std::string_view hashPart() const { - return ffi_StorePath_hash_data(*this); + return std::string_view(baseName).substr(0, HashLen); } static StorePath dummy; @@ -77,14 +63,6 @@ struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; -StorePathSet cloneStorePathSet(const StorePathSet & paths); -StorePathSet storePathsToSet(const StorePaths & paths); - -StorePathSet singleton(const StorePath & path); - -/* Size of the hash part of store paths, in base-32 characters. */ -const size_t storePathHashLen = 32; // i.e. 160 bits - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; @@ -93,18 +71,6 @@ struct StorePathWithOutputs StorePath path; std::set<std::string> outputs; - StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {}) - : path(path.clone()), outputs(outputs) - { } - - StorePathWithOutputs(StorePath && path, std::set<std::string> && outputs) - : path(std::move(path)), outputs(std::move(outputs)) - { } - - StorePathWithOutputs(const StorePathWithOutputs & other) - : path(other.path.clone()), outputs(other.outputs) - { } - std::string to_string(const Store & store) const; }; @@ -117,7 +83,7 @@ namespace std { template<> struct hash<nix::StorePath> { std::size_t operator()(const nix::StorePath & path) const noexcept { - return * (std::size_t *) path.hashData(); + return * (std::size_t *) path.to_string().data(); } }; diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 2635e3940..926f4ea1e 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -20,7 +20,7 @@ AutoCloseFD openLockFile(const Path & path, bool create) fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); if (!fd && (create || errno != ENOENT)) - throw SysError(format("opening lock file '%1%'") % path); + throw SysError("opening lock file '%1%'", path); return fd; } @@ -51,7 +51,7 @@ bool lockFile(int fd, LockType lockType, bool wait) while (flock(fd, type) != 0) { checkInterrupt(); if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); + throw SysError("acquiring/releasing lock"); else return false; } @@ -60,7 +60,7 @@ bool lockFile(int fd, LockType lockType, bool wait) checkInterrupt(); if (errno == EWOULDBLOCK) return false; if (errno != EINTR) - throw SysError(format("acquiring/releasing lock")); + throw SysError("acquiring/releasing lock"); } } @@ -124,7 +124,7 @@ bool PathLocks::lockPaths(const PathSet & paths, hasn't been unlinked). */ struct stat st; if (fstat(fd.get(), &st) == -1) - throw SysError(format("statting lock file '%1%'") % lockPath); + throw SysError("statting lock file '%1%'", lockPath); if (st.st_size != 0) /* This lock file has been unlinked, so we're holding a lock on a deleted file. This means that other @@ -160,7 +160,8 @@ void PathLocks::unlock() if (close(i.first) == -1) printError( - format("error (ignored): cannot close lock file on '%1%'") % i.second); + "error (ignored): cannot close lock file on '%1%'", + i.second); debug(format("lock released on '%1%'") % i.second); } diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 2bef51878..6cfe393a4 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -50,7 +50,7 @@ Generations findGenerations(Path profile, int & curGen) gen.number = n; struct stat st; if (lstat(gen.path.c_str(), &st) != 0) - throw SysError(format("statting '%1%'") % gen.path); + throw SysError("statting '%1%'", gen.path); gen.creationTime = st.st_mtime; gens.push_back(gen); } @@ -117,7 +117,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) static void removeFile(const Path & path) { if (remove(path.c_str()) == -1) - throw SysError(format("cannot unlink '%1%'") % path); + throw SysError("cannot unlink '%1%'", path); } @@ -149,7 +149,7 @@ void deleteGenerations(const Path & profile, const std::set<unsigned int> & gens Generations gens = findGenerations(profile, curGen); if (gensToDelete.find(curGen) != gensToDelete.end()) - throw Error(format("cannot delete current generation of profile %1%'") % profile); + throw Error("cannot delete current generation of profile %1%'", profile); for (auto & i : gens) { if (gensToDelete.find(i.number) == gensToDelete.end()) continue; @@ -226,7 +226,7 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b int days; if (!string2Int(strDays, days) || days < 1) - throw Error(format("invalid number of days specifier '%1%'") % timeSpec); + throw Error("invalid number of days specifier '%1%'", timeSpec); time_t oldTime = curTime - days * 24 * 3600; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 6652e1e26..a10d536a3 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -54,7 +54,7 @@ struct RefScanSink : Sink string tail; - RefScanSink() : hashSink(HashType::SHA256) { } + RefScanSink() : hashSink(htSHA256) { } void operator () (const unsigned char * data, size_t len); }; @@ -92,11 +92,11 @@ PathSet scanForReferences(const string & path, auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); if (pos == string::npos) - throw Error(format("bad reference '%1%'") % i); + throw Error("bad reference '%1%'", i); string s = string(baseName, 0, pos); assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); - // parseHash(HashType::SHA256, s); + // parseHash(htSHA256, s); sink.hashes.insert(s); backMap[s] = i; } diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 5a2d103b9..bd698d781 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -19,7 +19,7 @@ RemoteFSAccessor::RemoteFSAccessor(ref<Store> store, const Path & cacheDir) Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) { assert(cacheDir != ""); - return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); + return fmt("%s/%s.%s", cacheDir, store->parseStorePath(storePath).hashPart(), ext); } void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, @@ -51,7 +51,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) std::string restPath = std::string(path, storePath.size()); if (!store->isValidPath(store->parseStorePath(storePath))) - throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + throw InvalidPath("path '%1%' is not a valid store path", storePath); auto i = nars.find(storePath); if (i != nars.end()) return {i->second, restPath}; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0faa4d824..b7cc7a5fc 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -116,11 +116,11 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() struct sockaddr_un addr; addr.sun_family = AF_UNIX; if (socketPath.size() + 1 >= sizeof(addr.sun_path)) - throw Error(format("socket path '%1%' is too long") % socketPath); + throw Error("socket path '%1%' is too long", socketPath); strcpy(addr.sun_path, socketPath.c_str()); if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError(format("cannot connect to daemon at '%1%'") % socketPath); + throw SysError("cannot connect to daemon at '%1%'", socketPath); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); @@ -177,11 +177,11 @@ void RemoteStore::setOptions(Connection & conn) << settings.keepFailed << settings.keepGoing << settings.tryFallback - << (uint64_t) verbosity + << verbosity << settings.maxBuildJobs << settings.maxSilentTime << true - << (uint64_t) (settings.verboseBuild ? Verbosity::Error : Verbosity::Vomit) + << (settings.verboseBuild ? lvlError : lvlVomit) << 0 // obsolete log type << 0 /* obsolete print build trace */ << settings.buildCores @@ -228,7 +228,7 @@ struct ConnectionHandle ~ConnectionHandle() { - if (!daemonException && std::uncaught_exception()) { + if (!daemonException && std::uncaught_exceptions()) { handle.markBad(); debug("closing daemon connection because of an exception"); } @@ -268,7 +268,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i.clone()); + if (isValidPath(i)) res.insert(i); return res; } else { conn->to << wopQueryValidPaths; @@ -296,7 +296,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & i : paths) { conn->to << wopHasSubstitutes << printStorePath(i); conn.processStderr(); - if (readInt(conn->from)) res.insert(i.clone()); + if (readInt(conn->from)) res.insert(i); } return res; } else { @@ -329,7 +329,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, info.references = readStorePaths<StorePathSet>(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.clone(), std::move(info)); + infos.insert_or_assign(i, std::move(info)); } } else { @@ -365,17 +365,17 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, } catch (Error & e) { // Ugly backwards compatibility hack. if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(e.what()); + throw InvalidPath(e.info()); throw; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = std::make_shared<ValidPathInfo>(path.clone()); + info = std::make_shared<ValidPathInfo>(StorePath(path)); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->narHash = Hash(readString(conn->from), HashType::SHA256); + info->narHash = Hash(readString(conn->from), htSHA256); info->references = readStorePaths<StorePathSet>(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { @@ -396,7 +396,7 @@ void RemoteStore::queryReferrers(const StorePath & path, conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) - referrers.insert(i.clone()); + referrers.insert(i); } @@ -418,15 +418,6 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputNames << printStorePath(path); - conn.processStderr(); - return readStrings<PathSet>(conn->from); -} - - std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); @@ -471,7 +462,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << wopAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base::Base16, false); + << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) @@ -495,7 +486,7 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, conn->to << wopAddToStore << name - << ((hashAlgo == HashType::SHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << (method == FileIngestionMethod::Recursive ? 1 : 0) << printHashType(hashAlgo); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index bd541779c..80c8e9f11 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -51,8 +51,6 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const StorePath & path) override; - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -65,7 +63,7 @@ public: std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f43581bf6..427dd48ce 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -32,8 +32,10 @@ namespace nix { struct S3Error : public Error { Aws::S3::S3Errors err; - S3Error(Aws::S3::S3Errors err, const FormatOrString & fs) - : Error(fs), err(err) { }; + + template<typename... Args> + S3Error(Aws::S3::S3Errors err, const Args & ... args) + : Error(args...), err(err) { }; }; /* Helper: given an Outcome<R, E>, return R in case of success, or @@ -68,9 +70,9 @@ static void initAWS() shared.cc), so don't let aws-sdk-cpp override it. */ options.cryptoOptions.initAndCleanupOpenSSL = false; - if (verbosity >= Verbosity::Debug) { + if (verbosity >= lvlDebug) { options.loggingOptions.logLevel = - verbosity == Verbosity::Debug + verbosity == lvlDebug ? Aws::Utils::Logging::LogLevel::Debug : Aws::Utils::Logging::LogLevel::Trace; options.loggingOptions.logger_create_fn = [options]() { @@ -109,7 +111,9 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); if (retry) printError("AWS error '%s' (%s), will retry in %d ms", - error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); + error.GetExceptionName(), + error.GetMessage(), + CalculateDelayBeforeNextRetry(error, attemptedRetries)); return retry; } }; @@ -249,7 +253,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore // If bucket listing is disabled, 404s turn into 403s || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) return false; - throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage()); + throw Error("AWS error fetching '%s': %s", path, error.GetMessage()); } return true; diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index eb1daafc5..76c822c4e 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -29,7 +29,7 @@ SQLite::SQLite(const Path & path, bool create) { if (sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) - throw Error(format("cannot open SQLite database '%s'") % path); + throw Error("cannot open SQLite database '%s'", path); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) throwSQLiteError(db, "setting timeout"); @@ -204,7 +204,10 @@ void handleSQLiteBusy(const SQLiteBusy & e) if (now > lastWarned + 10) { lastWarned = now; - printError("warning: %s", e.what()); + logWarning({ + .name = "Sqlite busy", + .hint = hintfmt(e.what()) + }); } /* Sleep for a while since retrying the transaction right away diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index fd04c9b07..dd81ab051 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -3,7 +3,7 @@ #include <functional> #include <string> -#include "types.hh" +#include "error.hh" struct sqlite3; struct sqlite3_stmt; diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index f61c094a2..84548a6e4 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -58,7 +58,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string addCommonSSHOpts(args); if (socketPath != "") args.insert(args.end(), {"-S", socketPath}); - if (verbosity >= Verbosity::Chatty) + if (verbosity >= lvlChatty) args.push_back("-v"); } @@ -110,7 +110,7 @@ Path SSHMaster::startMaster() , "-o", "LocalCommand=echo started" , "-o", "PermitLocalCommand=yes" }; - if (verbosity >= Verbosity::Chatty) + if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4bd3121ed..60d1d22a2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -23,7 +23,7 @@ bool Store::isInStore(const Path & path) const Path Store::toStorePath(const Path & path) const { if (!isInStore(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); + throw Error("path '%1%' is not in the Nix store", path); Path::size_type slash = path.find('/', storeDir.size() + 1); if (slash == Path::npos) return path; @@ -55,21 +55,13 @@ StorePath Store::followLinksToStorePath(std::string_view path) const StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const { auto [path2, outputs] = nix::parsePathWithOutputs(path); - return StorePathWithOutputs(followLinksToStorePath(path2), std::move(outputs)); -} - - -string storePathToHash(const Path & path) -{ - auto base = baseNameOf(path); - assert(base.size() >= storePathHashLen); - return string(base, 0, storePathHashLen); + return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) }; } /* Store paths have the following form: - <store>/<h>-<name> + <realized-path> = <store>/<h>-<name> where @@ -93,11 +85,14 @@ string storePathToHash(const Path & path) <type> = one of: "text:<r1>:<r2>:...<rN>" for plain text files written to the store using - addTextToStore(); <r1> ... <rN> are the references of the - path. - "source" + addTextToStore(); <r1> ... <rN> are the store paths referenced + by this path, in the form described by <realized-path> + "source:<r1>:<r2>:...:<rN>:self" for paths copied to the store using addToStore() when recursive - = true and hashAlgo = "sha256" + = true and hashAlgo = "sha256". Just like in the text case, we + can have the store paths referenced by the path. + Additionally, we can have an optional :self label to denote self + reference. "output:<id>" for either the outputs created by derivations, OR paths copied to the store using addToStore() with recursive != true or @@ -125,6 +120,12 @@ string storePathToHash(const Path & path) the contents of the path (or expected contents of the path for fixed-output derivations) + Note that since an output derivation has always type output, while + something added by addToStore can have type output or source depending + on the hash, this means that the same input can be hashed differently + if added to the store via addToStore or via a derivation, in the sha256 + recursive case. + It would have been nicer to handle fixed-output derivations under "source", e.g. have something like "source:<rec><algo>", but we're stuck with this for now... @@ -142,9 +143,9 @@ StorePath Store::makeStorePath(const string & type, const Hash & hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base::Base16) + ":" + storeDir + ":" + std::string(name); - auto h = compressHash(hashString(HashType::SHA256, s), 20); - return StorePath::make(h.hash, name); + string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); + auto h = compressHash(hashString(htSHA256, s), 20); + return StorePath(h, name); } @@ -178,13 +179,16 @@ StorePath Store::makeFixedOutputPath( const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == HashType::SHA256 && method == FileIngestionMethod::Recursive) { + if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); - return makeStorePath("output:out", hashString(HashType::SHA256, - "fixed:out:" + makeFileIngestionPrefix(method) + - hash.to_string(Base::Base16) + ":"), name); + return makeStorePath("output:out", + hashString(htSHA256, + "fixed:out:" + + makeFileIngestionPrefix(method) + + hash.to_string(Base16, true) + ":"), + name); } } @@ -192,7 +196,7 @@ StorePath Store::makeFixedOutputPath( StorePath Store::makeTextPath(std::string_view name, const Hash & hash, const StorePathSet & references) const { - assert(hash.type == HashType::SHA256); + assert(hash.type == htSHA256); /* Stuff the references (if any) into the type. This is a bit hacky, but we can't put them in `s' since that would be ambiguous. */ @@ -213,7 +217,7 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, hashString(HashType::SHA256, s), references); + return makeTextPath(name, hashString(htSHA256, s), references); } @@ -240,7 +244,7 @@ bool Store::PathInfoCacheValue::isKnownNow() bool Store::isValidPath(const StorePath & storePath) { - auto hashPart = storePathToHash(printStorePath(storePath)); + std::string hashPart(storePath.hashPart()); { auto state_(state.lock()); @@ -308,7 +312,7 @@ void Store::queryPathInfo(const StorePath & storePath, std::string hashPart; try { - hashPart = storePathToHash(printStorePath(storePath)); + hashPart = storePath.hashPart(); { auto res = state.lock()->pathInfoCache.get(hashPart); @@ -426,7 +430,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base::Base16, false) + "\n"; + s += info->narHash.to_string(Base16, false) + "\n"; s += (format("%1%\n") % info->narSize).str(); } @@ -458,7 +462,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string(hashBase)) + .attr("narHash", info->narHash.to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -501,7 +505,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); + jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -550,7 +554,7 @@ void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMod for (auto & path : paths) { if (path.path.isDerivation()) unsupported("buildPaths"); - paths2.insert(path.path.clone()); + paths2.insert(path.path); } if (queryValidPaths(paths2).size() != paths2.size()) @@ -564,7 +568,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, auto srcUri = srcStore->getUri(); auto dstUri = dstStore->getUri(); - Activity act(*logger, Verbosity::Info, ActivityType::CopyPath, + Activity act(*logger, lvlInfo, actCopyPath, srcUri == "local" || srcUri == "daemon" ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri) : dstUri == "local" || dstUri == "daemon" @@ -581,7 +585,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, StringSink sink; srcStore->narFromPath({storePath}, sink); auto info2 = make_ref<ValidPathInfo>(*info); - info2->narHash = hashString(HashType::SHA256, *sink.s); + info2->narHash = hashString(htSHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); if (info->ultimate) info2->ultimate = false; info = info2; @@ -623,7 +627,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st if (missing.empty()) return; - Activity act(*logger, Verbosity::Info, ActivityType::CopyPaths, fmt("copying %d paths", missing.size())); + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); std::atomic<size_t> nrDone{0}; std::atomic<size_t> nrFailed{0}; @@ -649,7 +653,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); bytesExpected += info->narSize; - act.setExpected(ActivityType::CopyPath, bytesExpected); + act.setExpected(actCopyPath, bytesExpected); return srcStore->printStorePathSet(info->references); }, @@ -668,7 +672,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st nrFailed++; if (!settings.keepGoing) throw e; - logger->log(Verbosity::Error, fmt("could not copy %s: %s", storePathS, e.what())); + logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); showProgress(); return; } @@ -699,7 +703,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre if (hashGiven) { string s; getline(str, s); - info.narHash = Hash(s, HashType::SHA256); + info.narHash = Hash(s, htSHA256); getline(str, s); if (!string2Int(s, info.narSize)) throw Error("number expected"); } @@ -742,7 +746,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base::Base32) + ";" + + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -766,7 +770,7 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const return store.makeTextPath(path.name(), th.hash, references); }, [&](FileSystemHash fsh) { - auto refs = cloneStorePathSet(references); + auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; @@ -906,7 +910,7 @@ std::list<ref<Store>> getDefaultSubstituters() try { stores.push_back(openStore(uri)); } catch (Error & e) { - printError("warning: %s", e.what()); + logWarning(e.info()); } }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 441e3efc5..25d78c297 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -178,9 +178,10 @@ struct ValidPathInfo Strings shortRefs() const; + ValidPathInfo(const ValidPathInfo & other) = default; + ValidPathInfo(StorePath && path) : path(std::move(path)) { }; ValidPathInfo(const StorePath & path) : path(path) { }; - ValidPathInfo(const ValidPathInfo & other) = default; virtual ~ValidPathInfo() { } }; @@ -349,7 +350,7 @@ public: path and the cryptographic hash of the contents of srcPath. */ std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = HashType::SHA256, PathFilter & filter = defaultPathFilter) const; + HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). @@ -420,10 +421,6 @@ public: virtual StorePathSet queryDerivationOutputs(const StorePath & path) { unsupported("queryDerivationOutputs"); } - /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const StorePath & path) - { unsupported("queryDerivationOutputNames"); } - /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0; @@ -447,12 +444,12 @@ public: The function object `filter' can be used to exclude files (see libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; // FIXME: remove? virtual StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); } @@ -546,7 +543,7 @@ public: each path is included. */ void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - Base hashBase = Base::Base32, + Base hashBase = Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /* Return the size of the closure of the specified path, that is, @@ -577,6 +574,9 @@ public: ensurePath(). */ Derivation derivationFromPath(const StorePath & drvPath); + /* Read a derivation (which must already be valid). */ + Derivation readDerivation(const StorePath & drvPath); + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly or indirectly reached from it. `out' is not cleared. If @@ -722,10 +722,6 @@ public: }; -/* Extract the hash part of the given store path. */ -string storePathToHash(const Path & path); - - /* Copy a path from one store to another. */ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 857d54d99..ac42457fc 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -36,7 +36,7 @@ typedef enum { wopClearFailedPaths = 25, wopQueryPathInfo = 26, wopImportPaths = 27, // obsolete - wopQueryDerivationOutputNames = 28, + wopQueryDerivationOutputNames = 28, // obsolete wopQueryPathFromHashPart = 29, wopQuerySubstitutablePathInfos = 30, wopQueryValidPaths = 31, |