diff options
Diffstat (limited to 'src/libstore')
36 files changed, 1751 insertions, 1509 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 7e3e5ff88..aaacf8281 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -91,19 +91,18 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) return sink.s; } -Path BinaryCacheStore::narInfoFileFor(const Path & storePath) +std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) { - assertStorePath(storePath); - return storePathToHash(storePath) + ".narinfo"; + return storePathToHash(printStorePath(storePath)) + ".narinfo"; } void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) { auto narInfoFile = narInfoFileFor(narInfo->path); - upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo"); + upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - auto hashPart = storePathToHash(narInfo->path); + auto hashPart = storePathToHash(printStorePath(narInfo->path)); { auto state_(state.lock()); @@ -126,8 +125,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str if (ref != info.path) queryPathInfo(ref); } catch (InvalidPath &) { - throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid") - % info.path % ref); + throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", + printStorePath(info.path), printStorePath(ref)); } assert(nar->compare(0, narMagic.size(), narMagic) == 0); @@ -138,14 +137,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str narInfo->narHash = hashString(htSHA256, *nar); if (info.narHash && info.narHash != narInfo->narHash) - throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path); + throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); auto narAccessor = makeNarAccessor(nar); if (accessor_) - accessor_->addToCache(info.path, *nar, narAccessor); + accessor_->addToCache(printStorePath(info.path), *nar, narAccessor); /* Optionally write a JSON file containing a listing of the contents of the NAR. */ @@ -162,7 +161,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str } } - upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json"); + upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json"); } /* Compress the NAR. */ @@ -174,10 +173,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str narInfo->fileSize = narCompressed->size(); auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); - printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") - % narInfo->path % narInfo->narSize - % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) - % duration); + printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printStorePath(narInfo->path), narInfo->narSize, + ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), + duration); narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : @@ -254,14 +253,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ - if (secretKey) narInfo->sign(*secretKey); + if (secretKey) narInfo->sign(*this, *secretKey); writeNarInfo(narInfo); stats.narInfoWrite++; } -bool BinaryCacheStore::isValidPathUncached(const Path & storePath) +bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) { // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even @@ -269,7 +268,7 @@ bool BinaryCacheStore::isValidPathUncached(const Path & storePath) return fileExists(narInfoFileFor(storePath)); } -void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) +void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast<const NarInfo>(); @@ -295,12 +294,13 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) stats.narReadBytes += narSize; } -void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, +void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { auto uri = getUri(); + auto storePathS = printStorePath(storePath); auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo, - fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri}); + fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); PushActivity pact(act->id); auto narInfoFile = narInfoFileFor(storePath); @@ -326,7 +326,7 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, }}); } -Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, +StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { // FIXME: some cut&paste from LocalStore::addToStore(). @@ -345,20 +345,18 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, h = hashString(hashAlgo, s); } - ValidPathInfo info; - info.path = makeFixedOutputPath(recursive, h, name); + ValidPathInfo info(makeFixedOutputPath(recursive, h, name)); addToStore(info, sink.s, repair, CheckSigs, nullptr); - return info.path; + return std::move(info.path); } -Path BinaryCacheStore::addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) +StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) { - ValidPathInfo info; - info.path = computeStorePathForText(name, s, references); - info.references = references; + ValidPathInfo info(computeStorePathForText(name, s, references)); + info.references = cloneStorePathSet(references); if (repair || !isValidPath(info.path)) { StringSink sink; @@ -366,7 +364,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, addToStore(info, sink.s, repair, CheckSigs, nullptr); } - return info.path; + return std::move(info.path); } ref<FSAccessor> BinaryCacheStore::getFSAccessor() @@ -374,7 +372,7 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor() return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache); } -void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs) +void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { /* Note: this is inherently racy since there is no locking on binary caches. In particular, with S3 this unreliable, even @@ -390,24 +388,22 @@ void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & s writeNarInfo(narInfo); } -std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path) +std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) { - Path drvPath; + auto drvPath = path.clone(); - if (isDerivation(path)) - drvPath = path; - else { + if (!path.isDerivation()) { try { auto info = queryPathInfo(path); // FIXME: add a "Log" field to .narinfo - if (info->deriver == "") return nullptr; - drvPath = info->deriver; + if (!info->deriver) return nullptr; + drvPath = info->deriver->clone(); } catch (InvalidPath &) { return nullptr; } } - auto logPath = "log/" + baseNameOf(drvPath); + auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 2d7cd1947..fa2200ad8 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -65,18 +65,18 @@ private: std::string narMagic; - std::string narInfoFileFor(const Path & storePath); + std::string narInfoFileFor(const StorePath & storePath); void writeNarInfo(ref<NarInfo> narInfo); public: - bool isValidPathUncached(const Path & path) override; + bool isValidPathUncached(const StorePath & path) override; - void queryPathInfoUncached(const Path & path, + void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override; - Path queryPathFromHashPart(const string & hashPart) override + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } bool wantMassQuery() override { return wantMassQuery_; } @@ -85,27 +85,27 @@ public: RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; - Path addToStore(const string & name, const Path & srcPath, + StorePath addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override; + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; - void narFromPath(const Path & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { unsupported("buildDerivation"); } - void ensurePath(const Path & path) override + void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } ref<FSAccessor> getFSAccessor() override; - void addSignatures(const Path & storePath, const StringSet & sigs) override; + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - std::shared_ptr<std::string> getBuildLog(const Path & path) override; + std::shared_ptr<std::string> getBuildLog(const StorePath & path) override; int getPriority() override { return priority; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0de2f5bd2..05c4cb621 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -15,6 +15,7 @@ #include "parsed-derivations.hh" #include "machines.hh" #include "daemon.hh" +#include "worker-protocol.hh" #include <algorithm> #include <iostream> @@ -99,7 +100,7 @@ typedef set<GoalPtr, CompareGoalPtrs> Goals; typedef list<WeakGoalPtr> WeakGoals; /* A map of paths to goals (and the other way around). */ -typedef map<Path, WeakGoalPtr> WeakGoalMap; +typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; @@ -254,7 +255,7 @@ private: steady_time_point lastWokenUp; /* Cache for pathContentsGood(). */ - std::map<Path, bool> pathContentsGoodCache; + std::map<StorePath, bool> pathContentsGoodCache; public: @@ -301,10 +302,10 @@ public: ~Worker(); /* Make a goal (with caching). */ - GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const Path & drvPath, + GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(StorePath && drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const Path & storePath, RepairFlag repair = NoRepair); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -352,9 +353,9 @@ public: /* Check whether the given valid path exists and has the right contents. */ - bool pathContentsGood(const Path & path); + bool pathContentsGood(const StorePath & path); - void markContentsGood(const Path & path); + void markContentsGood(StorePath && path); void updateProgress() { @@ -471,7 +472,10 @@ static void commonChildInit(Pipe & logPipe) close(fdDevNull); } -void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) +void handleDiffHook( + uid_t uid, uid_t gid, + const Path & tryA, const Path & tryB, + const Path & drvPath, const Path & tmpDir) { auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { @@ -500,12 +504,6 @@ void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Pa class UserLock { private: - /* POSIX locks suck. If we have a lock on a file, and we open and - close that file again (without closing the original file - descriptor), we lose the lock. So we have to be *very* careful - not to open a lock file on which we are holding a lock. */ - static Sync<PathSet> lockedPaths_; - Path fnUserLock; AutoCloseFD fdUserLock; @@ -516,7 +514,6 @@ private: public: UserLock(); - ~UserLock(); void kill(); @@ -530,9 +527,6 @@ public: }; -Sync<PathSet> UserLock::lockedPaths_; - - UserLock::UserLock() { assert(settings.buildUsersGroup != ""); @@ -569,47 +563,34 @@ UserLock::UserLock() fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - { - auto lockedPaths(lockedPaths_.lock()); - if (!lockedPaths->insert(fnUserLock).second) - /* We already have a lock on this one. */ - continue; - } + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError(format("opening user lock '%1%'") % fnUserLock); - try { + if (lockFile(fd.get(), ltWrite, false)) { + fdUserLock = std::move(fd); + user = i; + uid = pw->pw_uid; - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError(format("opening user lock '%1%'") % fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error(format("the Nix user should not be a member of '%1%'") - % settings.buildUsersGroup); + /* Sanity check... */ + if (uid == getuid() || uid == geteuid()) + throw Error(format("the Nix user should not be a member of '%1%'") + % settings.buildUsersGroup); #if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name); - - supplementaryGIDs.resize(ngroups); + /* Get the list of supplementary groups of this build user. This + is usually either empty or contains a group such as "kvm". */ + supplementaryGIDs.resize(10); + int ngroups = supplementaryGIDs.size(); + int err = getgrouplist(pw->pw_name, pw->pw_gid, + supplementaryGIDs.data(), &ngroups); + if (err == -1) + throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name); + + supplementaryGIDs.resize(ngroups); #endif - return; - } - - } catch (...) { - lockedPaths_.lock()->erase(fnUserLock); + return; } } @@ -619,14 +600,6 @@ UserLock::UserLock() } -UserLock::~UserLock() -{ - auto lockedPaths(lockedPaths_.lock()); - auto erased = lockedPaths->erase(fnUserLock); - assert(erased); -} - - void UserLock::kill() { killUser(uid); @@ -694,7 +667,7 @@ HookInstance::HookInstance() throw SysError("dupping builder's stdout/stderr"); Strings args = { - baseNameOf(settings.buildHook), + std::string(baseNameOf(settings.buildHook.get())), std::to_string(verbosity), }; @@ -741,7 +714,7 @@ private: bool useDerivation; /* The path of the derivation. */ - Path drvPath; + StorePath drvPath; /* The specific outputs that we need to build. Empty means all of them. */ @@ -766,14 +739,14 @@ private: /* All input paths (that is, the union of FS closures of the immediate input paths). */ - PathSet inputPaths; + StorePathSet inputPaths; /* Outputs that are already valid. If we're repairing, these are the outputs that are valid *and* not corrupt. */ - PathSet validPaths; + StorePathSet validPaths; /* Outputs that are corrupt or not valid. */ - PathSet missingPaths; + StorePathSet missingPaths; /* User selected for running the builder. */ std::unique_ptr<UserLock> buildUser; @@ -853,7 +826,7 @@ private: /* Hash rewriting. */ StringMap inputRewrites, outputRewrites; - typedef map<Path, Path> RedirectedOutputs; + typedef map<StorePath, StorePath> RedirectedOutputs; RedirectedOutputs redirectedOutputs; BuildMode buildMode; @@ -861,7 +834,7 @@ private: /* If we're repairing without a chroot, there may be outputs that are valid but corrupt. So we redirect these outputs to temporary paths. */ - PathSet redirectedBadOutputs; + StorePathSet redirectedBadOutputs; BuildResult result; @@ -900,23 +873,23 @@ private: std::vector<std::thread> daemonWorkerThreads; /* Paths that were added via recursive Nix calls. */ - PathSet addedPaths; + StorePathSet addedPaths; /* Recursive Nix calls are only allowed to build or realize paths in the original input closure or added via a recursive Nix call (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where /nix/store/<bla> is some arbitrary path in a binary cache). */ - bool isAllowed(const Path & path) + bool isAllowed(const StorePath & path) { return inputPaths.count(path) || addedPaths.count(path); } - friend class RestrictedStore; + friend struct RestrictedStore; public: - DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, + DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - DerivationGoal(const Path & drvPath, const BasicDerivation & drv, + DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); @@ -931,14 +904,14 @@ public: i.e. a derivation named "aardvark" always comes before "baboon". And substitution goals always happen before derivation goals (due to "b$"). */ - return "b$" + storePathToName(drvPath) + "$" + drvPath; + return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } void work() override; - Path getDrvPath() + StorePath getDrvPath() { - return drvPath; + return drvPath.clone(); } /* Add wanted outputs to an already existing derivation goal. */ @@ -978,7 +951,7 @@ private: /* Add 'path' to the set of paths that may be referenced by the outputs, and make it appear in the sandbox. */ - void addDependency(const Path & path); + void addDependency(const StorePath & path); /* Make a file owned by the builder. */ void chownToBuilder(const Path & path); @@ -1012,15 +985,12 @@ private: void flushLine(); /* Return the set of (in)valid paths. */ - PathSet checkPathValidity(bool returnValid, bool checkHash); - - /* Abort the goal if `path' failed to build. */ - bool pathFailed(const Path & path); + StorePathSet checkPathValidity(bool returnValid, bool checkHash); /* Forcibly kill the child process, if any. */ void killChild(); - Path addHashRewrite(const Path & path); + void addHashRewrite(const StorePath & path); void repairClosure(); @@ -1031,23 +1001,23 @@ private: void done(BuildResult::Status status, const string & msg = ""); - PathSet exportReferences(PathSet storePaths); + StorePathSet exportReferences(const StorePathSet & storePaths); }; const Path DerivationGoal::homeDir = "/homeless-shelter"; -DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, +DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(true) - , drvPath(drvPath) + , drvPath(std::move(drvPath)) , wantedOutputs(wantedOutputs) , buildMode(buildMode) { state = &DerivationGoal::getDerivation; - name = (format("building of '%1%'") % drvPath).str(); + name = fmt("building of '%s'", worker.store.printStorePath(drvPath)); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -1055,16 +1025,16 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut } -DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv, +DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(false) - , drvPath(drvPath) + , drvPath(std::move(drvPath)) , buildMode(buildMode) { - this->drv = std::unique_ptr<BasicDerivation>(new BasicDerivation(drv)); + this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); + name = fmt("building of %s", worker.store.showPaths(drv.outputPaths())); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -1174,7 +1144,7 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - printError(format("cannot build missing derivation '%1%'") % drvPath); + printError("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)); done(BuildResult::MiscFailure); return; } @@ -1203,7 +1173,7 @@ void DerivationGoal::haveDerivation() worker.store.addTempRoot(i.second.path); /* Check what outputs paths are not already valid. */ - PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); + auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); /* If they are all valid, then we're done. */ if (invalidOutputs.size() == 0 && buildMode == bmNormal) { @@ -1211,7 +1181,7 @@ void DerivationGoal::haveDerivation() return; } - parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + parsedDrv = std::make_unique<ParsedDerivation>(drvPath.clone(), *drv); /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build @@ -1232,7 +1202,9 @@ void DerivationGoal::outputsSubstituted() trace("all outputs substituted (maybe)"); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, (format("some substitutes for the outputs of derivation '%1%' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ") % drvPath).str()); + done(BuildResult::TransientFailure, + fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", + worker.store.printStorePath(drvPath))); return; } @@ -1259,14 +1231,15 @@ void DerivationGoal::outputsSubstituted() return; } if (buildMode == bmCheck && nrInvalid > 0) - throw Error(format("some outputs of '%1%' are not valid, so checking is not possible") % drvPath); + throw Error("some outputs of '%s' are not valid, so checking is not possible", + worker.store.printStorePath(drvPath)); /* Otherwise, at least one of the output paths could not be produced using a substitute. So we have to build instead. */ /* Make sure checkPathValidity() from now on checks all outputs. */ - wantedOutputs = PathSet(); + wantedOutputs.clear(); /* The inputs must be built before we can build this goal. */ if (useDerivation) @@ -1276,8 +1249,8 @@ void DerivationGoal::outputsSubstituted() for (auto & i : drv->inputSrcs) { if (worker.store.isValidPath(i)) continue; if (!settings.useSubstitutes) - throw Error(format("dependency '%1%' of '%2%' does not exist, and substitution is disabled") - % i % drvPath); + throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); addWaitee(worker.makeSubstitutionGoal(i)); } @@ -1296,7 +1269,7 @@ void DerivationGoal::repairClosure() that produced those outputs. */ /* Get the output closure. */ - PathSet outputClosure; + StorePathSet outputClosure; for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; worker.store.computeFSClosure(i.second.path, outputClosure); @@ -1309,26 +1282,26 @@ void DerivationGoal::repairClosure() /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output closure. */ - PathSet inputClosure; + StorePathSet inputClosure; if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map<Path, Path> outputsToDrv; + std::map<StorePath, StorePath> outputsToDrv; for (auto & i : inputClosure) - if (isDerivation(i)) { + if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv[j.second.path] = i; + outputsToDrv.insert_or_assign(j.second.path.clone(), i.clone()); } /* Check each path (slow!). */ - PathSet broken; for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - printError(format("found corrupted or missing path '%1%' in the output closure of '%2%'") % i % drvPath); - Path drvPath2 = outputsToDrv[i]; - if (drvPath2 == "") + printError("found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); + auto drvPath2 = outputsToDrv.find(i); + if (drvPath2 == outputsToDrv.end()) addWaitee(worker.makeSubstitutionGoal(i, Repair)); else - addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); + addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); } if (waitees.empty()) { @@ -1344,7 +1317,8 @@ void DerivationGoal::closureRepaired() { trace("closure repaired"); if (nrFailed > 0) - throw Error(format("some paths in the output closure of derivation '%1%' could not be repaired") % drvPath); + throw Error("some paths in the output closure of derivation '%s' could not be repaired", + worker.store.printStorePath(drvPath)); done(BuildResult::AlreadyValid); } @@ -1355,10 +1329,9 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) - throw Error(format("some dependencies of '%1%' are missing") % drvPath); - printError( - format("cannot build derivation '%1%': %2% dependencies couldn't be built") - % drvPath % nrFailed); + throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); + printError("cannot build derivation '%s': %s dependencies couldn't be built", + worker.store.printStorePath(drvPath), nrFailed); done(BuildResult::DependencyFailed); return; } @@ -1381,19 +1354,21 @@ void DerivationGoal::inputsRealised() that are specified as inputs. */ assert(worker.store.isValidPath(i.first)); Derivation inDrv = worker.store.derivationFromPath(i.first); - for (auto & j : i.second) - if (inDrv.outputs.find(j) != inDrv.outputs.end()) - worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); + for (auto & j : i.second) { + auto k = inDrv.outputs.find(j); + if (k != inDrv.outputs.end()) + worker.store.computeFSClosure(k->second.path, inputPaths); else throw Error( - format("derivation '%1%' requires non-existent output '%2%' from input derivation '%3%'") - % drvPath % j % i.first); + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(i.first)); + } } /* Second, the input sources. */ worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - debug(format("added input paths %1%") % showPaths(inputPaths)); + debug("added input paths %s", worker.store.showPaths(inputPaths)); /* Is this a fixed-output derivation? */ fixedOutput = drv->isFixedOutput(); @@ -1423,7 +1398,7 @@ void DerivationGoal::tryToBuild() few seconds and then retry this goal. */ PathSet lockFiles; for (auto & outPath : drv->outputPaths()) - lockFiles.insert(worker.store.toRealPath(outPath)); + lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath))); if (!outputLocks.lockPaths(lockFiles, "", false)) { worker.waitForAWhile(shared_from_this()); @@ -1439,23 +1414,22 @@ void DerivationGoal::tryToBuild() build this derivation, so no further checks are necessary. */ validPaths = checkPathValidity(true, buildMode == bmRepair); if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { - debug(format("skipping build of derivation '%1%', someone beat us to it") % drvPath); + debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); done(BuildResult::AlreadyValid); return; } - missingPaths = drv->outputPaths(); + missingPaths = cloneStorePathSet(drv->outputPaths()); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); /* If any of the outputs already exist but are not valid, delete them. */ for (auto & i : drv->outputs) { - Path path = i.second.path; - if (worker.store.isValidPath(path)) continue; - debug(format("removing invalid path '%1%'") % path); - deletePath(worker.store.toRealPath(path)); + if (worker.store.isValidPath(i.second.path)) continue; + debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); + deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path))); } /* Don't do a remote build if the derivation has the attribute @@ -1468,11 +1442,11 @@ void DerivationGoal::tryToBuild() buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmCheck ? "checking outputs of '%s'" : nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", drvPath, curRound, nrRounds); - fmt("building '%s'", drvPath); + "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + fmt("building '%s'", worker.store.printStorePath(drvPath)); if (hook) msg += fmt(" on '%s'", machineName); act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, - Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds}); + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); worker.updateProgress(); }; @@ -1541,7 +1515,7 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) if (pathExists(storePath)) rename(storePath.c_str(), oldPath.c_str()); if (rename(tmpPath.c_str(), storePath.c_str()) == -1) - throw SysError(format("moving '%1%' to '%2%'") % tmpPath % storePath); + throw SysError("moving '%s' to '%s'", tmpPath, storePath); deletePath(oldPath); } @@ -1566,7 +1540,7 @@ void DerivationGoal::buildDone() kill it. */ int status = hook ? hook->pid.kill() : pid.kill(); - debug(format("builder process for '%1%' finished") % drvPath); + debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); result.timesBuilt++; result.stopTime = time(0); @@ -1622,12 +1596,15 @@ void DerivationGoal::buildDone() /* Move paths out of the chroot for easier debugging of build failures. */ if (useChroot && buildMode == bmNormal) - for (auto & i : missingPaths) - if (pathExists(chrootRootDir + i)) - rename((chrootRootDir + i).c_str(), i.c_str()); + for (auto & i : missingPaths) { + auto p = worker.store.printStorePath(i); + if (pathExists(chrootRootDir + p)) + rename((chrootRootDir + p).c_str(), p.c_str()); + } - std::string msg = (format("builder for '%1%' %2%") - % drvPath % statusToString(status)).str(); + auto msg = fmt("builder for '%s' %s", + worker.store.printStorePath(drvPath), + statusToString(status)); if (!settings.verboseBuild && !logTail.empty()) { msg += (format("; last %d log lines:") % logTail.size()).str(); @@ -1648,13 +1625,13 @@ void DerivationGoal::buildDone() if (settings.postBuildHook != "") { Activity act(*logger, lvlInfo, actPostBuildHook, fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{drvPath}); + Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); auto outputPaths = drv->outputPaths(); std::map<std::string, std::string> hookEnvironment = getEnv(); - hookEnvironment.emplace("DRV_PATH", drvPath); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); + hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); RunOptions opts(settings.postBuildHook, {}); opts.environment = hookEnvironment; @@ -1707,7 +1684,7 @@ void DerivationGoal::buildDone() /* Delete unused redirected outputs (when doing hash rewriting). */ for (auto & i : redirectedOutputs) - deletePath(i.second); + deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second))); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ @@ -1772,7 +1749,7 @@ HookReply DerivationGoal::tryBuildHook() << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform - << drvPath + << worker.store.printStorePath(drvPath) << parsedDrv->getRequiredSystemFeatures(); worker.hook->sink.flush(); @@ -1805,7 +1782,7 @@ HookReply DerivationGoal::tryBuildHook() else if (reply == "postpone") return rpPostpone; else if (reply != "accept") - throw Error(format("bad hook reply '%1%'") % reply); + throw Error("bad hook reply '%s'", reply); } catch (SysError & e) { if (e.errNo == EPIPE) { @@ -1823,11 +1800,11 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hook all the inputs that have to be copied to the remote system. */ - hook->sink << inputPaths; + writeStorePaths(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - hook->sink << missingPaths; + writeStorePaths(worker.store, hook->sink, missingPaths); hook->sink = FdSink(); hook->toHook.writeSide = -1; @@ -1844,10 +1821,10 @@ HookReply DerivationGoal::tryBuildHook() } -void chmod_(const Path & path, mode_t mode) +static void chmod_(const Path & path, mode_t mode) { if (chmod(path.c_str(), mode) == -1) - throw SysError(format("setting permissions on '%1%'") % path); + throw SysError("setting permissions on '%s'", path); } @@ -1858,33 +1835,25 @@ int childEntry(void * arg) } -PathSet DerivationGoal::exportReferences(PathSet storePaths) +StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) { - PathSet paths; - - for (auto storePath : storePaths) { - - /* Check that the store path is valid. */ - if (!worker.store.isInStore(storePath)) - throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") - % storePath); - - storePath = worker.store.toStorePath(storePath); + StorePathSet paths; + for (auto & storePath : storePaths) { if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", storePath); + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - worker.store.computeFSClosure(storePath, paths); + worker.store.computeFSClosure(singleton(storePath), paths); } /* If there are derivations in the graph, then include their outputs as well. This is useful if you want to do things like passing all build-time dependencies of some path to a derivation that builds a NixOS DVD image. */ - PathSet paths2(paths); + auto paths2 = cloneStorePathSet(paths); for (auto & j : paths2) { - if (isDerivation(j)) { + if (j.isDerivation()) { Derivation drv = worker.store.derivationFromPath(j); for (auto & k : drv.outputs) worker.store.computeFSClosure(k.second.path, paths); @@ -1932,7 +1901,7 @@ void DerivationGoal::startBuilder() throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", drv->platform, concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - drvPath, + worker.store.printStorePath(drvPath), settings.thisSystem, concatStringsSep<StringSet>(", ", settings.systemFeatures)); @@ -1948,12 +1917,12 @@ void DerivationGoal::startBuilder() auto noChroot = parsedDrv->getBoolAttr("__noChroot"); if (settings.sandboxMode == smEnabled) { if (noChroot) - throw Error(format("derivation '%1%' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'") % drvPath); + throw Error("derivation '%s' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); #if __APPLE__ if (additionalSandboxProfile != "") - throw Error(format("derivation '%1%' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'") % drvPath); + throw Error("derivation '%s' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); #endif useChroot = true; } @@ -1989,14 +1958,13 @@ void DerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - auto drvName = storePathToName(drvPath); - tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); + tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); chownToBuilder(tmpDir); /* Substitute output placeholders with the actual output paths. */ for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = output.second.path; + inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path); /* Construct the environment passed to the builder. */ initEnv(); @@ -2012,19 +1980,22 @@ void DerivationGoal::startBuilder() temporary build directory. The text files have the format used by `nix-store --register-validity'. However, the deriver fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph"); + string s = get(drv->env, "exportReferencesGraph").value_or(""); Strings ss = tokenizeString<Strings>(s); if (ss.size() % 2 != 0) throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); for (Strings::iterator i = ss.begin(); i != ss.end(); ) { string fileName = *i++; - checkStoreName(fileName); /* !!! abuse of this function */ - Path storePath = *i++; + static std::regex regex("[A-Za-z_][A-Za-z0-9_.]*"); + if (!std::regex_match(fileName, regex)) + throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); + + auto storePath = worker.store.parseStorePath(*i++); /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); + exportReferences(singleton(storePath)), false, false)); } } @@ -2054,17 +2025,19 @@ void DerivationGoal::startBuilder() dirsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ - PathSet closure; + StorePathSet closure; for (auto & i : dirsInChroot) try { if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure); + worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); } catch (InvalidPath & e) { } catch (Error & e) { throw Error(format("while processing 'sandbox-paths': %s") % e.what()); } - for (auto & i : closure) - dirsInChroot[i] = i; + for (auto & i : closure) { + auto p = worker.store.printStorePath(i); + dirsInChroot.insert_or_assign(p, p); + } PathSet allowedPaths = settings.allowedImpureHostPrefixes; @@ -2086,7 +2059,8 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation '%1%' requested impure path '%2%', but it was not in allowed-impure-host-deps") % drvPath % i); + throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", + worker.store.printStorePath(drvPath), i); dirsInChroot[i] = i; } @@ -2096,7 +2070,7 @@ void DerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.toRealPath(drvPath) + ".chroot"; + chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot"; deletePath(chrootRootDir); /* Clean up the chroot directory automatically. */ @@ -2155,14 +2129,15 @@ void DerivationGoal::startBuilder() throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir); for (auto & i : inputPaths) { - Path r = worker.store.toRealPath(i); + auto p = worker.store.printStorePath(i); + Path r = worker.store.toRealPath(p); struct stat st; if (lstat(r.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % i); + throw SysError("getting attributes of path '%s'", p); if (S_ISDIR(st.st_mode)) - dirsInChroot[i] = r; + dirsInChroot.insert_or_assign(p, r); else - linkOrCopy(r, chrootRootDir + i); + linkOrCopy(r, chrootRootDir + p); } /* If we're repairing, checking or rebuilding part of a @@ -2171,7 +2146,7 @@ void DerivationGoal::startBuilder() (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputs) - dirsInChroot.erase(i.second.path); + dirsInChroot.erase(worker.store.printStorePath(i.second.path)); #elif __APPLE__ /* We don't really have any parent prep work to do (yet?) @@ -2203,17 +2178,17 @@ void DerivationGoal::startBuilder() corrupt outputs in advance. So rewrite them as well. */ if (buildMode == bmRepair) for (auto & i : missingPaths) - if (worker.store.isValidPath(i) && pathExists(i)) { + if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) { addHashRewrite(i); - redirectedBadOutputs.insert(i); + redirectedBadOutputs.insert(i.clone()); } } if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) { printMsg(lvlChatty, format("executing pre-build hook '%1%'") % settings.preBuildHook); - auto args = useChroot ? Strings({drvPath, chrootRootDir}) : - Strings({ drvPath }); + auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : + Strings({ worker.store.printStorePath(drvPath) }); enum BuildHookState { stBegin, stExtraChrootDirs @@ -2480,7 +2455,7 @@ void DerivationGoal::initTmpDir() { there is no size constraint). */ if (!parsedDrv->getStructuredAttrs()) { - StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile")); + StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or("")); int fileNr = 0; for (auto & i : drv->env) { if (passAsFile.find(i.first) == passAsFile.end()) { @@ -2510,6 +2485,7 @@ void DerivationGoal::initTmpDir() { env["PWD"] = tmpDirInSandbox; } + void DerivationGoal::initEnv() { env.clear(); @@ -2581,7 +2557,7 @@ void DerivationGoal::writeStructuredAttrs() /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites); json["outputs"] = outputs; /* Handle exportReferencesGraph. */ @@ -2591,9 +2567,9 @@ void DerivationGoal::writeStructuredAttrs() std::ostringstream str; { JSONPlaceholder jsonRoot(str, true); - PathSet storePaths; + StorePathSet storePaths; for (auto & p : *i) - storePaths.insert(p.get<std::string>()); + storePaths.insert(worker.store.parseStorePath(p.get<std::string>())); worker.store.pathInfoToJSON(jsonRoot, exportReferences(storePaths), false, true); } @@ -2692,22 +2668,22 @@ struct RestrictedStore : public LocalFSStore std::string getUri() override { return next->getUri(); } - PathSet queryAllValidPaths() override + StorePathSet queryAllValidPaths() override { - PathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); + StorePathSet paths; + for (auto & p : goal.inputPaths) paths.insert(p.clone()); + for (auto & p : goal.addedPaths) paths.insert(p.clone()); return paths; } - void queryPathInfoUncached(const Path & path, + void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override { if (goal.isAllowed(path)) { try { /* Censor impure information. */ auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path)); - info->deriver.clear(); + info->deriver.reset(); info->registrationTime = 0; info->ultimate = false; info->sigs.clear(); @@ -2719,19 +2695,19 @@ struct RestrictedStore : public LocalFSStore callback(nullptr); }; - void queryReferrers(const Path & path, PathSet & referrers) override + void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - PathSet queryDerivationOutputs(const Path & path) override + StorePathSet queryDerivationOutputs(const StorePath & path) override { throw Error("queryDerivationOutputs"); } - StringSet queryDerivationOutputNames(const Path & path) override + StringSet queryDerivationOutputNames(const StorePath & path) override { throw Error("queryDerivationOutputNames"); } - Path queryPathFromHashPart(const string & hashPart) override + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } - Path addToStore(const string & name, const Path & srcPath, + StorePath addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override { throw Error("addToStore"); } @@ -2744,7 +2720,7 @@ struct RestrictedStore : public LocalFSStore goal.addDependency(info.path); } - Path addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(const string & dump, const string & name, bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); @@ -2752,89 +2728,87 @@ struct RestrictedStore : public LocalFSStore return path; } - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair = NoRepair) override + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair = NoRepair) override { auto path = next->addTextToStore(name, s, references, repair); goal.addDependency(path); return path; } - void narFromPath(const Path & path, Sink & sink) override + void narFromPath(const StorePath & path, Sink & sink) override { if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", path); + throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); LocalFSStore::narFromPath(path, sink); } - void ensurePath(const Path & path) override + void ensurePath(const StorePath & path) override { if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", path); + throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); /* Nothing to be done; 'path' must already be valid. */ } - void buildPaths(const PathSet & paths, BuildMode buildMode) override + void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); - PathSet newPaths; + StorePathSet newPaths; for (auto & path : paths) { - DrvPathWithOutputs i = parseDrvPathWithOutputs(path); - if (isDerivation(i.first)) { - if (!goal.isAllowed(i.first)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", i.first); - auto drv = derivationFromPath(i.first); + if (path.path.isDerivation()) { + if (!goal.isAllowed(path.path)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); + auto drv = derivationFromPath(path.path); for (auto & output : drv.outputs) - if (wantOutput(output.first, i.second)) - newPaths.insert(output.second.path); - } else if (!goal.isAllowed(path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", path); + if (wantOutput(output.first, path.outputs)) + newPaths.insert(output.second.path.clone()); + } else if (!goal.isAllowed(path.path)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } next->buildPaths(paths, buildMode); - PathSet closure; + StorePathSet closure; next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) override { unsupported("buildDerivation"); } - void addTempRoot(const Path & path) + void addTempRoot(const StorePath & path) override { } - void addIndirectRoot(const Path & path) + void addIndirectRoot(const Path & path) override { } - Roots findRoots() + Roots findRoots(bool censor) override { return Roots(); } - void collectGarbage(const GCOptions & options, GCResults & results) + void collectGarbage(const GCOptions & options, GCResults & results) override { } - void addSignatures(const Path & storePath, const StringSet & sigs) + void addSignatures(const StorePath & storePath, const StringSet & sigs) override { unsupported("addSignatures"); } - void queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) + void queryMissing(const std::vector<StorePathWithOutputs> & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) override { /* This is slightly impure since it leaks information to the client about what paths will be built/substituted or are already present. Probably not a big deal. */ - PathSet allowed; + std::vector<StorePathWithOutputs> allowed; for (auto & path : targets) { - DrvPathWithOutputs i = parseDrvPathWithOutputs(path); - if (goal.isAllowed(i.first)) - allowed.insert(i.first); + if (goal.isAllowed(path.path)) + allowed.emplace_back(path); else - unknown.insert(i.first); + unknown.insert(path.path.clone()); } next->queryMissing(allowed, willBuild, willSubstitute, @@ -2887,7 +2861,7 @@ void DerivationGoal::startDaemon() debug("received daemon connection"); - auto workerThread = std::thread([this, store, remote{std::move(remote)}]() { + auto workerThread = std::thread([store, remote{std::move(remote)}]() { FdSource from(remote.get()); FdSink to(remote.get()); try { @@ -2925,28 +2899,26 @@ void DerivationGoal::stopDaemon() } -void DerivationGoal::addDependency(const Path & path) +void DerivationGoal::addDependency(const StorePath & path) { - worker.store.assertStorePath(path); - if (isAllowed(path)) return; - addedPaths.insert(path); + addedPaths.insert(path.clone()); /* If we're doing a sandbox build, then we have to make the path appear in the sandbox. */ if (useChroot) { - debug("materialising '%s' in the sandbox", path); + debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); #if __linux__ - Path source = worker.store.toRealPath(path); - Path target = chrootRootDir + path; + Path source = worker.store.toRealPath(worker.store.printStorePath(path)); + Path target = chrootRootDir + worker.store.printStorePath(path); debug("bind-mounting %s -> %s", target, source); if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", path); + throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); struct stat st; if (lstat(source.c_str(), &st)) @@ -2973,13 +2945,14 @@ void DerivationGoal::addDependency(const Path & path) int status = child.wait(); if (status != 0) - throw Error("could not add path '%s' to sandbox", path); + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); } else linkOrCopy(source, target); #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", path); + throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", + worker.store.printStorePath(path)); #endif } @@ -3395,8 +3368,10 @@ void DerivationGoal::runChild() } /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) - dirsInChroot[i] = i; + for (auto & i : inputPaths) { + auto p = worker.store.printStorePath(i); + dirsInChroot[p] = p; + } /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ if (settings.darwinLogSandboxViolations) { @@ -3412,13 +3387,13 @@ void DerivationGoal::runChild() /* Our rwx outputs */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : missingPaths) { - sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); - } + for (auto & i : missingPaths) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i)); + /* Also add redirected outputs to the chroot */ - for (auto & i : redirectedOutputs) { - sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str(); - } + for (auto & i : redirectedOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i.second)); + sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) @@ -3437,19 +3412,19 @@ void DerivationGoal::runChild() if (lstat(path.c_str(), &st)) { if (i.second.optional && errno == ENOENT) continue; - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%s", path); } if (S_ISDIR(st.st_mode)) - sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str(); + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); else - sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str(); + sandboxProfile += fmt("\t(literal \"%s\")\n", path); } sandboxProfile += ")\n"; /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ sandboxProfile += "(allow file-read*\n"; for (auto & i : ancestry) { - sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); + sandboxProfile += fmt("\t(literal \"%s\")\n", i); } sandboxProfile += ")\n"; @@ -3490,8 +3465,7 @@ void DerivationGoal::runChild() #endif else { builder = drv->builder.c_str(); - string builderBasename = baseNameOf(drv->builder); - args.push_back(builderBasename); + args.push_back(std::string(baseNameOf(drv->builder))); } for (auto & i : drv->args) @@ -3505,7 +3479,7 @@ void DerivationGoal::runChild() try { logger = makeJSONLogger(*logger); - BasicDerivation drv2(*drv); + BasicDerivation & drv2(*drv); for (auto & e : drv2.env) e.second = rewriteStrings(e.second, inputRewrites); @@ -3538,16 +3512,15 @@ void DerivationGoal::runChild() /* Parse a list of reference specifiers. Each element must either be a store path, or the symbolic name of the output of the derivation (such as `out'). */ -PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) +StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) { - PathSet result; + StorePathSet result; for (auto & i : paths) { if (store.isStorePath(i)) - result.insert(i); - else if (drv.outputs.find(i) != drv.outputs.end()) - result.insert(drv.outputs.find(i)->second.path); - else throw BuildError( - format("derivation contains an illegal reference specifier '%1%'") % i); + result.insert(store.parseStorePath(i)); + else if (drv.outputs.count(i)) + result.insert(drv.outputs.find(i)->second.path.clone()); + else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; } @@ -3580,19 +3553,17 @@ void DerivationGoal::registerOutputs() /* The paths that can be referenced are the input closures, the output paths, and any paths that have been built via recursive Nix calls. */ - PathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); - for (auto & p : addedPaths) referenceablePaths.insert(p); + StorePathSet referenceablePaths; + for (auto & p : inputPaths) referenceablePaths.insert(p.clone()); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path.clone()); + for (auto & p : addedPaths) referenceablePaths.insert(p.clone()); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ for (auto & i : drv->outputs) { - Path path = i.second.path; - if (missingPaths.find(path) == missingPaths.end()) continue; - - ValidPathInfo info; + auto path = worker.store.printStorePath(i.second.path); + if (!missingPaths.count(i.second.path)) continue; Path actualPath = path; if (useChroot) { @@ -3609,22 +3580,25 @@ void DerivationGoal::registerOutputs() } if (needsHashRewrite()) { - Path redirected = redirectedOutputs[path]; - if (buildMode == bmRepair - && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() - && pathExists(redirected)) - replaceValidPath(path, redirected); - if (buildMode == bmCheck && redirected != "") - actualPath = redirected; + auto r = redirectedOutputs.find(i.second.path); + if (r != redirectedOutputs.end()) { + auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second)); + if (buildMode == bmRepair + && redirectedBadOutputs.count(i.second.path) + && pathExists(redirected)) + replaceValidPath(path, redirected); + if (buildMode == bmCheck) + actualPath = redirected; + } } struct stat st; if (lstat(actualPath.c_str(), &st) == -1) { if (errno == ENOENT) throw BuildError( - format("builder for '%1%' failed to produce output path '%2%'") - % drvPath % path); - throw SysError(format("getting attributes of path '%1%'") % actualPath); + "builder for '%s' failed to produce output path '%s'", + worker.store.printStorePath(drvPath), path); + throw SysError("getting attributes of path '%s'", actualPath); } #ifndef __CYGWIN__ @@ -3661,6 +3635,8 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ + std::string ca; + if (fixedOutput) { bool recursive; Hash h; @@ -3677,7 +3653,7 @@ void DerivationGoal::registerOutputs() the derivation to its content-addressed location. */ Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); - Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path)); + auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); if (h != h2) { @@ -3686,9 +3662,9 @@ void DerivationGoal::registerOutputs() worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - dest, h.to_string(SRI), h2.to_string(SRI))); + worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); - Path actualDest = worker.store.toRealPath(dest); + Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest)); if (worker.store.isValidPath(dest)) std::rethrow_exception(delayedException); @@ -3697,16 +3673,16 @@ void DerivationGoal::registerOutputs() PathLocks outputLocks({actualDest}); deletePath(actualDest); if (rename(actualPath.c_str(), actualDest.c_str()) == -1) - throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest); + throw SysError("moving '%s' to '%s'", actualPath, worker.store.printStorePath(dest)); } - path = dest; + path = worker.store.printStorePath(dest); actualPath = actualDest; } else - assert(path == dest); + assert(worker.store.parseStorePath(path) == dest); - info.ca = makeFixedOutputCA(recursive, h2); + ca = makeFixedOutputCA(recursive, h2); } /* Get rid of all weird permissions. This also checks that @@ -3720,11 +3696,11 @@ void DerivationGoal::registerOutputs() verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); HashResult hash; - PathSet references = scanForReferences(actualPath, referenceablePaths, hash); + auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); if (buildMode == bmCheck) { - if (!worker.store.isValidPath(path)) continue; - auto info = *worker.store.queryPathInfo(path); + if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; + ValidPathInfo info(*worker.store.queryPathInfo(worker.store.parseStorePath(path))); if (hash.first != info.narHash) { worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { @@ -3736,20 +3712,22 @@ void DerivationGoal::registerOutputs() handleDiffHook( buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), - path, dst, drvPath, tmpDir); + path, dst, worker.store.printStorePath(drvPath), tmpDir); - throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") - % drvPath % path % dst); + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", + worker.store.printStorePath(drvPath), path, dst); } else - throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs") - % drvPath % path); + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", + worker.store.printStorePath(drvPath), path); } /* Since we verified the build, it's now ultimately trusted. */ if (!info.ultimate) { info.ultimate = true; worker.store.signPathInfo(info); - worker.store.registerValidPaths({info}); + ValidPathInfos infos; + infos.push_back(std::move(info)); + worker.store.registerValidPaths(infos); } continue; @@ -3757,29 +3735,30 @@ void DerivationGoal::registerOutputs() /* For debugging, print out the referenced and unreferenced paths. */ for (auto & i : inputPaths) { - PathSet::iterator j = references.find(i); + auto j = references.find(i); if (j == references.end()) - debug(format("unreferenced input: '%1%'") % i); + debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); else - debug(format("referenced input: '%1%'") % i); + debug("referenced input: '%1%'", worker.store.printStorePath(i)); } if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(path); + worker.markContentsGood(worker.store.parseStorePath(path)); } - info.path = path; + ValidPathInfo info(worker.store.parseStorePath(path)); info.narHash = hash.first; info.narSize = hash.second; - info.references = references; - info.deriver = drvPath; + info.references = std::move(references); + info.deriver = drvPath.clone(); info.ultimate = true; + info.ca = ca; worker.store.signPathInfo(info); if (!info.references.empty()) info.ca.clear(); - infos[i.first] = info; + infos.emplace(i.first, std::move(info)); } if (buildMode == bmCheck) return; @@ -3794,16 +3773,19 @@ void DerivationGoal::registerOutputs() for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { result.isNonDeterministic = true; - Path prev = i->second.path + checkSuffix; + Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); auto msg = prevExists - ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) - : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); + ? fmt("output '%s' of '%s' differs from '%s' from previous round", + worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) + : fmt("output '%s' of '%s' differs from previous round", + worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), - prev, i->second.path, drvPath, tmpDir); + prev, worker.store.printStorePath(i->second.path), + worker.store.printStorePath(drvPath), tmpDir); if (settings.enforceDeterminism) throw NotDeterministic(msg); @@ -3816,16 +3798,17 @@ void DerivationGoal::registerOutputs() /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { - Path prev = i.second.path + checkSuffix; + auto path = worker.store.printStorePath(i.second.path); + Path prev = path + checkSuffix; deletePath(prev); - Path dst = i.second.path + checkSuffix; - if (rename(i.second.path.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % i.second.path % dst); + Path dst = path + checkSuffix; + if (rename(path.c_str(), dst.c_str())) + throw SysError("renaming '%s' to '%s'", path, dst); } } if (curRound < nrRounds) { - prevInfos = infos; + prevInfos = std::move(infos); return; } @@ -3833,7 +3816,7 @@ void DerivationGoal::registerOutputs() if the result was not determistic? */ if (curRound == nrRounds) { for (auto & i : drv->outputs) { - Path prev = i.second.path + checkSuffix; + Path prev = worker.store.printStorePath(i.second.path) + checkSuffix; deletePath(prev); } } @@ -3858,7 +3841,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) { std::map<Path, const ValidPathInfo &> outputsByPath; for (auto & output : outputs) - outputsByPath.emplace(output.second.path, output.second); + outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); for (auto & output : outputs) { auto & outputName = output.first; @@ -3874,76 +3857,77 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) /* Compute the closure and closure size of some output. This is slightly tricky because some of its references (namely other outputs) may not be valid yet. */ - auto getClosure = [&](const Path & path) + auto getClosure = [&](const StorePath & path) { uint64_t closureSize = 0; - PathSet pathsDone; - std::queue<Path> pathsLeft; - pathsLeft.push(path); + StorePathSet pathsDone; + std::queue<StorePath> pathsLeft; + pathsLeft.push(path.clone()); while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); + auto path = pathsLeft.front().clone(); pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; + if (!pathsDone.insert(path.clone()).second) continue; - auto i = outputsByPath.find(path); + auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; for (auto & ref : i->second.references) - pathsLeft.push(ref); + pathsLeft.push(ref.clone()); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; for (auto & ref : info->references) - pathsLeft.push(ref); + pathsLeft.push(ref.clone()); } } - return std::make_pair(pathsDone, closureSize); + return std::make_pair(std::move(pathsDone), closureSize); }; auto applyChecks = [&](const Checks & checks) { if (checks.maxSize && info.narSize > *checks.maxSize) throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - info.path, info.narSize, *checks.maxSize); + worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); if (checks.maxClosureSize) { uint64_t closureSize = getClosure(info.path).second; if (closureSize > *checks.maxClosureSize) throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - info.path, closureSize, *checks.maxClosureSize); + worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); } auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive) { if (!value) return; - PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); + auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); - PathSet used = recursive ? getClosure(info.path).first : info.references; + auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); - PathSet badPaths; + StorePathSet badPaths; for (auto & i : used) if (allowed) { if (!spec.count(i)) - badPaths.insert(i); + badPaths.insert(i.clone()); } else { if (spec.count(i)) - badPaths.insert(i); + badPaths.insert(i.clone()); } if (!badPaths.empty()) { string badPathsStr; for (auto & i : badPaths) { badPathsStr += "\n "; - badPathsStr += i; + badPathsStr += worker.store.printStorePath(i); } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr); + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", + worker.store.printStorePath(info.path), badPathsStr); } }; @@ -3975,7 +3959,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) Strings res; for (auto j = i->begin(); j != i->end(); ++j) { if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); res.push_back(j->get<std::string>()); } checks.disallowedRequisites = res; @@ -4012,7 +3996,7 @@ Path DerivationGoal::openLogFile() if (!settings.keepLog) return ""; - string baseName = baseNameOf(drvPath); + auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); /* Create a log file. */ Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); @@ -4051,9 +4035,7 @@ void DerivationGoal::deleteTmpDir(bool force) /* Don't keep temporary directories for builtins because they might have privileged stuff (like a copy of netrc). */ if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError( - format("note: keeping build directory '%2%'") - % drvPath % tmpDir); + printError("note: keeping build directory '%s'", tmpDir); chmod(tmpDir.c_str(), 0755); } else @@ -4132,31 +4114,31 @@ void DerivationGoal::flushLine() } -PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) +StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { - PathSet result; + StorePathSet result; for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; bool good = worker.store.isValidPath(i.second.path) && (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path); + if (good == returnValid) result.insert(i.second.path.clone()); } return result; } -Path DerivationGoal::addHashRewrite(const Path & path) +void DerivationGoal::addHashRewrite(const StorePath & path) { - string h1 = string(path, worker.store.storeDir.size() + 1, 32); - string h2 = string(hashString(htSHA256, "rewrite:" + drvPath + ":" + path).to_string(Base32, false), 0, 32); - Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33); - deletePath(p); - assert(path.size() == p.size()); + auto h1 = std::string(((std::string_view) path.to_string()).substr(0, 32)); + auto p = worker.store.makeStorePath( + "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), + Hash(htSHA256), path.name()); + auto h2 = std::string(((std::string_view) p.to_string()).substr(0, 32)); + deletePath(worker.store.printStorePath(p)); inputRewrites[h1] = h2; outputRewrites[h2] = h1; - redirectedOutputs[path] = p; - return p; + redirectedOutputs.insert_or_assign(path.clone(), std::move(p)); } @@ -4194,7 +4176,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ - Path storePath; + StorePath storePath; /* The remaining substituters. */ std::list<ref<Store>> subs; @@ -4230,7 +4212,7 @@ private: GoalState state; public: - SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair); ~SubstitutionGoal(); void timedOut() override { abort(); }; @@ -4239,7 +4221,7 @@ public: { /* "a$" ensures substitution goals happen before derivation goals. */ - return "a$" + storePathToName(storePath) + "$" + storePath; + return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } void work() override; @@ -4256,7 +4238,7 @@ public: void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; - Path getStorePath() { return storePath; } + StorePath getStorePath() { return storePath.clone(); } void amDone(ExitCode result) override { @@ -4265,13 +4247,13 @@ public: }; -SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair) : Goal(worker) + , storePath(std::move(storePath)) , repair(repair) { - this->storePath = storePath; state = &SubstitutionGoal::init; - name = (format("substitution of '%1%'") % storePath).str(); + name = fmt("substitution of '%s'", worker.store.printStorePath(storePath)); trace("created"); maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions); } @@ -4310,7 +4292,7 @@ void SubstitutionGoal::init() } if (settings.readOnlyMode) - throw Error(format("cannot substitute path '%1%' - no write access to the Nix store") % storePath); + throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>(); @@ -4325,7 +4307,7 @@ void SubstitutionGoal::tryNext() if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ - debug(format("path '%1%' is required, but there is no substituter that can build it") % storePath); + debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a @@ -4389,7 +4371,7 @@ void SubstitutionGoal::tryNext() && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { printError("warning: substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), storePath); + sub->getUri(), worker.store.printStorePath(storePath)); tryNext(); return; } @@ -4412,7 +4394,7 @@ void SubstitutionGoal::referencesValid() trace("all references realised"); if (nrFailed > 0) { - debug(format("some references of path '%1%' could not be realised") % storePath); + debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); return; } @@ -4451,7 +4433,7 @@ void SubstitutionGoal::tryToRun() /* Wake up the worker loop when we're done. */ Finally updateStats([this]() { outPipe.writeSide = -1; }); - Activity act(*logger, actSubstitute, Logger::Fields{storePath, sub->getUri()}); + Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), @@ -4498,10 +4480,9 @@ void SubstitutionGoal::finished() return; } - worker.markContentsGood(storePath); + worker.markContentsGood(storePath.clone()); - printMsg(lvlChatty, - format("substitution of path '%1%' succeeded") % storePath); + printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); maintainRunningSubstitutions.reset(); @@ -4567,13 +4548,13 @@ Worker::~Worker() } -GoalPtr Worker::makeDerivationGoal(const Path & path, +GoalPtr Worker::makeDerivationGoal(const StorePath & path, const StringSet & wantedOutputs, BuildMode buildMode) { - GoalPtr goal = derivationGoals[path].lock(); + GoalPtr goal = derivationGoals[path.clone()].lock(); // FIXME if (!goal) { - goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); - derivationGoals[path] = goal; + goal = std::make_shared<DerivationGoal>(path.clone(), wantedOutputs, *this, buildMode); + derivationGoals.insert_or_assign(path.clone(), goal); wakeUp(goal); } else (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); @@ -4581,21 +4562,21 @@ GoalPtr Worker::makeDerivationGoal(const Path & path, } -std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const Path & drvPath, +std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(StorePath && drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); + auto goal = std::make_shared<DerivationGoal>(std::move(drvPath), drv, *this, buildMode); wakeUp(goal); return goal; } -GoalPtr Worker::makeSubstitutionGoal(const Path & path, RepairFlag repair) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) { - GoalPtr goal = substitutionGoals[path].lock(); + GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME if (!goal) { - goal = std::make_shared<SubstitutionGoal>(path, *this, repair); - substitutionGoals[path] = goal; + goal = std::make_shared<SubstitutionGoal>(path.clone(), *this, repair); + substitutionGoals.insert_or_assign(path.clone(), goal); wakeUp(goal); } return goal; @@ -4934,38 +4915,38 @@ unsigned int Worker::exitStatus() } -bool Worker::pathContentsGood(const Path & path) +bool Worker::pathContentsGood(const StorePath & path) { - std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); + auto i = pathContentsGoodCache.find(path); if (i != pathContentsGoodCache.end()) return i->second; - printInfo(format("checking path '%1%'...") % path); + printInfo("checking path '%s'...", store.printStorePath(path)); auto info = store.queryPathInfo(path); bool res; - if (!pathExists(path)) + if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.type, path); + HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } - pathContentsGoodCache[path] = res; - if (!res) printError(format("path '%1%' is corrupted or missing!") % path); + pathContentsGoodCache.insert_or_assign(path.clone(), res); + if (!res) printError("path '%s' is corrupted or missing!", store.printStorePath(path)); return res; } -void Worker::markContentsGood(const Path & path) +void Worker::markContentsGood(StorePath && path) { - pathContentsGoodCache[path] = true; + pathContentsGoodCache.insert_or_assign(std::move(path), true); } ////////////////////////////////////////////////////////////////////// -static void primeCache(Store & store, const PathSet & paths) +static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths) { - PathSet willBuild, willSubstitute, unknown; + StorePathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -4976,24 +4957,23 @@ static void primeCache(Store & store, const PathSet & paths) } -void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) +void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) { Worker worker(*this); primeCache(*this, drvPaths); Goals goals; - for (auto & i : drvPaths) { - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); - if (isDerivation(i2.first)) - goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode)); + for (auto & path : drvPaths) { + if (path.path.isDerivation()) + goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); else - goals.insert(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); + goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); } worker.run(goals); - PathSet failed; + StorePathSet failed; for (auto & i : goals) { if (i->getExitCode() != Goal::ecSuccess) { DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); @@ -5007,11 +4987,11 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) } -BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, +BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath.clone(), drv, buildMode); BuildResult result; @@ -5027,12 +5007,12 @@ BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivat } -void LocalStore::ensurePath(const Path & path) +void LocalStore::ensurePath(const StorePath & path) { /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {path}); + primeCache(*this, {StorePathWithOutputs(path)}); Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); @@ -5041,11 +5021,11 @@ void LocalStore::ensurePath(const Path & path) worker.run(goals); if (goal->getExitCode() != Goal::ecSuccess) - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", path); + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } -void LocalStore::repairPath(const Path & path) +void LocalStore::repairPath(const StorePath & path) { Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); @@ -5056,13 +5036,13 @@ void LocalStore::repairPath(const Path & path) if (goal->getExitCode() != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ - auto deriver = queryPathInfo(path)->deriver; - if (deriver != "" && isValidPath(deriver)) { + auto info = queryPathInfo(path); + if (info->deriver && isValidPath(*info->deriver)) { goals.clear(); - goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); + goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); worker.run(goals); } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", path); + throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); } } diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0a37459b0..73c0f5f7f 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -9,7 +9,7 @@ struct Package { Path path; bool active; int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} + Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; typedef std::vector<Package> Packages; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index b1af3b4fc..f6ae5d2e6 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -24,7 +24,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) Path storePath = getAttr("out"); auto mainUrl = getAttr("url"); - bool unpack = get(drv.env, "unpack", "") == "1"; + bool unpack = get(drv.env, "unpack").value_or("") == "1"; /* Note: have to use a fresh downloader here because we're in a forked process. */ diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d3c17e772..c53dfdd56 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -260,14 +260,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, switch (op) { case wopIsValidPath: { - /* 'readStorePath' could raise an error leading to the connection - being closed. To be able to recover from an invalid path error, - call 'startWork' early, and do 'assertStorePath' afterwards so - that the 'Error' exception handler doesn't close the - connection. */ - Path path = readString(from); - logger->startWork(); - store->assertStorePath(path); + auto path = store->parseStorePath(readString(from)); + logger->startWork(); bool result = store->isValidPath(path); logger->stopWork(); to << result; @@ -275,34 +269,36 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQueryValidPaths: { - PathSet paths = readStorePaths<PathSet>(*store, from); + auto paths = readStorePaths<StorePathSet>(*store, from); logger->startWork(); - PathSet res = store->queryValidPaths(paths); + auto res = store->queryValidPaths(paths); logger->stopWork(); - to << res; + writeStorePaths(*store, to, res); break; } case wopHasSubstitutes: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); - PathSet res = store->querySubstitutablePaths({path}); + StorePathSet paths; // FIXME + paths.insert(path.clone()); + auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - to << (res.find(path) != res.end()); + to << (res.count(path) != 0); break; } case wopQuerySubstitutablePaths: { - PathSet paths = readStorePaths<PathSet>(*store, from); + auto paths = readStorePaths<StorePathSet>(*store, from); logger->startWork(); - PathSet res = store->querySubstitutablePaths(paths); + auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - to << res; + writeStorePaths(*store, to, res); break; } case wopQueryPathHash: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); @@ -314,23 +310,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopQueryReferrers: case wopQueryValidDerivers: case wopQueryDerivationOutputs: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); - PathSet paths; + StorePathSet paths; if (op == wopQueryReferences) - paths = store->queryPathInfo(path)->references; + for (auto & i : store->queryPathInfo(path)->references) + paths.insert(i.clone()); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); else if (op == wopQueryValidDerivers) paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - to << paths; + writeStorePaths(*store, to, paths); break; } case wopQueryDerivationOutputNames: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); StringSet names; names = store->queryDerivationOutputNames(path); @@ -340,20 +337,20 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQueryDeriver: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); - auto deriver = store->queryPathInfo(path)->deriver; + auto info = store->queryPathInfo(path); logger->stopWork(); - to << deriver; + to << (info->deriver ? store->printStorePath(*info->deriver) : ""); break; } case wopQueryPathFromHashPart: { - string hashPart = readString(from); + auto hashPart = readString(from); logger->startWork(); - Path path = store->queryPathFromHashPart(hashPart); + auto path = store->queryPathFromHashPart(hashPart); logger->stopWork(); - to << path; + to << (path ? store->printStorePath(*path) : ""); break; } @@ -383,26 +380,26 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); logger->stopWork(); - to << path; + to << store->printStorePath(path); break; } case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - PathSet refs = readStorePaths<PathSet>(*store, from); + auto refs = readStorePaths<StorePathSet>(*store, from); logger->startWork(); - Path path = store->addTextToStore(suffix, s, refs, NoRepair); + auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); - to << path; + to << store->printStorePath(path); break; } case wopExportPath: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); readInt(from); // obsolete logger->startWork(); TunnelSink sink(to); @@ -415,15 +412,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopImportPaths: { logger->startWork(); TunnelSource source(from, to); - Paths paths = store->importPaths(source, nullptr, + auto paths = store->importPaths(source, nullptr, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); - to << paths; + Strings paths2; + for (auto & i : paths) paths2.push_back(store->printStorePath(i)); + to << paths2; break; } case wopBuildPaths: { - PathSet drvs = readStorePaths<PathSet>(*store, from); + std::vector<StorePathWithOutputs> drvs; + for (auto & s : readStrings<Strings>(from)) + drvs.push_back(store->parseDrvPathWithOutputs(s)); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -441,7 +442,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopBuildDerivation: { - Path drvPath = readStorePath(*store, from); + auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; readDerivation(from, *store, drv); BuildMode buildMode = (BuildMode) readInt(from); @@ -455,7 +456,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopEnsurePath: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); store->ensurePath(path); logger->stopWork(); @@ -464,7 +465,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddTempRoot: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); store->addTempRoot(path); logger->stopWork(); @@ -502,7 +503,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, for (auto & [target, links] : roots) for (auto & link : links) - to << link << target; + to << link << store->printStorePath(target); break; } @@ -510,7 +511,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths<PathSet>(*store, from); + options.pathsToDelete = readStorePaths<StorePathSet>(*store, from); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -568,44 +569,52 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQuerySubstitutablePathInfo: { - Path path = absPath(readString(from)); + auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); + StorePathSet paths; + paths.insert(path.clone()); // FIXME + store->querySubstitutablePathInfos(paths, infos); logger->stopWork(); - SubstitutablePathInfos::iterator i = infos.find(path); + auto i = infos.find(path); if (i == infos.end()) to << 0; else { - to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize; + to << 1 + << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); + writeStorePaths(*store, to, i->second.references); + to << i->second.downloadSize + << i->second.narSize; } break; } case wopQuerySubstitutablePathInfos: { - PathSet paths = readStorePaths<PathSet>(*store, from); + auto paths = readStorePaths<StorePathSet>(*store, from); logger->startWork(); SubstitutablePathInfos infos; store->querySubstitutablePathInfos(paths, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { - to << i.first << i.second.deriver << i.second.references - << i.second.downloadSize << i.second.narSize; + to << store->printStorePath(i.first) + << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); + writeStorePaths(*store, to, i.second.references); + to << i.second.downloadSize << i.second.narSize; } break; } case wopQueryAllValidPaths: { logger->startWork(); - PathSet paths = store->queryAllValidPaths(); + auto paths = store->queryAllValidPaths(); logger->stopWork(); - to << paths; + writeStorePaths(*store, to, paths); break; } case wopQueryPathInfo: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); std::shared_ptr<const ValidPathInfo> info; logger->startWork(); try { @@ -617,8 +626,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - to << info->deriver << info->narHash.to_string(Base16, false) << info->references - << info->registrationTime << info->narSize; + to << (info->deriver ? store->printStorePath(*info->deriver) : "") + << info->narHash.to_string(Base16, false); + writeStorePaths(*store, to, info->references); + to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs @@ -651,7 +662,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddSignatures: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); StringSet sigs = readStrings<StringSet>(from); logger->startWork(); if (!trusted) @@ -663,22 +674,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopNarFromPath: { - auto path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); logger->stopWork(); - dumpPath(path, to); + dumpPath(store->printStorePath(path), to); break; } case wopAddToStoreNar: { bool repair, dontCheckSigs; - ValidPathInfo info; - info.path = readStorePath(*store, from); - from >> info.deriver; - if (!info.deriver.empty()) - store->assertStorePath(info.deriver); + ValidPathInfo info(store->parseStorePath(readString(from))); + auto deriver = readString(from); + if (deriver != "") + info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths<PathSet>(*store, from); + info.references = readStorePaths<StorePathSet>(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); from >> info.ca >> repair >> dontCheckSigs; @@ -709,13 +719,18 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQueryMissing: { - PathSet targets = readStorePaths<PathSet>(*store, from); + std::vector<StorePathWithOutputs> targets; + for (auto & s : readStrings<Strings>(from)) + targets.push_back(store->parseDrvPathWithOutputs(s)); logger->startWork(); - PathSet willBuild, willSubstitute, unknown; + StorePathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - to << willBuild << willSubstitute << unknown << downloadSize << narSize; + writeStorePaths(*store, to, willBuild); + writeStorePaths(*store, to, willSubstitute); + writeStorePaths(*store, to, unknown); + to << downloadSize << narSize; break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 23fcfb281..726e34479 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -21,17 +21,39 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const HashType hashType = parseHashType(algo); if (hashType == htUnknown) - throw Error(format("unknown hash algorithm '%1%'") % algo); + throw Error("unknown hash algorithm '%s'", algo); hash = Hash(this->hash, hashType); } -Path BasicDerivation::findOutput(const string & id) const +BasicDerivation::BasicDerivation(const BasicDerivation & other) + : platform(other.platform) + , builder(other.builder) + , args(other.args) + , env(other.env) +{ + for (auto & i : other.outputs) + outputs.insert_or_assign(i.first, + DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); + for (auto & i : other.inputSrcs) + inputSrcs.insert(i.clone()); +} + + +Derivation::Derivation(const Derivation & other) + : BasicDerivation(other) +{ + for (auto & i : other.inputDrvs) + inputDrvs.insert_or_assign(i.first.clone(), i.second); +} + + +const StorePath & BasicDerivation::findOutput(const string & id) const { auto i = outputs.find(id); if (i == outputs.end()) - throw Error(format("derivation has no output '%1%'") % id); + throw Error("derivation has no output '%s'", id); return i->second.path; } @@ -42,18 +64,17 @@ bool BasicDerivation::isBuiltin() const } -Path writeDerivation(ref<Store> store, +StorePath writeDerivation(ref<Store> store, const Derivation & drv, const string & name, RepairFlag repair) { - PathSet references; - references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); + auto references = cloneStorePathSet(drv.inputSrcs); for (auto & i : drv.inputDrvs) - references.insert(i.first); + references.insert(i.first.clone()); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ string suffix = name + drvExtension; - string contents = drv.unparse(); + string contents = drv.unparse(*store, false); return settings.readOnlyMode ? store->computeStorePathForText(suffix, contents, references) : store->addTextToStore(suffix, contents, references, repair); @@ -121,7 +142,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } -static Derivation parseDerivation(const string & s) +static Derivation parseDerivation(const Store & store, const string & s) { Derivation drv; istringstream_nocopy str(s); @@ -129,13 +150,12 @@ static Derivation parseDerivation(const string & s) /* Parse the list of outputs. */ while (!endOfList(str)) { - DerivationOutput out; - expect(str, "("); string id = parseString(str); - expect(str, ","); out.path = parsePath(str); - expect(str, ","); out.hashAlgo = parseString(str); - expect(str, ","); out.hash = parseString(str); + expect(str, "("); std::string id = parseString(str); + expect(str, ","); auto path = store.parseStorePath(parsePath(str)); + expect(str, ","); auto hashAlgo = parseString(str); + expect(str, ","); auto hash = parseString(str); expect(str, ")"); - drv.outputs[id] = out; + drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); } /* Parse the list of input derivations. */ @@ -144,11 +164,11 @@ static Derivation parseDerivation(const string & s) expect(str, "("); Path drvPath = parsePath(str); expect(str, ",["); - drv.inputDrvs[drvPath] = parseStrings(str, false); + drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false)); expect(str, ")"); } - expect(str, ",["); drv.inputSrcs = parseStrings(str, true); + expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); expect(str, ","); drv.platform = parseString(str); expect(str, ","); drv.builder = parseString(str); @@ -171,25 +191,24 @@ static Derivation parseDerivation(const string & s) } -Derivation readDerivation(const Path & drvPath) +Derivation readDerivation(const Store & store, const Path & drvPath) { try { - return parseDerivation(readFile(drvPath)); + return parseDerivation(store, readFile(drvPath)); } catch (FormatError & e) { throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); } } -Derivation Store::derivationFromPath(const Path & drvPath) +Derivation Store::derivationFromPath(const StorePath & drvPath) { - assertStorePath(drvPath); ensurePath(drvPath); auto accessor = getFSAccessor(); try { - return parseDerivation(accessor->readFile(drvPath)); + return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); } catch (FormatError & e) { - throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); + throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); } } @@ -220,33 +239,56 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j) } -string Derivation::unparse() const +string Derivation::unparse(const Store & store, bool maskOutputs, + std::map<std::string, StringSet> * actualInputs) const { string s; s.reserve(65536); s += "Derive(["; - bool first = true; - for (auto & i : outputs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i.first); - s += ','; printString(s, i.second.path); - s += ','; printString(s, i.second.hashAlgo); - s += ','; printString(s, i.second.hash); - s += ')'; + StringSet maskedOutputs; + + if (maskOutputs) { + bool first = true; + maskedOutputs = tokenizeString<StringSet>(get(env, "outputs").value_or("out"), " "); + for (auto & i : maskedOutputs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i); + s += ",\"\",\"\",\"\")"; + } + } else { + bool first = true; + for (auto & i : outputs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i.first); + s += ','; printString(s, store.printStorePath(i.second.path)); + s += ','; printString(s, i.second.hashAlgo); + s += ','; printString(s, i.second.hash); + s += ')'; + } } s += "],["; - first = true; - for (auto & i : inputDrvs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i.first); - s += ','; printStrings(s, i.second.begin(), i.second.end()); - s += ')'; + bool first = true; + if (actualInputs) { + for (auto & i : *actualInputs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i.first); + s += ','; printStrings(s, i.second.begin(), i.second.end()); + s += ')'; + } + } else { + for (auto & i : inputDrvs) { + if (first) first = false; else s += ','; + s += '('; printString(s, store.printStorePath(i.first)); + s += ','; printStrings(s, i.second.begin(), i.second.end()); + s += ')'; + } } s += "],"; - printStrings(s, inputSrcs.begin(), inputSrcs.end()); + auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow + printStrings(s, paths.begin(), paths.end()); s += ','; printString(s, platform); s += ','; printString(s, builder); @@ -257,7 +299,7 @@ string Derivation::unparse() const for (auto & i : env) { if (first) first = false; else s += ','; s += '('; printString(s, i.first); - s += ','; printString(s, i.second); + s += ','; printString(s, maskOutputs && maskedOutputs.count(i.first) ? "" : i.second); s += ')'; } @@ -267,6 +309,7 @@ string Derivation::unparse() const } +// FIXME: remove bool isDerivation(const string & fileName) { return hasSuffix(fileName, drvExtension); @@ -304,7 +347,7 @@ DrvHashes drvHashes; paths have been replaced by the result of a recursive call to this function, and that for fixed-output derivations we return a hash of its output path. */ -Hash hashDerivationModulo(Store & store, Derivation drv) +Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { @@ -312,42 +355,41 @@ Hash hashDerivationModulo(Store & store, Derivation drv) return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" + i->second.hash + ":" - + i->second.path); + + store.printStorePath(i->second.path)); } /* For other derivations, replace the inputs paths with recursive calls to this function.*/ - DerivationInputs inputs2; + std::map<std::string, StringSet> inputs2; for (auto & i : drv.inputDrvs) { - Hash h = drvHashes[i.first]; - if (!h) { + auto h = drvHashes.find(i.first); + if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); - Derivation drv2 = readDerivation(store.toRealPath(i.first)); - h = hashDerivationModulo(store, drv2); - drvHashes[i.first] = h; + h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, + readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first; } - inputs2[h.to_string(Base16, false)] = i.second; + inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } - drv.inputDrvs = inputs2; - return hashString(htSHA256, drv.unparse()); + return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); } -DrvPathWithOutputs parseDrvPathWithOutputs(const string & s) +StorePathWithOutputs Store::parseDrvPathWithOutputs(const std::string & s) { size_t n = s.find("!"); return n == s.npos - ? DrvPathWithOutputs(s, std::set<string>()) - : DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ",")); + ? StorePathWithOutputs{parseStorePath(s), std::set<string>()} + : StorePathWithOutputs{parseStorePath(std::string_view(s.data(), n)), + tokenizeString<std::set<string>>(string(s, n + 1), ",")}; } -Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs) +std::string StorePathWithOutputs::to_string(const Store & store) const { return outputs.empty() - ? drvPath - : drvPath + "!" + concatStringsSep(",", outputs); + ? store.printStorePath(path) + : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); } @@ -357,28 +399,28 @@ bool wantOutput(const string & output, const std::set<string> & wanted) } -PathSet BasicDerivation::outputPaths() const +StorePathSet BasicDerivation::outputPaths() const { - PathSet paths; + StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path); + paths.insert(i.second.path.clone()); return paths; } -Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) { drv.outputs.clear(); auto nr = readNum<size_t>(in); for (size_t n = 0; n < nr; n++) { auto name = readString(in); - DerivationOutput o; - in >> o.path >> o.hashAlgo >> o.hash; - store.assertStorePath(o.path); - drv.outputs[name] = o; + auto path = store.parseStorePath(readString(in)); + auto hashAlgo = readString(in); + auto hash = readString(in); + drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); } - drv.inputSrcs = readStorePaths<PathSet>(store, in); + drv.inputSrcs = readStorePaths<StorePathSet>(store, in); in >> drv.platform >> drv.builder; drv.args = readStrings<Strings>(in); @@ -393,16 +435,16 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) } -Sink & operator << (Sink & out, const BasicDerivation & drv) +void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) { out << drv.outputs.size(); for (auto & i : drv.outputs) - out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; - out << drv.inputSrcs << drv.platform << drv.builder << drv.args; + out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash; + writeStorePaths(store, out, drv.inputSrcs); + out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) out << i.first << i.second; - return out; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 8e02c9bc5..c2df66229 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -10,26 +10,18 @@ namespace nix { -/* Extension of derivations in the Nix store. */ -const string drvExtension = ".drv"; - - /* Abstract syntax of derivations. */ struct DerivationOutput { - Path path; - string hashAlgo; /* hash used for expected hash computation */ - string hash; /* expected hash, may be null */ - DerivationOutput() - { - } - DerivationOutput(Path path, string hashAlgo, string hash) - { - this->path = path; - this->hashAlgo = hashAlgo; - this->hash = hash; - } + StorePath path; + std::string hashAlgo; /* hash used for expected hash computation */ + std::string hash; /* expected hash, may be null */ + DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash) + : path(std::move(path)) + , hashAlgo(std::move(hashAlgo)) + , hash(std::move(hash)) + { } void parseHashInfo(bool & recursive, Hash & hash) const; }; @@ -37,24 +29,26 @@ typedef std::map<string, DerivationOutput> DerivationOutputs; /* For inputs that are sub-derivations, we specify exactly which output IDs we are interested in. */ -typedef std::map<Path, StringSet> DerivationInputs; +typedef std::map<StorePath, StringSet> DerivationInputs; typedef std::map<string, string> StringPairs; struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ - PathSet inputSrcs; /* inputs that are sources */ + StorePathSet inputSrcs; /* inputs that are sources */ string platform; Path builder; Strings args; StringPairs env; + BasicDerivation() { } + explicit BasicDerivation(const BasicDerivation & other); virtual ~BasicDerivation() { }; /* Return the path corresponding to the output identifier `id' in the given derivation. */ - Path findOutput(const string & id) const; + const StorePath & findOutput(const std::string & id) const; bool isBuiltin() const; @@ -62,7 +56,7 @@ struct BasicDerivation bool isFixedOutput() const; /* Return the output paths of a derivation. */ - PathSet outputPaths() const; + StorePathSet outputPaths() const; }; @@ -71,7 +65,12 @@ struct Derivation : BasicDerivation DerivationInputs inputDrvs; /* inputs that are sub-derivations */ /* Print a derivation. */ - std::string unparse() const; + std::string unparse(const Store & store, bool maskOutputs, + std::map<std::string, StringSet> * actualInputs = nullptr) const; + + Derivation() { } + Derivation(Derivation && other) = default; + explicit Derivation(const Derivation & other); }; @@ -79,38 +78,29 @@ class Store; /* Write a derivation to the Nix store, and return its path. */ -Path writeDerivation(ref<Store> store, +StorePath writeDerivation(ref<Store> store, const Derivation & drv, const string & name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ -Derivation readDerivation(const Path & drvPath); +Derivation readDerivation(const Store & store, const Path & drvPath); -/* Check whether a file name ends with the extension for - derivations. */ +// FIXME: remove bool isDerivation(const string & fileName); -Hash hashDerivationModulo(Store & store, Derivation drv); +Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /* Memoisation of hashDerivationModulo(). */ -typedef std::map<Path, Hash> DrvHashes; +typedef std::map<StorePath, Hash> DrvHashes; extern DrvHashes drvHashes; // FIXME: global, not thread-safe -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path and - the outputs. */ -typedef std::pair<string, std::set<string> > DrvPathWithOutputs; -DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); - -Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs); - bool wantOutput(const string & output, const std::set<string> & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, Store & store, BasicDerivation & drv); -Sink & operator << (Sink & out, const BasicDerivation & drv); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv); +void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); std::string hashPlaceholder(const std::string & outputName); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 38faa1bfd..2eb39af3a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -650,10 +650,10 @@ struct CurlDownloader : public Downloader #ifdef ENABLE_S3 auto [bucketName, key, params] = parseS3Uri(request.uri); - std::string profile = get(params, "profile", ""); - std::string region = get(params, "region", Aws::Region::US_EAST_1); - std::string scheme = get(params, "scheme", ""); - std::string endpoint = get(params, "endpoint", ""); + std::string profile = get(params, "profile").value_or(""); + std::string region = get(params, "region").value_or(Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme").value_or(""); + std::string endpoint = get(params, "endpoint").value_or(""); S3Helper s3Helper(profile, region, scheme, endpoint); @@ -811,13 +811,13 @@ CachedDownloadResult Downloader::downloadCached( if (p != string::npos) name = string(url, p + 1); } - Path expectedStorePath; + std::optional<StorePath> expectedStorePath; if (request.expectedHash) { expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); - if (store->isValidPath(expectedStorePath)) { + if (store->isValidPath(*expectedStorePath)) { CachedDownloadResult result; - result.storePath = expectedStorePath; - result.path = store->toRealPath(expectedStorePath); + result.storePath = store->printStorePath(*expectedStorePath); + result.path = store->toRealPath(result.storePath); assert(!request.getLastModified); // FIXME return result; } @@ -833,7 +833,7 @@ CachedDownloadResult Downloader::downloadCached( PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); - Path storePath; + std::optional<StorePath> storePath; string expectedETag; @@ -842,9 +842,10 @@ CachedDownloadResult Downloader::downloadCached( CachedDownloadResult result; if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = readLink(fileLink); - store->addTempRoot(storePath); - if (store->isValidPath(storePath)) { + storePath = store->parseStorePath(readLink(fileLink)); + // FIXME + store->addTempRoot(*storePath); + if (store->isValidPath(*storePath)) { auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; @@ -858,7 +859,7 @@ CachedDownloadResult Downloader::downloadCached( } } } else - storePath = ""; + storePath.reset(); } if (!skip) { @@ -871,46 +872,45 @@ CachedDownloadResult Downloader::downloadCached( result.etag = res.etag; if (!res.cached) { - ValidPathInfo info; StringSink sink; dumpString(*res.data, sink); Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); - info.path = store->makeFixedOutputPath(false, hash, name); + ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(false, hash); store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - storePath = info.path; + storePath = info.path.clone(); } - assert(!storePath.empty()); - replaceSymlink(storePath, fileLink); + assert(storePath); + replaceSymlink(store->printStorePath(*storePath), fileLink); writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { - if (storePath.empty()) throw; + if (!storePath) throw; warn("warning: %s; using cached result", e.msg()); result.etag = expectedETag; } } if (request.unpack) { - Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; + Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked"; PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); - Path unpackedStorePath; + std::optional<StorePath> unpackedStorePath; if (pathExists(unpackedLink)) { - unpackedStorePath = readLink(unpackedLink); - store->addTempRoot(unpackedStorePath); - if (!store->isValidPath(unpackedStorePath)) - unpackedStorePath = ""; + unpackedStorePath = store->parseStorePath(readLink(unpackedLink)); + store->addTempRoot(*unpackedStorePath); + if (!store->isValidPath(*unpackedStorePath)) + unpackedStorePath.reset(); else result.lastModified = lstat(unpackedLink).st_mtime; } - if (unpackedStorePath.empty()) { + if (!unpackedStorePath) { printInfo("unpacking '%s'...", url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url)); + unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir, std::string(baseNameOf(url))); auto members = readDirectory(tmpDir); if (members.size() != 1) throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); @@ -921,15 +921,15 @@ CachedDownloadResult Downloader::downloadCached( // Store the last-modified date of the tarball in the symlink // mtime. This saves us from having to store it somewhere // else. - replaceSymlink(unpackedStorePath, unpackedLink, result.lastModified); - storePath = unpackedStorePath; + replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink, result.lastModified); + storePath = std::move(*unpackedStorePath); } - if (expectedStorePath != "" && storePath != expectedStorePath) { + if (expectedStorePath && *storePath != *expectedStorePath) { unsigned int statusCode = 102; Hash gotHash = request.unpack - ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first - : hashFile(request.expectedHash.type, store->toRealPath(storePath)); + ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first + : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))); throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", url, request.expectedHash.to_string(), gotHash.to_string()); } @@ -937,8 +937,8 @@ CachedDownloadResult Downloader::downloadCached( if (request.gcRoot) store->addIndirectRoot(fileLink); - result.storePath = storePath; - result.path = store->toRealPath(storePath); + result.storePath = store->printStorePath(*storePath); + result.path = store->toRealPath(result.storePath); return result; } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index ef3fea7c8..4692d1a7b 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -24,9 +24,9 @@ struct HashAndWriteSink : Sink } }; -void Store::exportPaths(const Paths & paths, Sink & sink) +void Store::exportPaths(const StorePathSet & paths, Sink & sink) { - Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end())); + auto sorted = topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); std::string doneLabel("paths exported"); @@ -42,7 +42,7 @@ void Store::exportPaths(const Paths & paths, Sink & sink) sink << 0; } -void Store::exportPath(const Path & path, Sink & sink) +void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); @@ -55,15 +55,21 @@ void Store::exportPath(const Path & path, Sink & sink) Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) - throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path - % info->narHash.to_string() % hash.to_string()); - - hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0; + throw Error("hash of path '%s' has changed from '%s' to '%s'!", + printStorePath(path), info->narHash.to_string(), hash.to_string()); + + hashAndWriteSink + << exportMagic + << printStorePath(path); + writeStorePaths(*this, hashAndWriteSink, info->references); + hashAndWriteSink + << (info->deriver ? printStorePath(*info->deriver) : "") + << 0; } -Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) +StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) { - Paths res; + StorePaths res; while (true) { auto n = readNum<uint64_t>(source); if (n == 0) break; @@ -77,16 +83,15 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, if (magic != exportMagic) throw Error("Nix archive cannot be imported; wrong format"); - ValidPathInfo info; - - info.path = readStorePath(*this, source); + ValidPathInfo info(parseStorePath(readString(source))); //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = readStorePaths<PathSet>(*this, source); + info.references = readStorePaths<StorePathSet>(*this, source); - info.deriver = readString(source); - if (info.deriver != "") assertStorePath(info.deriver); + auto deriver = readString(source); + if (deriver != "") + info.deriver = parseStorePath(deriver); info.narHash = hashString(htSHA256, *tee.source.data); info.narSize = tee.source.data->size(); @@ -97,7 +102,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); - res.push_back(info.path); + res.push_back(info.path.clone()); } return res; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 842980fb7..ed81186af 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -85,12 +85,10 @@ void LocalStore::addIndirectRoot(const Path & path) } -Path LocalFSStore::addPermRoot(const Path & _storePath, +Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) { - Path storePath(canonPath(_storePath)); Path gcRoot(canonPath(_gcRoot)); - assertStorePath(storePath); if (isInStore(gcRoot)) throw Error(format( @@ -102,7 +100,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot); - makeSymlink(gcRoot, storePath); + makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); } @@ -117,10 +115,10 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, % gcRoot % rootsDir); } - if (baseNameOf(gcRoot) == baseNameOf(storePath)) + if (baseNameOf(gcRoot) == std::string(storePath.to_string())) writeFile(gcRoot, ""); else - makeSymlink(gcRoot, storePath); + makeSymlink(gcRoot, printStorePath(storePath)); } /* Check that the root can be found by the garbage collector. @@ -129,13 +127,12 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, check if the root is in a directory in or linked from the gcroots directory. */ if (settings.checkRootReachability) { - Roots roots = findRoots(false); - if (roots[storePath].count(gcRoot) == 0) + auto roots = findRoots(false); + if (roots[storePath.clone()].count(gcRoot) == 0) printError( - format( - "warning: '%1%' is not in a directory where the garbage collector looks for roots; " - "therefore, '%2%' might be removed by the garbage collector") - % gcRoot % storePath); + "warning: '%1%' is not in a directory where the garbage collector looks for roots; " + "therefore, '%2%' might be removed by the garbage collector", + gcRoot, printStorePath(storePath)); } /* Grab the global GC root, causing us to block while a GC is in @@ -147,7 +144,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, } -void LocalStore::addTempRoot(const Path & path) +void LocalStore::addTempRoot(const StorePath & path) { auto state(_state.lock()); @@ -188,7 +185,7 @@ void LocalStore::addTempRoot(const Path & path) debug(format("acquiring write lock on '%1%'") % fnTempRoots); lockFile(state->fdTempRoots.get(), ltWrite, true); - string s = path + '\0'; + string s = printStorePath(path) + '\0'; writeFull(state->fdTempRoots.get(), s); /* Downgrade to a read lock. */ @@ -246,8 +243,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) while ((end = contents.find((char) 0, pos)) != string::npos) { Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); - assertStorePath(root); - tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); + tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } @@ -260,10 +256,11 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) { auto foundRoot = [&](const Path & path, const Path & target) { Path storePath = toStorePath(target); - if (isStorePath(storePath) && isValidPath(storePath)) - roots[storePath].emplace(path); + // FIXME + if (isStorePath(storePath) && isValidPath(parseStorePath(storePath))) + roots[parseStorePath(storePath)].emplace(path); else - printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath); + printInfo("skipping invalid root from '%1%' to '%2%'", path, storePath); }; try { @@ -299,9 +296,10 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } else if (type == DT_REG) { - Path storePath = storeDir + "/" + baseNameOf(path); - if (isStorePath(storePath) && isValidPath(storePath)) - roots[storePath].emplace(path); + Path storePath = storeDir + "/" + std::string(baseNameOf(path)); + // FIXME + if (isStorePath(storePath) && isValidPath(parseStorePath(storePath))) + roots[parseStorePath(storePath)].emplace(path); } } @@ -309,7 +307,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) catch (SysError & e) { /* We only ignore permanent failures. */ if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) - printInfo(format("cannot read potential root '%1%'") % path); + printInfo("cannot read potential root '%1%'", path); else throw; } @@ -340,7 +338,9 @@ Roots LocalStore::findRoots(bool censor) return roots; } -static void readProcLink(const string & file, Roots & roots) +typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots; + +static void readProcLink(const string & file, UncheckedRoots & roots) { /* 64 is the starting buffer size gnu readlink uses... */ auto bufsiz = ssize_t{64}; @@ -369,7 +369,7 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } -static void readFileRoots(const char * path, Roots & roots) +static void readFileRoots(const char * path, UncheckedRoots & roots) { try { roots[readFile(path)].emplace(path); @@ -381,7 +381,7 @@ static void readFileRoots(const char * path, Roots & roots) void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { - Roots unchecked; + UncheckedRoots unchecked; auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { @@ -466,16 +466,16 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) #endif for (auto & [target, links] : unchecked) { - if (isInStore(target)) { - Path path = toStorePath(target); - if (isStorePath(path) && isValidPath(path)) { - debug(format("got additional root '%1%'") % path); - if (censor) - roots[path].insert(censored); - else - roots[path].insert(links.begin(), links.end()); - } - } + if (!isInStore(target)) continue; + Path pathS = toStorePath(target); + if (!isStorePath(pathS)) continue; + auto path = parseStorePath(pathS); + if (!isValidPath(path)) continue; + debug("got additional root '%1%'", pathS); + if (censor) + roots[path.clone()].insert(censored); + else + roots[path.clone()].insert(links.begin(), links.end()); } } @@ -485,18 +485,19 @@ struct GCLimitReached { }; struct LocalStore::GCState { - GCOptions options; + const GCOptions & options; GCResults & results; - PathSet roots; - PathSet tempRoots; - PathSet dead; - PathSet alive; + StorePathSet roots; + StorePathSet tempRoots; + StorePathSet dead; + StorePathSet alive; bool gcKeepOutputs; bool gcKeepDerivations; unsigned long long bytesInvalidated; bool moveToTrash = true; bool shouldDelete; - GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { } + GCState(const GCOptions & options, GCResults & results) + : options(options), results(results), bytesInvalidated(0) { } }; @@ -504,7 +505,7 @@ bool LocalStore::isActiveTempFile(const GCState & state, const Path & path, const string & suffix) { return hasSuffix(path, suffix) - && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end(); + && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size()))); } @@ -522,16 +523,17 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) unsigned long long size = 0; - if (isStorePath(path) && isValidPath(path)) { - PathSet referrers; - queryReferrers(path, referrers); + // FIXME + if (isStorePath(path) && isValidPath(parseStorePath(path))) { + StorePathSet referrers; + queryReferrers(parseStorePath(path), referrers); for (auto & i : referrers) - if (i != path) deletePathRecursive(state, i); - size = queryPathInfo(path)->narSize; - invalidatePathChecked(path); + if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i)); + size = queryPathInfo(parseStorePath(path))->narSize; + invalidatePathChecked(parseStorePath(path)); } - Path realPath = realStoreDir + "/" + baseNameOf(path); + Path realPath = realStoreDir + "/" + std::string(baseNameOf(path)); struct stat st; if (lstat(realPath.c_str(), &st)) { @@ -555,7 +557,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) try { if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) throw SysError(format("making '%1%' writable") % realPath); - Path tmp = trashDir + "/" + baseNameOf(path); + Path tmp = trashDir + "/" + std::string(baseNameOf(path)); if (rename(realPath.c_str(), tmp.c_str())) throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp); state.bytesInvalidated += size; @@ -575,7 +577,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) } -bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path) +bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path) { if (visited.count(path)) return false; @@ -584,41 +586,41 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p if (state.dead.count(path)) return false; if (state.roots.count(path)) { - debug(format("cannot delete '%1%' because it's a root") % path); - state.alive.insert(path); + debug("cannot delete '%1%' because it's a root", printStorePath(path)); + state.alive.insert(path.clone()); return true; } - visited.insert(path); + visited.insert(path.clone()); - if (!isStorePath(path) || !isValidPath(path)) return false; + //FIXME + if (!isStorePath(printStorePath(path)) || !isValidPath(path)) return false; - PathSet incoming; + StorePathSet incoming; /* Don't delete this path if any of its referrers are alive. */ queryReferrers(path, incoming); /* If keep-derivations is set and this is a derivation, then don't delete the derivation if any of the outputs are alive. */ - if (state.gcKeepDerivations && isDerivation(path)) { - PathSet outputs = queryDerivationOutputs(path); - for (auto & i : outputs) + if (state.gcKeepDerivations && path.isDerivation()) { + for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i); + incoming.insert(i.clone()); } /* If keep-outputs is set, then don't delete this path if there are derivers of this path that are not garbage. */ if (state.gcKeepOutputs) { - PathSet derivers = queryValidDerivers(path); + auto derivers = queryValidDerivers(path); for (auto & i : derivers) - incoming.insert(i); + incoming.insert(i.clone()); } for (auto & i : incoming) if (i != path) if (canReachRoot(state, visited, i)) { - state.alive.insert(path); + state.alive.insert(path.clone()); return true; } @@ -630,12 +632,13 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) { checkInterrupt(); - auto realPath = realStoreDir + "/" + baseNameOf(path); + auto realPath = realStoreDir + "/" + std::string(baseNameOf(path)); if (realPath == linksDir || realPath == trashDir) return; //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); - if (!isStorePath(path) || !isValidPath(path)) { + // FIXME + if (!isStorePath(path) || !isValidPath(parseStorePath(path))) { /* A lock file belonging to a path that we're building right now isn't garbage. */ if (isActiveTempFile(state, path, ".lock")) return; @@ -650,16 +653,17 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) if (isActiveTempFile(state, path, ".check")) return; } - PathSet visited; + StorePathSet visited; - if (canReachRoot(state, visited, path)) { - debug(format("cannot delete '%1%' because it's still reachable") % path); + if (canReachRoot(state, visited, parseStorePath(path))) { + debug("cannot delete '%s' because it's still reachable", path); } else { /* No path we visited was a root, so everything is garbage. But we only delete ‘path’ and its referrers here so that ‘nix-store --delete’ doesn't have the unexpected effect of recursing into derivations and outputs. */ - state.dead.insert(visited.begin(), visited.end()); + for (auto & i : visited) + state.dead.insert(i.clone()); if (state.shouldDelete) deletePathRecursive(state, path); } @@ -715,8 +719,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { - GCState state(results); - state.options = options; + GCState state(options, results); state.gcKeepOutputs = settings.gcKeepOutputs; state.gcKeepDerivations = settings.gcKeepDerivations; @@ -741,12 +744,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ - printError(format("finding garbage collector roots...")); + printError("finding garbage collector roots..."); Roots rootMap; if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true); - for (auto & i : rootMap) state.roots.insert(i.first); + for (auto & i : rootMap) state.roots.insert(i.first.clone()); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths @@ -754,9 +757,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) FDs fds; Roots tempRoots; findTempRoots(fds, tempRoots, true); - for (auto & root : tempRoots) - state.tempRoots.insert(root.first); - state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); + for (auto & root : tempRoots) { + state.tempRoots.insert(root.first.clone()); + state.roots.insert(root.first.clone()); + } /* After this point the set of roots or temporary roots cannot increase, since we hold locks on everything. So everything @@ -768,7 +772,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) createDirs(trashDir); } catch (SysError & e) { if (e.errNo == ENOSPC) { - printInfo(format("note: can't create trash directory: %1%") % e.msg()); + printInfo("note: can't create trash directory: %s", e.msg()); state.moveToTrash = false; } } @@ -780,22 +784,21 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (options.action == GCOptions::gcDeleteSpecific) { for (auto & i : options.pathsToDelete) { - assertStorePath(i); - tryToDelete(state, i); + tryToDelete(state, printStorePath(i)); if (state.dead.find(i) == state.dead.end()) - throw Error(format( + throw Error( "cannot delete path '%1%' since it is still alive. " "To find out why use: " - "nix-store --query --roots" - ) % i); + "nix-store --query --roots", + printStorePath(i)); } } else if (options.maxFreed > 0) { if (state.shouldDelete) - printError(format("deleting garbage...")); + printError("deleting garbage..."); else - printError(format("determining live/dead paths...")); + printError("determining live/dead paths..."); try { @@ -815,7 +818,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) string name = dirent->d_name; if (name == "." || name == "..") continue; Path path = storeDir + "/" + name; - if (isStorePath(path) && isValidPath(path)) + // FIXME + if (isStorePath(path) && isValidPath(parseStorePath(path))) entries.push_back(path); else tryToDelete(state, path); @@ -840,12 +844,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } if (state.options.action == GCOptions::gcReturnLive) { - state.results.paths = state.alive; + for (auto & i : state.alive) + state.results.paths.insert(printStorePath(i)); return; } if (state.options.action == GCOptions::gcReturnDead) { - state.results.paths = state.dead; + for (auto & i : state.dead) + state.results.paths.insert(printStorePath(i)); return; } @@ -859,7 +865,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Clean up the links directory. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { - printError(format("deleting unused links...")); + printError("deleting unused links..."); removeUnusedLinks(state); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1b8b5908c..458266be0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -87,25 +87,27 @@ struct LegacySSHStore : public Store return uriScheme + host; } - void queryPathInfoUncached(const Path & path, + void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override { try { auto conn(connections->get()); - debug("querying remote host '%s' for info on '%s'", host, path); + debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); - conn->to << cmdQueryPathInfos << PathSet{path}; + conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)}; conn->to.flush(); - auto info = std::make_shared<ValidPathInfo>(); - conn->from >> info->path; - if (info->path.empty()) return callback(nullptr); + auto p = readString(conn->from); + if (p.empty()) return callback(nullptr); + auto info = std::make_shared<ValidPathInfo>(parseStorePath(p)); assert(path == info->path); PathSet references; - conn->from >> info->deriver; - info->references = readStorePaths<PathSet>(*this, conn->from); + auto deriver = readString(conn->from); + if (deriver != "") + info->deriver = parseStorePath(deriver); + info->references = readStorePaths<StorePathSet>(*this, conn->from); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -127,7 +129,7 @@ struct LegacySSHStore : public Store RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override { - debug("adding path '%s' to remote host '%s'", info.path, host); + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); auto conn(connections->get()); @@ -135,10 +137,11 @@ struct LegacySSHStore : public Store conn->to << cmdAddToStoreNar - << info.path - << info.deriver - << info.narHash.to_string(Base16, false) - << info.references + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(Base16, false); + writeStorePaths(*this, conn->to, info.references); + conn->to << info.registrationTime << info.narSize << info.ultimate @@ -165,9 +168,10 @@ struct LegacySSHStore : public Store } conn->to << exportMagic - << info.path - << info.references - << info.deriver + << printStorePath(info.path); + writeStorePaths(*this, conn->to, info.references); + conn->to + << (info.deriver ? printStorePath(*info.deriver) : "") << 0 << 0; conn->to.flush(); @@ -175,39 +179,40 @@ struct LegacySSHStore : public Store } if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s', info.path, host"); + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } - void narFromPath(const Path & path, Sink & sink) override + void narFromPath(const StorePath & path, Sink & sink) override { auto conn(connections->get()); - conn->to << cmdDumpStorePath << path; + conn->to << cmdDumpStorePath << printStorePath(path); conn->to.flush(); copyNAR(conn->from, sink); } - Path queryPathFromHashPart(const string & hashPart) override + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } - Path addToStore(const string & name, const Path & srcPath, + StorePath addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override { unsupported("addToStore"); } - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override { unsupported("addTextToStore"); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { auto conn(connections->get()); conn->to << cmdBuildDerivation - << drvPath - << drv + << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + conn->to << settings.maxSilentTime << settings.buildTimeout; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) @@ -230,11 +235,11 @@ struct LegacySSHStore : public Store return status; } - void ensurePath(const Path & path) override + void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } - void computeFSClosure(const PathSet & paths, - PathSet & out, bool flipDirection = false, + void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override { if (flipDirection || includeDerivers) { @@ -246,16 +251,15 @@ struct LegacySSHStore : public Store conn->to << cmdQueryClosure - << includeOutputs - << paths; + << includeOutputs; + writeStorePaths(*this, conn->to, paths); conn->to.flush(); - auto res = readStorePaths<PathSet>(*this, conn->from); - - out.insert(res.begin(), res.end()); + for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) + out.insert(i.clone()); } - PathSet queryValidPaths(const PathSet & paths, + StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override { auto conn(connections->get()); @@ -263,11 +267,11 @@ struct LegacySSHStore : public Store conn->to << cmdQueryValidPaths << false // lock - << maybeSubstitute - << paths; + << maybeSubstitute; + writeStorePaths(*this, conn->to, paths); conn->to.flush(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } void connect() override diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 9f99ee76d..363be1443 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -44,15 +44,15 @@ protected: } } - PathSet queryAllValidPaths() override + StorePathSet queryAllValidPaths() override { - PathSet paths; + StorePathSet paths; for (auto & entry : readDirectory(binaryCacheDir)) { if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo")) continue; - paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)); + paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8))); } return paths; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 642e4070d..aa5abd835 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -21,7 +21,7 @@ struct LocalStoreAccessor : public FSAccessor Path toRealPath(const Path & path) { Path storePath = store->toStorePath(path); - if (!store->isValidPath(storePath)) + if (!store->isValidPath(store->parseStorePath(storePath))) throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -77,34 +77,32 @@ ref<FSAccessor> LocalFSStore::getFSAccessor() std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); } -void LocalFSStore::narFromPath(const Path & path, Sink & sink) +void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) { if (!isValidPath(path)) - throw Error(format("path '%s' is not valid") % path); - dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink); + throw Error("path '%s' is not valid", printStorePath(path)); + dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); } const string LocalFSStore::drvsLogDir = "drvs"; -std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_) +std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_) { - auto path(path_); + auto path = path_.clone(); - assertStorePath(path); - - - if (!isDerivation(path)) { + if (!path.isDerivation()) { try { - path = queryPathInfo(path)->deriver; + auto info = queryPathInfo(path); + if (!info->deriver) return nullptr; + path = info->deriver->clone(); } catch (InvalidPath &) { return nullptr; } - if (path == "") return nullptr; } - string baseName = baseNameOf(path); + auto baseName = std::string(baseNameOf(printStorePath(path))); for (int j = 0; j < 2; j++) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 023c90cd8..47b055bc1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -534,42 +534,36 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid) } -void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv) +void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) { - string drvName = storePathToName(drvPath); - assert(isDerivation(drvName)); + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); if (drv.isFixedOutput()) { DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) - throw Error(format("derivation '%1%' does not have an output named 'out'") % drvPath); + throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); bool recursive; Hash h; out->second.parseHashInfo(recursive, h); - Path outPath = makeFixedOutputPath(recursive, h, drvName); + auto outPath = makeFixedOutputPath(recursive, h, drvName); StringPairs::const_iterator j = drv.env.find("out"); - if (out->second.path != outPath || j == drv.env.end() || j->second != outPath) - throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'") - % drvPath % out->second.path % outPath); + if (out->second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + printStorePath(drvPath), printStorePath(out->second.path), printStorePath(outPath)); } else { - Derivation drvCopy(drv); - for (auto & i : drvCopy.outputs) { - i.second.path = ""; - drvCopy.env[i.first] = ""; - } - - Hash h = hashDerivationModulo(*this, drvCopy); + Hash h = hashDerivationModulo(*this, drv, true); for (auto & i : drv.outputs) { - Path outPath = makeOutputPath(i.first, h, drvName); + auto outPath = makeOutputPath(i.first, h, drvName); StringPairs::const_iterator j = drv.env.find(i.first); - if (i.second.path != outPath || j == drv.env.end() || j->second != outPath) - throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'") - % drvPath % i.second.path % outPath); + if (i.second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + printStorePath(drvPath), printStorePath(i.second.path), printStorePath(outPath)); } } } @@ -578,16 +572,15 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs) { - checkStoreName(storePathToName(info.path)); - if (info.ca != "" && !info.isContentAddressed(*this)) - throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path); + throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", + printStorePath(info.path)); state.stmtRegisterValidPath.use() - (info.path) + (printStorePath(info.path)) (info.narHash.to_string(Base16)) (info.registrationTime == 0 ? time(0) : info.registrationTime) - (info.deriver, info.deriver != "") + (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) @@ -599,8 +592,8 @@ uint64_t LocalStore::addValidPath(State & state, the database. This is useful for the garbage collector: it can efficiently query whether a path is an output of some derivation. */ - if (isDerivation(info.path)) { - Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path)); + if (info.path.isDerivation()) { + auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string())); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -613,34 +606,31 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtAddDerivationOutput.use() (id) (i.first) - (i.second.path) + (printStorePath(i.second.path)) .exec(); } } { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared<ValidPathInfo>(info)); + state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), std::make_shared<ValidPathInfo>(info)); } return id; } -void LocalStore::queryPathInfoUncached(const Path & path, +void LocalStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { - auto info = std::make_shared<ValidPathInfo>(); - info->path = path; - - assertStorePath(path); + auto info = std::make_shared<ValidPathInfo>(path.clone()); callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { auto state(_state.lock()); /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path))); if (!useQueryPathInfo.next()) return std::shared_ptr<ValidPathInfo>(); @@ -650,13 +640,13 @@ void LocalStore::queryPathInfoUncached(const Path & path, try { info->narHash = Hash(useQueryPathInfo.getStr(1)); } catch (BadHash & e) { - throw Error("in valid-path entry for '%s': %s", path, e.what()); + throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what()); } info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); - if (s) info->deriver = s; + if (s) info->deriver = parseStorePath(s); /* Note that narSize = NULL yields 0. */ info->narSize = useQueryPathInfo.getInt(4); @@ -673,7 +663,7 @@ void LocalStore::queryPathInfoUncached(const Path & path, auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->references.insert(useQueryReferences.getStr(0)); + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); return info; })); @@ -691,27 +681,27 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) - (info.path) + (printStorePath(info.path)) .exec(); } -uint64_t LocalStore::queryValidPathId(State & state, const Path & path) +uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { - auto use(state.stmtQueryPathInfo.use()(path)); + auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); if (!use.next()) - throw Error(format("path '%1%' is not valid") % path); + throw Error("path '%s' is not valid", printStorePath(path)); return use.getInt(0); } -bool LocalStore::isValidPath_(State & state, const Path & path) +bool LocalStore::isValidPath_(State & state, const StorePath & path) { - return state.stmtQueryPathInfo.use()(path).next(); + return state.stmtQueryPathInfo.use()(printStorePath(path)).next(); } -bool LocalStore::isValidPathUncached(const Path & path) +bool LocalStore::isValidPathUncached(const StorePath & path) { return retrySQLite<bool>([&]() { auto state(_state.lock()); @@ -720,39 +710,38 @@ bool LocalStore::isValidPathUncached(const Path & path) } -PathSet LocalStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) +StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { - PathSet res; + StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(i)) res.insert(i.clone()); return res; } -PathSet LocalStore::queryAllValidPaths() +StorePathSet LocalStore::queryAllValidPaths() { - return retrySQLite<PathSet>([&]() { + return retrySQLite<StorePathSet>([&]() { auto state(_state.lock()); auto use(state->stmtQueryValidPaths.use()); - PathSet res; - while (use.next()) res.insert(use.getStr(0)); + StorePathSet res; + while (use.next()) res.insert(parseStorePath(use.getStr(0))); return res; }); } -void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers) { - auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); + auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path))); while (useQueryReferrers.next()) - referrers.insert(useQueryReferrers.getStr(0)); + referrers.insert(parseStorePath(useQueryReferrers.getStr(0))); } -void LocalStore::queryReferrers(const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(const StorePath & path, StorePathSet & referrers) { - assertStorePath(path); return retrySQLite<void>([&]() { auto state(_state.lock()); queryReferrers(*state, path, referrers); @@ -760,42 +749,40 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) } -PathSet LocalStore::queryValidDerivers(const Path & path) +StorePathSet LocalStore::queryValidDerivers(const StorePath & path) { - assertStorePath(path); - - return retrySQLite<PathSet>([&]() { + return retrySQLite<StorePathSet>([&]() { auto state(_state.lock()); - auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path))); - PathSet derivers; + StorePathSet derivers; while (useQueryValidDerivers.next()) - derivers.insert(useQueryValidDerivers.getStr(1)); + derivers.insert(parseStorePath(useQueryValidDerivers.getStr(1))); return derivers; }); } -PathSet LocalStore::queryDerivationOutputs(const Path & path) +StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) { - return retrySQLite<PathSet>([&]() { + return retrySQLite<StorePathSet>([&]() { auto state(_state.lock()); auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() (queryValidPathId(*state, path))); - PathSet outputs; + StorePathSet outputs; while (useQueryDerivationOutputs.next()) - outputs.insert(useQueryDerivationOutputs.getStr(1)); + outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1))); return outputs; }); } -StringSet LocalStore::queryDerivationOutputNames(const Path & path) +StringSet LocalStore::queryDerivationOutputNames(const StorePath & path) { return retrySQLite<StringSet>([&]() { auto state(_state.lock()); @@ -812,31 +799,36 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path) } -Path LocalStore::queryPathFromHashPart(const string & hashPart) +std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart) { if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); Path prefix = storeDir + "/" + hashPart; - return retrySQLite<Path>([&]() -> std::string { + return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> { auto state(_state.lock()); auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); - if (!useQueryPathFromHashPart.next()) return ""; + if (!useQueryPathFromHashPart.next()) return {}; const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); - return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; + if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0) + return parseStorePath(s); + return {}; }); } -PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) +StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) { - if (!settings.useSubstitutes) return PathSet(); + if (!settings.useSubstitutes) return StorePathSet(); - auto remaining = paths; - PathSet res; + StorePathSet remaining; + for (auto & i : paths) + remaining.insert(i.clone()); + + StorePathSet res; for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; @@ -845,12 +837,12 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) auto valid = sub->queryValidPaths(remaining); - PathSet remaining2; + StorePathSet remaining2; for (auto & path : remaining) if (valid.count(path)) - res.insert(path); + res.insert(path.clone()); else - remaining2.insert(path); + remaining2.insert(path.clone()); std::swap(remaining, remaining2); } @@ -859,7 +851,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const PathSet & paths, +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; @@ -867,17 +859,16 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, if (sub->storeDir != storeDir) continue; for (auto & path : paths) { if (infos.count(path)) continue; - debug(format("checking substituter '%s' for path '%s'") - % sub->getUri() % path); + debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); try { auto info = sub->queryPathInfo(path); auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); - infos[path] = SubstitutablePathInfo{ - info->deriver, - info->references, + infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ + info->deriver ? info->deriver->clone() : std::optional<StorePath>(), + cloneStorePathSet(info->references), narInfo ? narInfo->fileSize : 0, - info->narSize}; + info->narSize}); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { @@ -911,7 +902,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) auto state(_state.lock()); SQLiteTxn txn(state->db); - PathSet paths; + StorePathSet paths; for (auto & i : infos) { assert(i.narHash.type == htSHA256); @@ -919,7 +910,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) updatePathInfo(*state, i); else addValidPath(*state, i, false); - paths.insert(i.path); + paths.insert(i.path.clone()); } for (auto & i : infos) { @@ -932,10 +923,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) this in addValidPath() above, because the references might not be valid yet. */ for (auto & i : infos) - if (isDerivation(i.path)) { + if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(i.path)); - checkDerivationOutputs(i.path, drv); + checkDerivationOutputs(i.path, + readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string()))); } /* Do a topological sort of the paths. This will throw an @@ -951,18 +942,18 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -void LocalStore::invalidatePath(State & state, const Path & path) +void LocalStore::invalidatePath(State & state, const StorePath & path) { - debug(format("invalidating path '%1%'") % path); + debug("invalidating path '%s'", printStorePath(path)); - state.stmtInvalidatePath.use()(path).exec(); + state.stmtInvalidatePath.use()(printStorePath(path)).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(storePathToHash(path)); + state_->pathInfoCache.erase(storePathToHash(printStorePath(path))); } } @@ -980,10 +971,10 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { if (!info.narHash) - throw Error("cannot add path '%s' because it lacks a hash", info.path); + throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) - throw Error("cannot add path '%s' because it lacks a valid signature", info.path); + throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); addTempRoot(info.path); @@ -991,12 +982,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, PathLocks outputLock; - Path realPath = realStoreDir + "/" + baseNameOf(info.path); + Path realPath = realStoreDir + "/" + std::string(info.path.to_string()); /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a lock on this path). */ - if (!locksHeld.count(info.path)) + if (!locksHeld.count(printStorePath(info.path))) outputLock.lockPaths({realPath}); if (repair || !isValidPath(info.path)) { @@ -1011,7 +1002,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, else { if (!info.references.empty()) settings.requireExperimentalFeature("ca-references"); - hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path)); + hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path))); } LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { @@ -1026,11 +1017,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - info.path, info.narHash.to_string(), hashResult.first.to_string()); + printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string()); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", - info.path, info.narSize, hashResult.second); + printStorePath(info.path), info.narSize, hashResult.second); autoGC(); @@ -1046,12 +1037,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -Path LocalStore::addToStoreFromDump(const string & dump, const string & name, +StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, bool recursive, HashType hashAlgo, RepairFlag repair) { Hash h = hashString(hashAlgo, dump); - Path dstPath = makeFixedOutputPath(recursive, h, name); + auto dstPath = makeFixedOutputPath(recursive, h, name); addTempRoot(dstPath); @@ -1060,7 +1051,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + Path realPath = realStoreDir + "/"; + realPath += dstPath.to_string(); PathLocks outputLock({realPath}); @@ -1091,8 +1083,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, optimisePath(realPath); // FIXME: combine with hashPath() - ValidPathInfo info; - info.path = dstPath; + ValidPathInfo info(dstPath.clone()); info.narHash = hash.first; info.narSize = hash.second; info.ca = makeFixedOutputCA(recursive, h); @@ -1106,7 +1097,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, } -Path LocalStore::addToStore(const string & name, const Path & _srcPath, +StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); @@ -1124,8 +1115,8 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, } -Path LocalStore::addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) +StorePath LocalStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); auto dstPath = makeTextPath(name, hash, references); @@ -1134,7 +1125,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + Path realPath = realStoreDir + "/"; + realPath += dstPath.to_string(); PathLocks outputLock({realPath}); @@ -1154,11 +1146,10 @@ Path LocalStore::addTextToStore(const string & name, const string & s, optimisePath(realPath); - ValidPathInfo info; - info.path = dstPath; + ValidPathInfo info(dstPath.clone()); info.narHash = narHash; info.narSize = sink.s->size(); - info.references = references; + info.references = cloneStorePathSet(references); info.ca = "text:" + hash.to_string(); registerValidPath(info); } @@ -1180,27 +1171,25 @@ Path LocalStore::createTempDirInStore() the GC between createTempDir() and addTempRoot(), so repeat until `tmpDir' exists. */ tmpDir = createTempDir(realStoreDir); - addTempRoot(tmpDir); + addTempRoot(parseStorePath(tmpDir)); } while (!pathExists(tmpDir)); return tmpDir; } -void LocalStore::invalidatePathChecked(const Path & path) +void LocalStore::invalidatePathChecked(const StorePath & path) { - assertStorePath(path); - retrySQLite<void>([&]() { auto state(_state.lock()); SQLiteTxn txn(state->db); if (isValidPath_(*state, path)) { - PathSet referrers; queryReferrers(*state, path, referrers); + StorePathSet referrers; queryReferrers(*state, path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) - throw PathInUse(format("cannot delete path '%1%' because it is in use by %2%") - % path % showPaths(referrers)); + throw PathInUse("cannot delete path '%s' because it is in use by %s", + printStorePath(path), showPaths(referrers)); invalidatePath(*state, path); } @@ -1219,18 +1208,19 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) existing and valid paths. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); - PathSet store; + StringSet store; for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); /* Check whether all valid paths actually exist. */ printInfo("checking path existence..."); - PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + StorePathSet validPaths; + PathSet done; fdGCLock = -1; - for (auto & i : validPaths2) - verifyPath(i, store, done, validPaths, repair, errors); + for (auto & i : queryAllValidPaths()) + verifyPath(printStorePath(i), store, done, validPaths, repair, errors); /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1265,21 +1255,20 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, format("checking contents of '%1%'") % i); + printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); std::unique_ptr<AbstractHashSink> hashSink; if (info->ca == "") hashSink = std::make_unique<HashSink>(info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path)); + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path))); - dumpPath(toRealPath(i), *hashSink); + dumpPath(toRealPath(printStorePath(i)), *hashSink); auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - printError(format("path '%1%' was modified! " - "expected hash '%2%', got '%3%'") - % i % info->narHash.to_string() % current.first.to_string()); + printError("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(), current.first.to_string()); if (repair) repairPath(i); else errors = true; } else { @@ -1287,14 +1276,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Fill in missing hashes. */ if (info->narHash == nullHash) { - printError(format("fixing missing hash on '%1%'") % i); + printError("fixing missing hash on '%s'", printStorePath(i)); info->narHash = current.first; update = true; } /* Fill in missing narSize fields (from old stores). */ if (info->narSize == 0) { - printError(format("updating size field on '%1%' to %2%") % i % current.second); + printError("updating size field on '%s' to %s", printStorePath(i), current.second); info->narSize = current.second; update = true; } @@ -1310,9 +1299,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* It's possible that the path got GC'ed, so ignore errors on invalid paths. */ if (isValidPath(i)) - printError(format("error: %1%") % e.msg()); + printError("error: %s", e.msg()); else - printError(format("warning: %1%") % e.msg()); + warn(e.msg()); errors = true; } } @@ -1322,43 +1311,43 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors) +void LocalStore::verifyPath(const Path & pathS, const StringSet & store, + PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (!done.insert(path).second) return; + if (!done.insert(pathS).second) return; - if (!isStorePath(path)) { - printError(format("path '%1%' is not in the Nix store") % path); - auto state(_state.lock()); - invalidatePath(*state, path); + if (!isStorePath(pathS)) { + printError("path '%s' is not in the Nix store", pathS); return; } - if (store.find(baseNameOf(path)) == store.end()) { + auto path = parseStorePath(pathS); + + if (!store.count(std::string(path.to_string()))) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; - PathSet referrers; queryReferrers(path, referrers); + StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(i, store, done, validPaths, repair, errors); - if (validPaths.find(i) != validPaths.end()) + verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + if (validPaths.count(i)) canInvalidate = false; } if (canInvalidate) { - printError(format("path '%1%' disappeared, removing from database...") % path); + printError("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); invalidatePath(*state, path); } else { - printError(format("path '%1%' disappeared, but it still has valid referrers!") % path); + printError("path '%s' disappeared, but it still has valid referrers!", pathS); if (repair) try { repairPath(path); } catch (Error & e) { - printError(format("warning: %1%") % e.msg()); + warn(e.msg()); errors = true; } else errors = true; @@ -1367,7 +1356,7 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, return; } - validPaths.insert(path); + validPaths.insert(std::move(path)); } @@ -1436,7 +1425,7 @@ void LocalStore::vacuumDB() } -void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) +void LocalStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { retrySQLite<void>([&]() { auto state(_state.lock()); @@ -1462,7 +1451,7 @@ void LocalStore::signPathInfo(ValidPathInfo & info) for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); - info.sign(secretKey); + info.sign(*this, secretKey); } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1d4b88898..16aeab0ad 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -119,36 +119,36 @@ public: std::string getUri() override; - bool isValidPathUncached(const Path & path) override; + bool isValidPathUncached(const StorePath & path) override; - PathSet queryValidPaths(const PathSet & paths, + StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; - PathSet queryAllValidPaths() override; + StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const Path & path, + void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override; - void queryReferrers(const Path & path, PathSet & referrers) override; + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; - PathSet queryValidDerivers(const Path & path) override; + StorePathSet queryValidDerivers(const StorePath & path) override; - PathSet queryDerivationOutputs(const Path & path) override; + StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const Path & path) override; + StringSet queryDerivationOutputNames(const StorePath & path) override; - Path queryPathFromHashPart(const string & hashPart) override; + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; - PathSet querySubstitutablePaths(const PathSet & paths) override; + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const PathSet & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; - Path addToStore(const string & name, const Path & srcPath, + StorePath addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; @@ -156,20 +156,22 @@ public: in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == false). */ - Path addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(const string & dump, const string & name, bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override; + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override; + void buildPaths( + const std::vector<StorePathWithOutputs> & paths, + BuildMode buildMode) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const Path & path) override; + void ensurePath(const StorePath & path) override; - void addTempRoot(const Path & path) override; + void addTempRoot(const StorePath & path) override; void addIndirectRoot(const Path & path) override; @@ -215,9 +217,9 @@ public: /* Repair the contents of the given path by redownloading it using a substituter (if available). */ - void repairPath(const Path & path); + void repairPath(const StorePath & path); - void addSignatures(const Path & storePath, const StringSet & sigs) override; + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; /* If free disk space in /nix/store if below minFree, delete garbage until it exceeds maxFree. */ @@ -231,17 +233,17 @@ private: void makeStoreWritable(); - uint64_t queryValidPathId(State & state, const Path & path); + uint64_t queryValidPathId(State & state, const StorePath & path); uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); - void invalidatePath(State & state, const Path & path); + void invalidatePath(State & state, const StorePath & path); /* Delete a path from the Nix store. */ - void invalidatePathChecked(const Path & path); + void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors); + void verifyPath(const Path & path, const StringSet & store, + PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); void updatePathInfo(State & state, const ValidPathInfo & info); @@ -256,7 +258,7 @@ private: void tryToDelete(GCState & state, const Path & path); - bool canReachRoot(GCState & state, PathSet & visited, const Path & path); + bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path); void deletePathRecursive(GCState & state, const Path & path); @@ -275,7 +277,7 @@ private: Path createTempDirInStore(); - void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); typedef std::unordered_set<ino_t> InodeHash; @@ -284,8 +286,8 @@ private: void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash); // Internal versions that are not wrapped in retry_sqlite. - bool isValidPath_(State & state, const Path & path); - void queryReferrers(State & state, const Path & path, PathSet & referrers); + bool isValidPath_(State & state, const StorePath & path); + void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); /* Add signatures to a ValidPathInfo using the secret keys specified by the ‘secret-key-files’ option. */ diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d690fea28..d3254554d 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil +libstore_LIBS = libutil libnixrust libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 2fceb9b9a..9c47fe524 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -9,13 +9,13 @@ namespace nix { -void Store::computeFSClosure(const PathSet & startPaths, - PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) +void Store::computeFSClosure(const StorePathSet & startPaths, + StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { struct State { size_t pending; - PathSet & paths; + StorePathSet & paths; std::exception_ptr exc; }; @@ -29,45 +29,47 @@ void Store::computeFSClosure(const PathSet & startPaths, { auto state(state_.lock()); if (state->exc) return; - if (!state->paths.insert(path).second) return; + if (!state->paths.insert(parseStorePath(path)).second) return; state->pending++; } - queryPathInfo(path, {[&, path](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future<ref<const ValidPathInfo>> fut) { // FIXME: calls to isValidPath() should be async try { auto info = fut.get(); + auto path = parseStorePath(pathS); + if (flipDirection) { - PathSet referrers; + StorePathSet referrers; queryReferrers(path, referrers); for (auto & ref : referrers) if (ref != path) - enqueue(ref); + enqueue(printStorePath(ref)); if (includeOutputs) for (auto & i : queryValidDerivers(path)) - enqueue(i); + enqueue(printStorePath(i)); - if (includeDerivers && isDerivation(path)) + if (includeDerivers && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - enqueue(i); + enqueue(printStorePath(i)); } else { for (auto & ref : info->references) if (ref != path) - enqueue(ref); + enqueue(printStorePath(ref)); - if (includeOutputs && isDerivation(path)) + if (includeOutputs && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i)) enqueue(i); + if (isValidPath(i)) enqueue(printStorePath(i)); - if (includeDerivers && isValidPath(info->deriver)) - enqueue(info->deriver); + if (includeDerivers && info->deriver && isValidPath(*info->deriver)) + enqueue(printStorePath(*info->deriver)); } @@ -87,7 +89,7 @@ void Store::computeFSClosure(const PathSet & startPaths, }; for (auto & startPath : startPaths) - enqueue(startPath); + enqueue(printStorePath(startPath)); { auto state(state_.lock()); @@ -97,15 +99,17 @@ void Store::computeFSClosure(const PathSet & startPaths, } -void Store::computeFSClosure(const Path & startPath, - PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) +void Store::computeFSClosure(const StorePath & startPath, + StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { - computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers); + StorePathSet paths; + paths.insert(startPath.clone()); + computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers); } -void Store::queryMissing(const PathSet & targets, - PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_, +void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, + StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, unsigned long long & downloadSize_, unsigned long long & narSize_) { Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); @@ -116,8 +120,8 @@ void Store::queryMissing(const PathSet & targets, struct State { - PathSet done; - PathSet & unknown, & willSubstitute, & willBuild; + std::unordered_set<std::string> done; + StorePathSet & unknown, & willSubstitute, & willBuild; unsigned long long & downloadSize; unsigned long long & narSize; }; @@ -126,31 +130,36 @@ void Store::queryMissing(const PathSet & targets, { size_t left; bool done = false; - PathSet outPaths; + StorePathSet outPaths; DrvState(size_t left) : left(left) { } }; - Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); + Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - std::function<void(Path)> doPath; + std::function<void(StorePathWithOutputs)> doPath; - auto mustBuildDrv = [&](const Path & drvPath, const Derivation & drv) { + auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); - state->willBuild.insert(drvPath); + state->willBuild.insert(drvPath.clone()); } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second))); }; auto checkOutput = [&]( - const Path & drvPath, ref<Derivation> drv, const Path & outPath, ref<Sync<DrvState>> drvState_) + const Path & drvPathS, ref<Derivation> drv, const Path & outPathS, ref<Sync<DrvState>> drvState_) { if (drvState_->lock()->done) return; + auto drvPath = parseStorePath(drvPathS); + auto outPath = parseStorePath(outPathS); + SubstitutablePathInfos infos; - querySubstitutablePathInfos({outPath}, infos); + StorePathSet paths; // FIXME + paths.insert(outPath.clone()); + querySubstitutablePathInfos(paths, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -161,74 +170,74 @@ void Store::queryMissing(const PathSet & targets, if (drvState->done) return; assert(drvState->left); drvState->left--; - drvState->outPaths.insert(outPath); + drvState->outPaths.insert(outPath.clone()); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, path)); + pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone()))); } } } }; - doPath = [&](const Path & path) { + doPath = [&](const StorePathWithOutputs & path) { { auto state(state_.lock()); - if (!state->done.insert(path).second) return; + if (!state->done.insert(path.to_string(*this)).second) return; } - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); - - if (isDerivation(i2.first)) { - if (!isValidPath(i2.first)) { + if (path.path.isDerivation()) { + if (!isValidPath(path.path)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path); + state->unknown.insert(path.path.clone()); return; } - Derivation drv = derivationFromPath(i2.first); - ParsedDerivation parsedDrv(i2.first, drv); + auto drv = make_ref<Derivation>(derivationFromPath(path.path)); + ParsedDerivation parsedDrv(path.path.clone(), *drv); PathSet invalid; - for (auto & j : drv.outputs) - if (wantOutput(j.first, i2.second) + for (auto & j : drv->outputs) + if (wantOutput(j.first, path.outputs) && !isValidPath(j.second.path)) - invalid.insert(j.second.path); + invalid.insert(printStorePath(j.second.path)); if (invalid.empty()) return; if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState)); + pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState)); } else - mustBuildDrv(i2.first, drv); + mustBuildDrv(path.path, *drv); } else { - if (isValidPath(path)) return; + if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({path}, infos); + StorePathSet paths; // FIXME + paths.insert(path.path.clone()); + querySubstitutablePathInfos(paths, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path); + state->unknown.insert(path.path.clone()); return; } - auto info = infos.find(path); + auto info = infos.find(path.path); assert(info != infos.end()); { auto state(state_.lock()); - state->willSubstitute.insert(path); + state->willSubstitute.insert(path.path.clone()); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, ref)); + pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref))); } }; @@ -239,39 +248,42 @@ void Store::queryMissing(const PathSet & targets, } -Paths Store::topoSortPaths(const PathSet & paths) +StorePaths Store::topoSortPaths(const StorePathSet & paths) { - Paths sorted; - PathSet visited, parents; + StorePaths sorted; + StorePathSet visited, parents; - std::function<void(const Path & path, const Path * parent)> dfsVisit; + std::function<void(const StorePath & path, const StorePath * parent)> dfsVisit; - dfsVisit = [&](const Path & path, const Path * parent) { - if (parents.find(path) != parents.end()) - throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent); + dfsVisit = [&](const StorePath & path, const StorePath * parent) { + if (parents.count(path)) + throw BuildError("cycle detected in the references of '%s' from '%s'", + printStorePath(path), printStorePath(*parent)); - if (!visited.insert(path).second) return; - parents.insert(path); + if (!visited.insert(path.clone()).second) return; + parents.insert(path.clone()); - PathSet references; + StorePathSet references; try { - references = queryPathInfo(path)->references; + references = cloneStorePathSet(queryPathInfo(path)->references); } catch (InvalidPath &) { } for (auto & i : references) /* Don't traverse into paths that don't exist. That can happen due to substitutes for non-existent paths. */ - if (i != path && paths.find(i) != paths.end()) + if (i != path && paths.count(i)) dfsVisit(i, &path); - sorted.push_front(path); + sorted.push_back(path.clone()); parents.erase(path); }; for (auto & i : paths) dfsVisit(i, nullptr); + std::reverse(sorted.begin(), sorted.end()); + return sorted; } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 37c4c72e0..2bd515536 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -188,11 +188,8 @@ public: if (!queryNAR.getInt(0)) return {oInvalid, 0}; - auto narInfo = make_ref<NarInfo>(); - auto namePart = queryNAR.getStr(1); - narInfo->path = cache.storeDir + "/" + - hashPart + (namePart.empty() ? "" : "-" + namePart); + auto narInfo = make_ref<NarInfo>(StorePath::fromBaseName(hashPart + "-" + namePart)); narInfo->url = queryNAR.getStr(2); narInfo->compression = queryNAR.getStr(3); if (!queryNAR.isNull(4)) @@ -201,9 +198,9 @@ public: narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) - narInfo->references.insert(cache.storeDir + "/" + r); + narInfo->references.insert(StorePath::fromBaseName(r)); if (!queryNAR.isNull(9)) - narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9); + narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); narInfo->ca = queryNAR.getStr(11); @@ -225,12 +222,12 @@ public: auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); - assert(hashPart == storePathToHash(info->path)); + //assert(hashPart == storePathToHash(info->path)); state->insertNAR.use() (cache.id) (hashPart) - (storePathToName(info->path)) + (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) @@ -238,7 +235,7 @@ public: (info->narHash.to_string()) (info->narSize) (concatStringsSep(" ", info->shortRefs())) - (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") + (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) (info->ca) (time(0)).exec(); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index cb568ccdc..fb02cf3fd 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,6 +4,7 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) + : ValidPathInfo(StorePath::make((unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")) // FIXME: hack { auto corrupt = [&]() { throw Error(format("NAR info file '%1%' is corrupt") % whence); @@ -18,6 +19,8 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & } }; + bool havePath = false; + size_t pos = 0; while (pos < s.size()) { @@ -32,8 +35,8 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & std::string value(s, colon + 2, eol - colon - 2); if (name == "StorePath") { - if (!store.isStorePath(value)) corrupt(); - path = value; + path = store.parseStorePath(value); + havePath = true; } else if (name == "URL") url = value; @@ -52,18 +55,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "References") { auto refs = tokenizeString<Strings>(value, " "); if (!references.empty()) corrupt(); - for (auto & r : refs) { - auto r2 = store.storeDir + "/" + r; - if (!store.isStorePath(r2)) corrupt(); - references.insert(r2); - } + for (auto & r : refs) + references.insert(StorePath::fromBaseName(r)); } else if (name == "Deriver") { - if (value != "unknown-deriver") { - auto p = store.storeDir + "/" + value; - if (!store.isStorePath(p)) corrupt(); - deriver = p; - } + if (value != "unknown-deriver") + deriver = StorePath::fromBaseName(value); } else if (name == "System") system = value; @@ -79,13 +76,13 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt(); + if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt(); } -std::string NarInfo::to_string() const +std::string NarInfo::to_string(const Store & store) const { std::string res; - res += "StorePath: " + path + "\n"; + res += "StorePath: " + store.printStorePath(path) + "\n"; res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; @@ -98,8 +95,8 @@ std::string NarInfo::to_string() const res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; - if (!deriver.empty()) - res += "Deriver: " + baseNameOf(deriver) + "\n"; + if (deriver) + res += "Deriver: " + std::string(deriver->to_string()) + "\n"; if (!system.empty()) res += "System: " + system + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 4995061fb..373c33427 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -14,11 +14,12 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; std::string system; - NarInfo() { } + NarInfo() = delete; + NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); - std::string to_string() const; + std::string to_string(const Store & store) const; }; } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 991512f21..8ac382e9d 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -254,7 +254,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) { Activity act(*logger, actOptimiseStore); - PathSet paths = queryAllValidPaths(); + auto paths = queryAllValidPaths(); InodeHash inodeHash = loadInodeHash(); act.progress(0, paths.size()); @@ -265,8 +265,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats) addTempRoot(i); if (!isValidPath(i)) continue; /* path was GC'ed, probably */ { - Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i)); - optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash); + Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); + optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash); } done++; act.progress(done, paths.size()); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 5553dd863..45c033c66 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -4,8 +4,8 @@ namespace nix { -ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) - : drvPath(drvPath), drv(drv) +ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) + : drvPath(std::move(drvPath)), drv(drv) { /* Parse the __json attribute, if any. */ auto jsonAttr = drv.env.find("__json"); @@ -13,7 +13,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) try { structuredAttrs = std::make_unique<nlohmann::json>(nlohmann::json::parse(jsonAttr->second)); } catch (std::exception & e) { - throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); + throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what()); } } } @@ -28,7 +28,7 @@ std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & n return {}; else { if (!i->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath); + throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath.to_string()); return i->get<std::string>(); } } else { @@ -48,7 +48,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const return def; else { if (!i->is_boolean()) - throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath); + throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath.to_string()); return i->get<bool>(); } } else { @@ -68,11 +68,11 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name return {}; else { if (!i->is_array()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string()); Strings res; for (auto j = i->begin(); j != i->end(); ++j) { if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string()); res.push_back(j->get<std::string>()); } return res; diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 6e67e1665..f4df5dd54 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -6,13 +6,13 @@ namespace nix { class ParsedDerivation { - Path drvPath; + StorePath drvPath; BasicDerivation & drv; std::unique_ptr<nlohmann::json> structuredAttrs; public: - ParsedDerivation(const Path & drvPath, BasicDerivation & drv); + ParsedDerivation(StorePath && drvPath, BasicDerivation & drv); ~ParsedDerivation(); diff --git a/src/libstore/path.cc b/src/libstore/path.cc new file mode 100644 index 000000000..81ae495a1 --- /dev/null +++ b/src/libstore/path.cc @@ -0,0 +1,99 @@ +#include "store-api.hh" + +namespace nix { + +extern "C" { + rust::Result<StorePath> ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir); + rust::Result<StorePath> ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir); + rust::Result<StorePath> ffi_StorePath_fromBaseName(rust::StringSlice baseName); + rust::String ffi_StorePath_to_string(const StorePath & _this); + StorePath ffi_StorePath_clone(const StorePath & _this); + rust::StringSlice ffi_StorePath_name(const StorePath & _this); +} + +StorePath StorePath::make(std::string_view path, std::string_view storeDir) +{ + return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap(); +} + +StorePath StorePath::make(unsigned char hash[20], std::string_view name) +{ + return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap(); +} + +StorePath StorePath::fromBaseName(std::string_view baseName) +{ + return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap(); +} + +rust::String StorePath::to_string() const +{ + return ffi_StorePath_to_string(*this); +} + +StorePath StorePath::clone() const +{ + return ffi_StorePath_clone(*this); +} + +bool StorePath::isDerivation() const +{ + return hasSuffix(name(), drvExtension); +} + +std::string_view StorePath::name() const +{ + return ffi_StorePath_name(*this); +} + +StorePath Store::parseStorePath(std::string_view path) const +{ + return StorePath::make(path, storeDir); +} + + +StorePathSet Store::parseStorePathSet(const PathSet & paths) const +{ + StorePathSet res; + for (auto & i : paths) res.insert(parseStorePath(i)); + return res; +} + +std::string Store::printStorePath(const StorePath & path) const +{ + auto s = storeDir + "/"; + s += (std::string_view) path.to_string(); + return s; +} + +PathSet Store::printStorePathSet(const StorePathSet & paths) const +{ + PathSet res; + for (auto & i : paths) res.insert(printStorePath(i)); + return res; +} + +StorePathSet cloneStorePathSet(const StorePathSet & paths) +{ + StorePathSet res; + for (auto & p : paths) + res.insert(p.clone()); + return res; +} + +StorePathSet storePathsToSet(const StorePaths & paths) +{ + StorePathSet res; + for (auto & p : paths) + res.insert(p.clone()); + return res; +} + +StorePathSet singleton(const StorePath & path) +{ + StorePathSet res; + res.insert(path.clone()); + return res; +} + +} diff --git a/src/libstore/path.hh b/src/libstore/path.hh new file mode 100644 index 000000000..273808f02 --- /dev/null +++ b/src/libstore/path.hh @@ -0,0 +1,81 @@ +#pragma once + +#include "rust-ffi.hh" + +namespace nix { + +/* See path.rs. */ +struct StorePath; + +extern "C" { + void ffi_StorePath_drop(void *); + bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b); + bool ffi_StorePath_eq(const StorePath & a, const StorePath & b); + unsigned char * ffi_StorePath_hash_data(const StorePath & p); +} + +struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> +{ + static StorePath make(std::string_view path, std::string_view storeDir); + + static StorePath make(unsigned char hash[20], std::string_view name); + + static StorePath fromBaseName(std::string_view baseName); + + rust::String to_string() const; + + bool operator < (const StorePath & other) const + { + return ffi_StorePath_less_than(*this, other); + } + + bool operator == (const StorePath & other) const + { + return ffi_StorePath_eq(*this, other); + } + + bool operator != (const StorePath & other) const + { + return !(*this == other); + } + + StorePath clone() const; + + /* Check whether a file name ends with the extension for + derivations. */ + bool isDerivation() const; + + std::string_view name() const; + + unsigned char * hashData() const + { + return ffi_StorePath_hash_data(*this); + } +}; + +typedef std::set<StorePath> StorePathSet; +typedef std::vector<StorePath> StorePaths; + +StorePathSet cloneStorePathSet(const StorePathSet & paths); +StorePathSet storePathsToSet(const StorePaths & paths); + +StorePathSet singleton(const StorePath & path); + +/* Size of the hash part of store paths, in base-32 characters. */ +const size_t storePathHashLen = 32; // i.e. 160 bits + +/* Extension of derivations in the Nix store. */ +const std::string drvExtension = ".drv"; + +} + +namespace std { + +template<> struct hash<nix::StorePath> { + std::size_t operator()(const nix::StorePath & path) const noexcept + { + return * (std::size_t *) path.hashData(); + } +}; + +} diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 29f6f6c17..2bef51878 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -40,7 +40,7 @@ Generations findGenerations(Path profile, int & curGen) Generations gens; Path profileDir = dirOf(profile); - string profileName = baseNameOf(profile); + auto profileName = std::string(baseNameOf(profile)); for (auto & i : readDirectory(profileDir)) { int n; @@ -108,7 +108,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) user environment etc. we've just built. */ Path generation; makeName(profile, num + 1, generation); - store->addPermRoot(outPath, generation, false, true); + store->addPermRoot(store->parseStorePath(outPath), generation, false, true); return generation; } diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 605ca9815..102e15921 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -89,7 +89,7 @@ PathSet scanForReferences(const string & path, hash part of the file name. (This assumes that all references have the form `HASH-bla'). */ for (auto & i : refs) { - string baseName = baseNameOf(i); + auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); if (pos == string::npos) throw Error(format("bad reference '%1%'") % i); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 5233fb2c2..5a2d103b9 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -50,7 +50,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) auto storePath = store->toStorePath(path); std::string restPath = std::string(path, storePath.size()); - if (!store->isValidPath(storePath)) + if (!store->isValidPath(store->parseStorePath(storePath))) throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); auto i = nars.find(storePath); @@ -96,7 +96,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) } catch (SysError &) { } } - store->narFromPath(storePath, sink); + store->narFromPath(store->parseStorePath(storePath), sink); auto narAccessor = makeNarAccessor(sink.s); addToCache(storePath, *sink.s, narAccessor); return {narAccessor, restPath}; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 2a89b7c98..8c55da268 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -22,23 +22,22 @@ namespace nix { -Path readStorePath(Store & store, Source & from) +template<> StorePathSet readStorePaths(const Store & store, Source & from) { - Path path = readString(from); - store.assertStorePath(path); - return path; + StorePathSet paths; + for (auto & i : readStrings<Strings>(from)) + paths.insert(store.parseStorePath(i)); + return paths; } -template<class T> T readStorePaths(Store & store, Source & from) +void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths) { - T paths = readStrings<T>(from); - for (auto & i : paths) store.assertStorePath(i); - return paths; + out << paths.size(); + for (auto & i : paths) + out << store.printStorePath(i); } -template PathSet readStorePaths(Store & store, Source & from); -template Paths readStorePaths(Store & store, Source & from); /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -254,60 +253,62 @@ ConnectionHandle RemoteStore::getConnection() } -bool RemoteStore::isValidPathUncached(const Path & path) +bool RemoteStore::isValidPathUncached(const StorePath & path) { auto conn(getConnection()); - conn->to << wopIsValidPath << path; + conn->to << wopIsValidPath << printStorePath(path); conn.processStderr(); return readInt(conn->from); } -PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) +StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - PathSet res; + StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(i)) res.insert(i.clone()); return res; } else { - conn->to << wopQueryValidPaths << paths; + conn->to << wopQueryValidPaths; + writeStorePaths(*this, conn->to, paths); conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } } -PathSet RemoteStore::queryAllValidPaths() +StorePathSet RemoteStore::queryAllValidPaths() { auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } -PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) +StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) { auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - PathSet res; + StorePathSet res; for (auto & i : paths) { - conn->to << wopHasSubstitutes << i; + conn->to << wopHasSubstitutes << printStorePath(i); conn.processStderr(); - if (readInt(conn->from)) res.insert(i); + if (readInt(conn->from)) res.insert(i.clone()); } return res; } else { - conn->to << wopQuerySubstitutablePaths << paths; + conn->to << wopQuerySubstitutablePaths; + writeStorePaths(*this, conn->to, paths); conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } } -void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) { if (paths.empty()) return; @@ -318,29 +319,31 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, for (auto & i : paths) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << i; + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; - info.deriver = readString(conn->from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(*this, conn->from); + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = readStorePaths<StorePathSet>(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos[i] = info; + infos.insert_or_assign(i.clone(), std::move(info)); } } else { - conn->to << wopQuerySubstitutablePathInfos << paths; + conn->to << wopQuerySubstitutablePathInfos; + writeStorePaths(*this, conn->to, paths); conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { - Path path = readStorePath(*this, conn->from); - SubstitutablePathInfo & info(infos[path]); - info.deriver = readString(conn->from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(*this, conn->from); + SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = readStorePaths<StorePathSet>(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -349,14 +352,14 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, } -void RemoteStore::queryPathInfoUncached(const Path & path, +void RemoteStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { std::shared_ptr<ValidPathInfo> info; { auto conn(getConnection()); - conn->to << wopQueryPathInfo << path; + conn->to << wopQueryPathInfo << printStorePath(path); try { conn.processStderr(); } catch (Error & e) { @@ -367,14 +370,13 @@ void RemoteStore::queryPathInfoUncached(const Path & path, } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { bool valid; conn->from >> valid; - if (!valid) throw InvalidPath(format("path '%s' is not valid") % path); + if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = std::make_shared<ValidPathInfo>(); - info->path = path; - info->deriver = readString(conn->from); - if (info->deriver != "") assertStorePath(info->deriver); + info = std::make_shared<ValidPathInfo>(path.clone()); + auto deriver = readString(conn->from); + if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths<PathSet>(*this, conn->from); + info->references = readStorePaths<StorePathSet>(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -387,52 +389,52 @@ void RemoteStore::queryPathInfoUncached(const Path & path, } -void RemoteStore::queryReferrers(const Path & path, - PathSet & referrers) +void RemoteStore::queryReferrers(const StorePath & path, + StorePathSet & referrers) { auto conn(getConnection()); - conn->to << wopQueryReferrers << path; + conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from); - referrers.insert(referrers2.begin(), referrers2.end()); + for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) + referrers.insert(i.clone()); } -PathSet RemoteStore::queryValidDerivers(const Path & path) +StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) { auto conn(getConnection()); - conn->to << wopQueryValidDerivers << path; + conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } -PathSet RemoteStore::queryDerivationOutputs(const Path & path) +StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) { auto conn(getConnection()); - conn->to << wopQueryDerivationOutputs << path; + conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } -PathSet RemoteStore::queryDerivationOutputNames(const Path & path) +PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path) { auto conn(getConnection()); - conn->to << wopQueryDerivationOutputNames << path; + conn->to << wopQueryDerivationOutputNames << printStorePath(path); conn.processStderr(); return readStrings<PathSet>(conn->from); } -Path RemoteStore::queryPathFromHashPart(const string & hashPart) +std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); conn->to << wopQueryPathFromHashPart << hashPart; conn.processStderr(); Path path = readString(conn->from); - if (!path.empty()) assertStorePath(path); - return path; + if (path.empty()) return {}; + return parseStorePath(path); } @@ -450,9 +452,10 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, copyNAR(source, sink); sink << exportMagic - << info.path - << info.references - << info.deriver + << printStorePath(info.path); + writeStorePaths(*this, sink, info.references); + sink + << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature << 0 // == no path follows ; @@ -460,14 +463,17 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = readStorePaths<PathSet>(*this, conn->from); + auto importedPaths = readStorePaths<StorePathSet>(*this, conn->from); assert(importedPaths.size() <= 1); } else { conn->to << wopAddToStoreNar - << info.path << info.deriver << info.narHash.to_string(Base16, false) - << info.references << info.registrationTime << info.narSize + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(Base16, false); + writeStorePaths(*this, conn->to, info.references); + conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; @@ -477,7 +483,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } -Path RemoteStore::addToStore(const string & name, const Path & _srcPath, +StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); @@ -511,54 +517,52 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, throw; } - return readStorePath(*this, conn->from); + return parseStorePath(readString(conn->from)); } -Path RemoteStore::addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) +StorePath RemoteStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); auto conn(getConnection()); - conn->to << wopAddTextToStore << name << s << references; + conn->to << wopAddTextToStore << name << s; + writeStorePaths(*this, conn->to, references); conn.processStderr(); - return readStorePath(*this, conn->from); + return parseStorePath(readString(conn->from)); } -void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) { auto conn(getConnection()); conn->to << wopBuildPaths; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { - conn->to << drvPaths; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) - conn->to << buildMode; - else - /* Old daemons did not take a 'buildMode' parameter, so we - need to validate it here on the client side. */ - if (buildMode != bmNormal) - throw Error("repairing or checking is not supported when building through the Nix daemon"); - } else { - /* For backwards compatibility with old daemons, strip output - identifiers. */ - PathSet drvPaths2; - for (auto & i : drvPaths) - drvPaths2.insert(string(i, 0, i.find('!'))); - conn->to << drvPaths2; - } + assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); + Strings ss; + for (auto & p : drvPaths) + ss.push_back(p.to_string(*this)); + conn->to << ss; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + conn->to << buildMode; + else + /* Old daemons did not take a 'buildMode' parameter, so we + need to validate it here on the client side. */ + if (buildMode != bmNormal) + throw Error("repairing or checking is not supported when building through the Nix daemon"); conn.processStderr(); readInt(conn->from); } -BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, +BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { auto conn(getConnection()); - conn->to << wopBuildDerivation << drvPath << drv << buildMode; + conn->to << wopBuildDerivation << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + conn->to << buildMode; conn.processStderr(); BuildResult res; unsigned int status; @@ -568,19 +572,19 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva } -void RemoteStore::ensurePath(const Path & path) +void RemoteStore::ensurePath(const StorePath & path) { auto conn(getConnection()); - conn->to << wopEnsurePath << path; + conn->to << wopEnsurePath << printStorePath(path); conn.processStderr(); readInt(conn->from); } -void RemoteStore::addTempRoot(const Path & path) +void RemoteStore::addTempRoot(const StorePath & path) { auto conn(getConnection()); - conn->to << wopAddTempRoot << path; + conn->to << wopAddTempRoot << printStorePath(path); conn.processStderr(); readInt(conn->from); } @@ -613,8 +617,8 @@ Roots RemoteStore::findRoots(bool censor) Roots result; while (count--) { Path link = readString(conn->from); - Path target = readStorePath(*this, conn->from); - result[target].emplace(link); + auto target = parseStorePath(readString(conn->from)); + result[std::move(target)].emplace(link); } return result; } @@ -625,7 +629,9 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) auto conn(getConnection()); conn->to - << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness + << wopCollectGarbage << options.action; + writeStorePaths(*this, conn->to, options.pathsToDelete); + conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ << 0 << 0 << 0; @@ -661,17 +667,17 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) } -void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) +void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { auto conn(getConnection()); - conn->to << wopAddSignatures << storePath << sigs; + conn->to << wopAddSignatures << printStorePath(storePath) << sigs; conn.processStderr(); readInt(conn->from); } -void RemoteStore::queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, +void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize) { { @@ -680,11 +686,15 @@ void RemoteStore::queryMissing(const PathSet & targets, // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; - conn->to << wopQueryMissing << targets; + conn->to << wopQueryMissing; + Strings ss; + for (auto & p : targets) + ss.push_back(p.to_string(*this)); + conn->to << ss; conn.processStderr(); - willBuild = readStorePaths<PathSet>(*this, conn->from); - willSubstitute = readStorePaths<PathSet>(*this, conn->from); - unknown = readStorePaths<PathSet>(*this, conn->from); + willBuild = readStorePaths<StorePathSet>(*this, conn->from); + willSubstitute = readStorePaths<StorePathSet>(*this, conn->from); + unknown = readStorePaths<StorePathSet>(*this, conn->from); conn->from >> downloadSize >> narSize; return; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 728df8b00..f301a97d8 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -35,50 +35,50 @@ public: /* Implementations of abstract store API methods. */ - bool isValidPathUncached(const Path & path) override; + bool isValidPathUncached(const StorePath & path) override; - PathSet queryValidPaths(const PathSet & paths, + StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; - PathSet queryAllValidPaths() override; + StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const Path & path, + void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override; - void queryReferrers(const Path & path, PathSet & referrers) override; + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; - PathSet queryValidDerivers(const Path & path) override; + StorePathSet queryValidDerivers(const StorePath & path) override; - PathSet queryDerivationOutputs(const Path & path) override; + StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const Path & path) override; + StringSet queryDerivationOutputNames(const StorePath & path) override; - Path queryPathFromHashPart(const string & hashPart) override; + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; - PathSet querySubstitutablePaths(const PathSet & paths) override; + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const PathSet & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; - Path addToStore(const string & name, const Path & srcPath, + StorePath addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override; + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override; + void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const Path & path) override; + void ensurePath(const StorePath & path) override; - void addTempRoot(const Path & path) override; + void addTempRoot(const StorePath & path) override; void addIndirectRoot(const Path & path) override; @@ -92,10 +92,10 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; - void addSignatures(const Path & storePath, const StringSet & sigs) override; + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + void queryMissing(const std::vector<StorePathWithOutputs> & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize) override; void connect() override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index cd547a964..58966a5b7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -222,7 +222,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore fetches the .narinfo file, rather than first checking for its existence via a HEAD request. Since .narinfos are small, doing a GET is unlikely to be slower than HEAD. */ - bool isValidPathUncached(const Path & storePath) override + bool isValidPathUncached(const StorePath & storePath) override { try { queryPathInfo(storePath); @@ -382,9 +382,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); } - PathSet queryAllValidPaths() override + StorePathSet queryAllValidPaths() override { - PathSet paths; + StorePathSet paths; std::string marker; do { @@ -405,7 +405,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore for (auto object : contents) { auto & key = object.GetKey(); if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; - paths.insert(storeDir + "/" + key.substr(0, key.size() - 8)); + paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown")); } marker = res.GetNextMarker(); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index b9d0c41ea..42ee06501 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -38,7 +38,7 @@ public: bool sameMachine() override { return false; } - void narFromPath(const Path & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink) override; ref<FSAccessor> getFSAccessor() override; @@ -66,10 +66,10 @@ private: }; }; -void SSHStore::narFromPath(const Path & path, Sink & sink) +void SSHStore::narFromPath(const StorePath & path, Sink & sink) { auto conn(connections->get()); - conn->to << wopNarFromPath << path; + conn->to << wopNarFromPath << printStorePath(path); conn->processStderr(); copyNAR(conn->from, sink); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f134f7967..0411db517 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -27,13 +27,6 @@ bool Store::isStorePath(const Path & path) const } -void Store::assertStorePath(const Path & path) const -{ - if (!isStorePath(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); -} - - Path Store::toStorePath(const Path & path) const { if (!isInStore(path)) @@ -60,17 +53,9 @@ Path Store::followLinksToStore(const Path & _path) const } -Path Store::followLinksToStorePath(const Path & path) const +StorePath Store::followLinksToStorePath(const Path & path) const { - return toStorePath(followLinksToStore(path)); -} - - -string storePathToName(const Path & path) -{ - auto base = baseNameOf(path); - assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-')); - return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1); + return parseStorePath(toStorePath(followLinksToStore(path))); } @@ -82,41 +67,6 @@ string storePathToHash(const Path & path) } -void checkStoreName(const string & name) -{ - string validChars = "+-._?="; - - auto baseError = format("The path name '%2%' is invalid: %3%. " - "Path names are alphanumeric and can include the symbols %1% " - "and must not begin with a period. " - "Note: If '%2%' is a source file and you cannot rename it on " - "disk, 'builtins.path { name = ... }' can be used to give it an " - "alternative name.") % validChars % name; - - if (name.empty()) - throw Error(baseError % "it is an empty string"); - - /* Disallow names starting with a dot for possible security - reasons (e.g., "." and ".."). */ - if (name[0] == '.') - throw Error(baseError % "it is illegal to start the name with a period"); - - /* Disallow names longer than 211 characters. ext4’s max is 256, - but we need extra space for the hash and .chroot extensions. */ - if (name.length() > 211) - throw Error(baseError % "name must be less than 212 characters"); - - for (auto & i : name) - if (!((i >= 'A' && i <= 'Z') || - (i >= 'a' && i <= 'z') || - (i >= '0' && i <= '9') || - validChars.find(i) != string::npos)) - { - throw Error(baseError % (format("the '%1%' character is invalid") % i)); - } -} - - /* Store paths have the following form: <store>/<h>-<name> @@ -188,43 +138,48 @@ void checkStoreName(const string & name) */ -Path Store::makeStorePath(const string & type, - const Hash & hash, const string & name) const +StorePath Store::makeStorePath(const string & type, + const Hash & hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name; - - checkStoreName(name); - - return storeDir + "/" - + compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) - + "-" + name; + string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name); + auto h = compressHash(hashString(htSHA256, s), 20); + return StorePath::make(h.hash, name); } -Path Store::makeOutputPath(const string & id, - const Hash & hash, const string & name) const +StorePath Store::makeOutputPath(const string & id, + const Hash & hash, std::string_view name) const { return makeStorePath("output:" + id, hash, - name + (id == "out" ? "" : "-" + id)); + std::string(name) + (id == "out" ? "" : "-" + id)); } -static std::string makeType(string && type, const PathSet & references) +static std::string makeType( + const Store & store, + string && type, + const StorePathSet & references, + bool hasSelfReference = false) { for (auto & i : references) { type += ":"; - type += i; + type += store.printStorePath(i); } + if (hasSelfReference) type += ":self"; return std::move(type); } -Path Store::makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name, const PathSet & references) const +StorePath Store::makeFixedOutputPath( + bool recursive, + const Hash & hash, + std::string_view name, + const StorePathSet & references, + bool hasSelfReference) const { if (hash.type == htSHA256 && recursive) { - return makeStorePath(makeType("source", references), hash, name); + return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, @@ -234,28 +189,27 @@ Path Store::makeFixedOutputPath(bool recursive, } -Path Store::makeTextPath(const string & name, const Hash & hash, - const PathSet & references) const +StorePath Store::makeTextPath(std::string_view name, const Hash & hash, + const StorePathSet & references) const { assert(hash.type == htSHA256); /* Stuff the references (if any) into the type. This is a bit hacky, but we can't put them in `s' since that would be ambiguous. */ - return makeStorePath(makeType("text", references), hash, name); + return makeStorePath(makeType(*this, "text", references), hash, name); } -std::pair<Path, Hash> Store::computeStorePathForPath(const string & name, +std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const { Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - Path dstPath = makeFixedOutputPath(recursive, h, name); - return std::pair<Path, Hash>(dstPath, h); + return std::make_pair(makeFixedOutputPath(recursive, h, name), h); } -Path Store::computeStorePathForText(const string & name, const string & s, - const PathSet & references) const +StorePath Store::computeStorePathForText(const string & name, const string & s, + const StorePathSet & references) const { return makeTextPath(name, hashString(htSHA256, s), references); } @@ -274,11 +228,9 @@ std::string Store::getUri() } -bool Store::isValidPath(const Path & storePath) +bool Store::isValidPath(const StorePath & storePath) { - assertStorePath(storePath); - - auto hashPart = storePathToHash(storePath); + auto hashPart = storePathToHash(printStorePath(storePath)); { auto state_(state.lock()); @@ -312,7 +264,7 @@ bool Store::isValidPath(const Path & storePath) /* Default implementation for stores that only implement queryPathInfoUncached(). */ -bool Store::isValidPathUncached(const Path & path) +bool Store::isValidPathUncached(const StorePath & path) { try { queryPathInfo(path); @@ -323,7 +275,7 @@ bool Store::isValidPathUncached(const Path & path) } -ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) +ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath) { std::promise<ref<const ValidPathInfo>> promise; @@ -340,22 +292,20 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) } -void Store::queryPathInfo(const Path & storePath, +void Store::queryPathInfo(const StorePath & storePath, Callback<ref<const ValidPathInfo>> callback) noexcept { std::string hashPart; try { - assertStorePath(storePath); - - hashPart = storePathToHash(storePath); + hashPart = storePathToHash(printStorePath(storePath)); { auto res = state.lock()->pathInfoCache.get(hashPart); if (res) { stats.narInfoReadAverted++; if (!*res) - throw InvalidPath(format("path '%s' is not valid") % storePath); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); return callback(ref<const ValidPathInfo>(*res)); } } @@ -369,8 +319,8 @@ void Store::queryPathInfo(const Path & storePath, state_->pathInfoCache.upsert(hashPart, res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); if (res.first == NarInfoDiskCache::oInvalid || - (res.second->path != storePath && storePathToName(storePath) != "")) - throw InvalidPath(format("path '%s' is not valid") % storePath); + res.second->path != storePath) + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } return callback(ref<const ValidPathInfo>(res.second)); } @@ -381,7 +331,7 @@ void Store::queryPathInfo(const Path & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { + {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -394,9 +344,7 @@ void Store::queryPathInfo(const Path & storePath, state_->pathInfoCache.upsert(hashPart, info); } - if (!info - || (info->path != storePath && storePathToName(storePath) != "")) - { + if (!info || info->path != parseStorePath(storePath)) { stats.narInfoMissing++; throw InvalidPath("path '%s' is not valid", storePath); } @@ -407,27 +355,27 @@ void Store::queryPathInfo(const Path & storePath, } -PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) +StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { struct State { size_t left; - PathSet valid; + StorePathSet valid; std::exception_ptr exc; }; - Sync<State> state_(State{paths.size(), PathSet()}); + Sync<State> state_(State{paths.size(), StorePathSet()}); std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path ) { + auto doQuery = [&](const Path & path) { checkInterrupt(); - queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(path); + state->valid.insert(parseStorePath(path)); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -439,7 +387,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, path)); + pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME pool.process(); @@ -447,7 +395,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti auto state(state_.lock()); if (!state->left) { if (state->exc) std::rethrow_exception(state->exc); - return state->valid; + return std::move(state->valid); } state.wait(wakeup); } @@ -457,13 +405,13 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string Store::makeValidityRegistration(const PathSet & paths, +string Store::makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash) { string s = ""; for (auto & i : paths) { - s += i + "\n"; + s += printStorePath(i) + "\n"; auto info = queryPathInfo(i); @@ -472,31 +420,30 @@ string Store::makeValidityRegistration(const PathSet & paths, s += (format("%1%\n") % info->narSize).str(); } - Path deriver = showDerivers ? info->deriver : ""; + auto deriver = showDerivers && info->deriver ? printStorePath(*info->deriver) : ""; s += deriver + "\n"; s += (format("%1%\n") % info->references.size()).str(); for (auto & j : info->references) - s += j + "\n"; + s += printStorePath(j) + "\n"; } return s; } -void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, +void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid) { auto jsonList = jsonOut.list(); - for (auto storePath : storePaths) { + for (auto & storePath : storePaths) { auto jsonPath = jsonList.object(); - jsonPath.attr("path", storePath); + jsonPath.attr("path", printStorePath(storePath)); try { auto info = queryPathInfo(storePath); - storePath = info->path; jsonPath .attr("narHash", info->narHash.to_string()) @@ -505,7 +452,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths { auto jsonRefs = jsonPath.list("references"); for (auto & ref : info->references) - jsonRefs.elem(ref); + jsonRefs.elem(printStorePath(ref)); } if (info->ca != "") @@ -514,14 +461,14 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths std::pair<uint64_t, uint64_t> closureSizes; if (showClosureSize) { - closureSizes = getClosureSize(storePath); + closureSizes = getClosureSize(info->path); jsonPath.attr("closureSize", closureSizes.first); } if (includeImpureInfo) { - if (info->deriver != "") - jsonPath.attr("deriver", info->deriver); + if (info->deriver) + jsonPath.attr("deriver", printStorePath(*info->deriver)); if (info->registrationTime) jsonPath.attr("registrationTime", info->registrationTime); @@ -557,10 +504,10 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths } -std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath) +std::pair<uint64_t, uint64_t> Store::getClosureSize(const StorePath & storePath) { uint64_t totalNarSize = 0, totalDownloadSize = 0; - PathSet closure; + StorePathSet closure; computeFSClosure(storePath, closure, false, false); for (auto & p : closure) { auto info = queryPathInfo(p); @@ -584,30 +531,34 @@ const Store::Stats & Store::getStats() } -void Store::buildPaths(const PathSet & paths, BuildMode buildMode) +void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) { - for (auto & path : paths) - if (isDerivation(path)) + StorePathSet paths2; + + for (auto & path : paths) { + if (path.path.isDerivation()) unsupported("buildPaths"); + paths2.insert(path.path.clone()); + } - if (queryValidPaths(paths).size() != paths.size()) + if (queryValidPaths(paths2).size() != paths2.size()) unsupported("buildPaths"); } void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) + const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { auto srcUri = srcStore->getUri(); auto dstUri = dstStore->getUri(); Activity act(*logger, lvlInfo, actCopyPath, srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) + ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri) : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); + ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri) + : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri), + {srcStore->printStorePath(storePath), srcUri, dstUri}); PushActivity pact(act.id); auto info = srcStore->queryPathInfo(storePath); @@ -640,23 +591,23 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, total += len; act.progress(total, info->narSize); }); - srcStore->narFromPath({storePath}, wrapperSink); + srcStore->narFromPath(storePath, wrapperSink); }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); }); dstStore->addToStore(*info, *source, repair, checkSigs); } -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { - PathSet valid = dstStore->queryValidPaths(storePaths, substitute); + auto valid = dstStore->queryValidPaths(storePaths, substitute); PathSet missing; for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(path); + if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); if (missing.empty()) return; @@ -677,23 +628,25 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa PathSet(missing.begin(), missing.end()), [&](const Path & storePath) { - if (dstStore->isValidPath(storePath)) { + if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { nrDone++; showProgress(); return PathSet(); } - auto info = srcStore->queryPathInfo(storePath); + auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); - return info->references; + return srcStore->printStorePathSet(info->references); }, - [&](const Path & storePath) { + [&](const Path & storePathS) { checkInterrupt(); + auto storePath = dstStore->parseStorePath(storePathS); + if (!dstStore->isValidPath(storePath)) { MaintainCount<decltype(nrRunning)> mc(nrRunning); showProgress(); @@ -703,7 +656,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what()); + logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); showProgress(); return; } @@ -716,20 +669,36 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, + const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { - PathSet closure; - srcStore->computeFSClosure({storePaths}, closure); + StorePathSet closure; + srcStore->computeFSClosure(storePaths, closure); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); } -ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) +ValidPathInfo::ValidPathInfo(const ValidPathInfo & other) + : path(other.path.clone()) + , deriver(other.deriver ? other.deriver->clone(): std::optional<StorePath>{}) + , narHash(other.narHash) + , references(cloneStorePathSet(other.references)) + , registrationTime(other.registrationTime) + , narSize(other.narSize) + , id(other.id) + , ultimate(other.ultimate) + , sigs(other.sigs) + , ca(other.ca) { - ValidPathInfo info; - getline(str, info.path); - if (str.eof()) { info.path = ""; return info; } +} + + +std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven) +{ + std::string path; + getline(str, path); + if (str.eof()) { return {}; } + ValidPathInfo info(store.parseStorePath(path)); if (hashGiven) { string s; getline(str, s); @@ -737,16 +706,29 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) getline(str, s); if (!string2Int(s, info.narSize)) throw Error("number expected"); } - getline(str, info.deriver); + std::string deriver; + getline(str, deriver); + if (deriver != "") info.deriver = store.parseStorePath(deriver); string s; int n; getline(str, s); if (!string2Int(s, n)) throw Error("number expected"); while (n--) { getline(str, s); - info.references.insert(s); + info.references.insert(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); - return info; + return std::optional<ValidPathInfo>(std::move(info)); +} + + +std::string Store::showPaths(const StorePathSet & paths) +{ + std::string s; + for (auto & i : paths) { + if (s.size() != 0) s += ", "; + s += "'" + printStorePath(i) + "'"; + } + return s; } @@ -756,34 +738,34 @@ string showPaths(const PathSet & paths) } -std::string ValidPathInfo::fingerprint() const +std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0 || !narHash) - throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known") - % path); + throw Error("cannot calculate fingerprint of path '%s' because its size/hash is not known", + store.printStorePath(path)); return - "1;" + path + ";" + "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", references); + + concatStringsSep(",", store.printStorePathSet(references)); } -void ValidPathInfo::sign(const SecretKey & secretKey) +void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) { - sigs.insert(secretKey.signDetached(fingerprint())); + sigs.insert(secretKey.signDetached(fingerprint(store))); } bool ValidPathInfo::isContentAddressed(const Store & store) const { auto warn = [&]() { - printError(format("warning: path '%s' claims to be content-addressed but isn't") % path); + printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); }; if (hasPrefix(ca, "text:")) { Hash hash(std::string(ca, 5)); - if (store.makeTextPath(storePathToName(path), hash, references) == path) + if (store.makeTextPath(path.name(), hash, references) == path) return true; else warn(); @@ -792,9 +774,13 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const else if (hasPrefix(ca, "fixed:")) { bool recursive = ca.compare(6, 2, "r:") == 0; Hash hash(std::string(ca, recursive ? 8 : 6)); - auto refs = references; - replaceInSet(refs, path, std::string("self")); - if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path) + auto refs = cloneStorePathSet(references); + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path) return true; else warn(); @@ -810,15 +796,15 @@ size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & pu size_t good = 0; for (auto & sig : sigs) - if (checkSignature(publicKeys, sig)) + if (checkSignature(store, publicKeys, sig)) good++; return good; } -bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const { - return verifyDetached(fingerprint(), sig, publicKeys); + return verifyDetached(fingerprint(store), sig, publicKeys); } @@ -826,7 +812,7 @@ Strings ValidPathInfo::shortRefs() const { Strings refs; for (auto & r : references) - refs.push_back(baseNameOf(r)); + refs.push_back(std::string(r.to_string())); return refs; } @@ -939,7 +925,7 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr<Store> { - switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { + switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) { case tDaemon: return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); case tLocal: { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e2e720fc4..13ca3fef2 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,6 @@ #pragma once +#include "path.hh" #include "hash.hh" #include "serialise.hh" #include "crypto.hh" @@ -43,14 +44,11 @@ enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; -/* Size of the hash part of store paths, in base-32 characters. */ -const size_t storePathHashLen = 32; // i.e. 160 bits - /* Magic header of exportPath() output (obsolete). */ const uint32_t exportMagic = 0x4558494e; -typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots; +typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots; struct GCOptions @@ -84,7 +82,7 @@ struct GCOptions bool ignoreLiveness{false}; /* For `gcDeleteSpecific', the paths to delete. */ - PathSet pathsToDelete; + StorePathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; @@ -105,21 +103,21 @@ struct GCResults struct SubstitutablePathInfo { - Path deriver; - PathSet references; + std::optional<StorePath> deriver; + StorePathSet references; unsigned long long downloadSize; /* 0 = unknown or inapplicable */ unsigned long long narSize; /* 0 = unknown */ }; -typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos; +typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; struct ValidPathInfo { - Path path; - Path deriver; + StorePath path; + std::optional<StorePath> deriver; Hash narHash; - PathSet references; + StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -144,7 +142,7 @@ struct ValidPathInfo Ideally, the content-addressability assertion would just be a Boolean, and the store path would be computed from - ‘storePathToName(path)’, ‘narHash’ and ‘references’. However, + the name component, ‘narHash’ and ‘references’. However, 1) we've accumulated several types of content-addressed paths over the years; and 2) fixed-output derivations support multiple hash algorithms and serialisation methods (flat file @@ -172,9 +170,9 @@ struct ValidPathInfo the NAR, and the sorted references. The size field is strictly speaking superfluous, but might prevent endless/excessive data attacks. */ - std::string fingerprint() const; + std::string fingerprint(const Store & store) const; - void sign(const SecretKey & secretKey); + void sign(const Store & store, const SecretKey & secretKey); /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -187,10 +185,13 @@ struct ValidPathInfo size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; /* Verify a single signature. */ - bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; + bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; Strings shortRefs() const; + ValidPathInfo(StorePath && path) : path(std::move(path)) { } + explicit ValidPathInfo(const ValidPathInfo & other); + virtual ~ValidPathInfo() { } }; @@ -241,6 +242,23 @@ struct BuildResult }; +struct StorePathWithOutputs +{ + StorePath path; + std::set<std::string> outputs; + + StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {}) + : path(path.clone()), outputs(outputs) + { } + + StorePathWithOutputs(const StorePathWithOutputs & other) + : path(other.path.clone()), outputs(other.outputs) + { } + + std::string to_string(const Store & store) const; +}; + + class Store : public std::enable_shared_from_this<Store>, public Config { public: @@ -259,6 +277,7 @@ protected: struct State { + // FIXME: fix key LRUCache<std::string, std::shared_ptr<const ValidPathInfo>> pathInfoCache; }; @@ -274,6 +293,24 @@ public: virtual std::string getUri() = 0; + StorePath parseStorePath(std::string_view path) const; + + std::string printStorePath(const StorePath & path) const; + + // FIXME: remove + StorePathSet parseStorePathSet(const PathSet & paths) const; + + PathSet printStorePathSet(const StorePathSet & path) const; + + /* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path + and the outputs. */ + StorePathWithOutputs parseDrvPathWithOutputs(const string & s); + + /* Display a set of paths in human-readable form (i.e., between quotes + and separated by commas). */ + std::string showPaths(const StorePathSet & paths); + /* Return true if ‘path’ is in the Nix store (but not the Nix store itself). */ bool isInStore(const Path & path) const; @@ -282,9 +319,6 @@ public: the Nix store. */ bool isStorePath(const Path & path) const; - /* Throw an exception if ‘path’ is not a store path. */ - void assertStorePath(const Path & path) const; - /* Chop off the parts after the top-level store name, e.g., /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ Path toStorePath(const Path & path) const; @@ -294,26 +328,27 @@ public: /* Same as followLinksToStore(), but apply toStorePath() to the result. */ - Path followLinksToStorePath(const Path & path) const; + StorePath followLinksToStorePath(const Path & path) const; /* Constructs a unique store path name. */ - Path makeStorePath(const string & type, - const Hash & hash, const string & name) const; + StorePath makeStorePath(const string & type, + const Hash & hash, std::string_view name) const; - Path makeOutputPath(const string & id, - const Hash & hash, const string & name) const; + StorePath makeOutputPath(const string & id, + const Hash & hash, std::string_view name) const; - Path makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name, - const PathSet & references = {}) const; + StorePath makeFixedOutputPath(bool recursive, + const Hash & hash, std::string_view name, + const StorePathSet & references = {}, + bool hasSelfReference = false) const; - Path makeTextPath(const string & name, const Hash & hash, - const PathSet & references) const; + StorePath makeTextPath(std::string_view name, const Hash & hash, + const StorePathSet & references) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ - std::pair<Path, Hash> computeStorePathForPath(const string & name, + std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; @@ -331,21 +366,21 @@ public: simply yield a different store path, so other users wouldn't be affected), but it has some backwards compatibility issues (the hashing scheme changes), so I'm not doing that for now. */ - Path computeStorePathForText(const string & name, const string & s, - const PathSet & references) const; + StorePath computeStorePathForText(const string & name, const string & s, + const StorePathSet & references) const; /* Check whether a path is valid. */ - bool isValidPath(const Path & path); + bool isValidPath(const StorePath & path); protected: - virtual bool isValidPathUncached(const Path & path); + virtual bool isValidPathUncached(const StorePath & path); public: /* Query which of the given paths is valid. Optionally, try to substitute missing paths. */ - virtual PathSet queryValidPaths(const PathSet & paths, + virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); /* Query the set of all valid paths. Note that for some store @@ -353,54 +388,54 @@ public: (i.e. you'll get /nix/store/<hash> rather than /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the full store path. */ - virtual PathSet queryAllValidPaths() + virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } /* Query information about a valid path. It is permitted to omit the name part of the store path. */ - ref<const ValidPathInfo> queryPathInfo(const Path & path); + ref<const ValidPathInfo> queryPathInfo(const StorePath & path); /* Asynchronous version of queryPathInfo(). */ - void queryPathInfo(const Path & path, + void queryPathInfo(const StorePath & path, Callback<ref<const ValidPathInfo>> callback) noexcept; protected: - virtual void queryPathInfoUncached(const Path & path, + virtual void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0; public: /* Queries the set of incoming FS references for a store path. The result is not cleared. */ - virtual void queryReferrers(const Path & path, PathSet & referrers) + virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) { unsupported("queryReferrers"); } /* Return all currently valid derivations that have `path' as an output. (Note that the result of `queryDeriver()' is the derivation that was actually used to produce `path', which may not exist anymore.) */ - virtual PathSet queryValidDerivers(const Path & path) { return {}; }; + virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /* Query the outputs of the derivation denoted by `path'. */ - virtual PathSet queryDerivationOutputs(const Path & path) + virtual StorePathSet queryDerivationOutputs(const StorePath & path) { unsupported("queryDerivationOutputs"); } /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const Path & path) + virtual StringSet queryDerivationOutputNames(const StorePath & path) { unsupported("queryDerivationOutputNames"); } /* Query the full store path given the hash part of a valid store - path, or "" if the path doesn't exist. */ - virtual Path queryPathFromHashPart(const string & hashPart) = 0; + path, or empty if the path doesn't exist. */ + virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0; /* Query which of the given paths have substitutes. */ - virtual PathSet querySubstitutablePaths(const PathSet & paths) { return {}; }; + virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download sizes) of a set of paths. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const PathSet & paths, + virtual void querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) { return; }; virtual bool wantMassQuery() { return false; } @@ -419,12 +454,12 @@ public: validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see libutil/archive.hh). */ - virtual Path addToStore(const string & name, const Path & srcPath, + virtual StorePath addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; // FIXME: remove? - virtual Path addToStoreFromDump(const string & dump, const string & name, + virtual StorePath addToStoreFromDump(const string & dump, const string & name, bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); @@ -432,11 +467,11 @@ public: /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ - virtual Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair = NoRepair) = 0; + virtual StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair = NoRepair) = 0; /* Write a NAR dump of a store path. */ - virtual void narFromPath(const Path & path, Sink & sink) = 0; + virtual void narFromPath(const StorePath & path, Sink & sink) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If @@ -446,22 +481,24 @@ public: output paths can be created by running the builder, after recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal); + virtual void buildPaths( + const std::vector<StorePathWithOutputs> & paths, + BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). Note that ‘drvPath’ is only used for informational purposes. */ - virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) = 0; /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the path). */ - virtual void ensurePath(const Path & path) = 0; + virtual void ensurePath(const StorePath & path) = 0; /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ - virtual void addTempRoot(const Path & path) + virtual void addTempRoot(const StorePath & path) { unsupported("addTempRoot"); } /* Add an indirect root, which is merely a symlink to `path' from @@ -507,7 +544,7 @@ public: /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or `nix-store --register-validity'. */ - string makeValidityRegistration(const PathSet & paths, + string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); /* Write a JSON representation of store path metadata, such as the @@ -515,14 +552,14 @@ public: variable elements such as the registration time are included. If ‘showClosureSize’ is true, the closure size of each path is included. */ - void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, + void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid = DisallowInvalid); /* Return the size of the closure of the specified path, that is, the sum of the size of the NAR serialisation of each path in the closure. */ - std::pair<uint64_t, uint64_t> getClosureSize(const Path & storePath); + std::pair<uint64_t, uint64_t> getClosureSize(const StorePath & storePath); /* Optimise the disk space usage of the Nix store by hard-linking files with the same contents. */ @@ -538,14 +575,14 @@ public: /* Add signatures to the specified store path. The signatures are not verified. */ - virtual void addSignatures(const Path & storePath, const StringSet & sigs) + virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) { unsupported("addSignatures"); } /* Utility functions. */ /* Read a derivation, after ensuring its existence through ensurePath(). */ - Derivation derivationFromPath(const Path & drvPath); + Derivation derivationFromPath(const StorePath & drvPath); /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly @@ -554,36 +591,36 @@ public: `storePath' is returned; that is, the closures under the `referrers' relation instead of the `references' relation is returned. */ - virtual void computeFSClosure(const PathSet & paths, - PathSet & out, bool flipDirection = false, + virtual void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); - void computeFSClosure(const Path & path, - PathSet & out, bool flipDirection = false, + void computeFSClosure(const StorePath & path, + StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - virtual void queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize); /* Sort a set of paths topologically under the references relation. If p refers to q, then p precedes q in this list. */ - Paths topoSortPaths(const PathSet & paths); + StorePaths topoSortPaths(const StorePathSet & paths); /* Export multiple paths in the format expected by ‘nix-store --import’. */ - void exportPaths(const Paths & paths, Sink & sink); + void exportPaths(const StorePathSet & paths, Sink & sink); - void exportPath(const Path & path, Sink & sink); + void exportPath(const StorePath & path, Sink & sink); /* Import a sequence of NAR dumps created by exportPaths() into the Nix store. Optionally, the contents of the NARs are preloaded into the specified FS accessor to speed up subsequent access. */ - Paths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, + StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs = CheckSigs); struct Stats @@ -607,7 +644,7 @@ public: /* Return the build log of the specified store path, if available, or null otherwise. */ - virtual std::shared_ptr<std::string> getBuildLog(const Path & path) + virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path) { return nullptr; } /* Hack to allow long-running processes like hydra-queue-runner to @@ -673,11 +710,11 @@ public: LocalFSStore(const Params & params); - void narFromPath(const Path & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink) override; ref<FSAccessor> getFSAccessor() override; /* Register a permanent GC root. */ - Path addPermRoot(const Path & storePath, + Path addPermRoot(const StorePath & storePath, const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); virtual Path getRealStoreDir() { return storeDir; } @@ -688,25 +725,17 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } - std::shared_ptr<std::string> getBuildLog(const Path & path) override; + std::shared_ptr<std::string> getBuildLog(const StorePath & path) override; }; -/* Extract the name part of the given store path. */ -string storePathToName(const Path & path); - /* Extract the hash part of the given store path. */ string storePathToHash(const Path & path); -/* Check whether ‘name’ is a valid store path name part, i.e. contains - only the characters [a-zA-Z0-9\+\-\.\_\?\=] and doesn't start with - a dot. */ -void checkStoreName(const string & name); - /* Copy a path from one store to another. */ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); /* Copy store paths from one store to another. The paths may be copied @@ -714,7 +743,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, (i.e. if A is a reference of B, then A is copied before B), but the set of store paths is not automatically closed; use copyClosure() for that. */ -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -722,7 +751,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa /* Copy the closure of the specified paths from one store to another. */ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, + const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -805,7 +834,9 @@ struct RegisterStoreImplementation string showPaths(const PathSet & paths); -ValidPathInfo decodeValidPathInfo(std::istream & str, +std::optional<ValidPathInfo> decodeValidPathInfo( + const Store & store, + std::istream & str, bool hashGiven = false); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 6762b609d..857d54d99 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -65,8 +65,9 @@ typedef enum { class Store; struct Source; -Path readStorePath(Store & store, Source & from); -template<class T> T readStorePaths(Store & store, Source & from); +template<class T> T readStorePaths(const Store & store, Source & from); + +void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); } |