diff options
Diffstat (limited to 'src')
99 files changed, 2583 insertions, 2164 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 27dd35784..69d1c6f7e 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -88,7 +88,7 @@ static int _main(int argc, char * * argv) return 0; } - string drvPath; + std::optional<StorePath> drvPath; string storeUri; while (true) { @@ -100,7 +100,7 @@ static int _main(int argc, char * * argv) auto amWilling = readInt(source); auto neededSystem = readString(source); - source >> drvPath; + drvPath = store->parseStorePath(readString(source)); auto requiredFeatures = readStrings<std::set<std::string>>(source); auto canBuildLocally = amWilling @@ -236,26 +236,27 @@ connected: { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); - copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, substitute); + copyPaths(store, ref<Store>(sshStore), store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); } uploadLock = -1; - BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath))); - drv.inputSrcs = inputs; + BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string()))); + drv.inputSrcs = store->parseStorePathSet(inputs); - auto result = sshStore->buildDerivation(drvPath, drv); + auto result = sshStore->buildDerivation(*drvPath, drv); if (!result.success()) - throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg); + throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); - PathSet missing; + StorePathSet missing; for (auto & path : outputs) - if (!store->isValidPath(path)) missing.insert(path); + if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path)); if (!missing.empty()) { Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); - store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */ + for (auto & i : missing) + store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */ copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index ff7bce45e..cddbce20d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -44,6 +44,19 @@ static char * dupString(const char * s) } +static char * dupStringWithLen(const char * s, size_t size) +{ + char * t; +#if HAVE_BOEHMGC + t = GC_STRNDUP(s, size); +#else + t = strndup(s, size); +#endif + if (!t) throw std::bad_alloc(); + return t; +} + + static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) { checkInterrupt(); @@ -358,10 +371,10 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) auto path = r.second; if (store->isInStore(r.second)) { - PathSet closure; - store->computeFSClosure(store->toStorePath(r.second), closure); + StorePathSet closure; + store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure); for (auto & path : closure) - allowedPaths->insert(path); + allowedPaths->insert(store->printStorePath(path)); } else allowedPaths->insert(r.second); } @@ -585,9 +598,11 @@ void mkString(Value & v, const char * s) } -Value & mkString(Value & v, const string & s, const PathSet & context) +Value & mkString(Value & v, std::string_view s, const PathSet & context) { - mkString(v, s.c_str()); + v.type = tString; + v.string.s = dupStringWithLen(s.data(), s.size()); + v.string.context = 0; if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) @@ -1131,10 +1146,9 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) { - std::optional<FunctionCallTrace> trace; - if (evalSettings.traceFunctionCalls) { - trace.emplace(pos); - } + std::unique_ptr<FunctionCallTrace> trace; + if (evalSettings.traceFunctionCalls) + trace = std::make_unique<FunctionCallTrace>(pos); forceValue(fun, pos); @@ -1680,15 +1694,16 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) throwEvalError("file names are not allowed to end in '%1%'", drvExtension); Path dstPath; - if (srcToStore[path] != "") - dstPath = srcToStore[path]; + auto i = srcToStore.find(path); + if (i != srcToStore.end()) + dstPath = store->printStorePath(i->second); else { - dstPath = settings.readOnlyMode - ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first - : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); - srcToStore[path] = dstPath; - printMsg(lvlChatty, format("copied source '%1%' -> '%2%'") - % path % dstPath); + auto p = settings.readOnlyMode + ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first + : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); + dstPath = store->printStorePath(p); + srcToStore.insert_or_assign(path, std::move(p)); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); } context.insert(dstPath); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index baca7f03f..526d8b198 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -17,6 +17,7 @@ namespace nix { class Store; class EvalState; +struct StorePath; enum RepairFlag : bool; namespace flake { @@ -46,14 +47,14 @@ struct Env }; -Value & mkString(Value & v, const string & s, const PathSet & context = PathSet()); +Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet()); void copyContext(const Value & v, PathSet & context); /* Cache for calls to addToStore(); maps source paths to the store paths. */ -typedef std::map<Path, Path> SrcToStore; +typedef std::map<Path, StorePath> SrcToStore; std::ostream & operator << (std::ostream & str, const Value & v); diff --git a/src/libexpr/flake/eval-cache.cc b/src/libexpr/flake/eval-cache.cc index b32d502f7..8d01ef0fc 100644 --- a/src/libexpr/flake/eval-cache.cc +++ b/src/libexpr/flake/eval-cache.cc @@ -77,7 +77,7 @@ void EvalCache::addDerivation( (fingerprint.hash, fingerprint.hashSize) (attrPath) (ValueType::Derivation) - (drv.drvPath + " " + drv.outPath + " " + drv.outputName).exec(); + (std::string(drv.drvPath.to_string()) + " " + std::string(drv.outPath.to_string()) + " " + drv.outputName).exec(); } std::optional<EvalCache::Derivation> EvalCache::getDerivation( @@ -104,7 +104,7 @@ std::optional<EvalCache::Derivation> EvalCache::getDerivation( debug("evaluation cache hit for '%s'", attrPath); - return Derivation { ss[0], ss[1], ss[2] }; + return Derivation { StorePath::fromBaseName(ss[0]), StorePath::fromBaseName(ss[1]), ss[2] }; } EvalCache & EvalCache::singleton() diff --git a/src/libexpr/flake/eval-cache.hh b/src/libexpr/flake/eval-cache.hh index 03aea142e..f81d48ba5 100644 --- a/src/libexpr/flake/eval-cache.hh +++ b/src/libexpr/flake/eval-cache.hh @@ -2,6 +2,7 @@ #include "sync.hh" #include "flake.hh" +#include "path.hh" namespace nix { struct SQLite; struct SQLiteStmt; } @@ -19,8 +20,8 @@ public: struct Derivation { - Path drvPath; - Path outPath; + StorePath drvPath; + StorePath outPath; std::string outputName; }; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 80726a257..a644f6ad3 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -163,7 +163,7 @@ static SourceInfo fetchInput(EvalState & state, const FlakeRef & resolvedRef) SourceInfo info(ref); info.storePath = gitInfo.storePath; info.revCount = gitInfo.revCount; - info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.narHash = state.store->queryPathInfo(state.store->parseStorePath(info.storePath))->narHash; info.lastModified = gitInfo.lastModified; return info; }; @@ -212,7 +212,7 @@ static Flake getFlake(EvalState & state, const FlakeRef & originalRef, refMap.push_back({originalRef, resolvedRef}); refMap.push_back({flakeRef, resolvedRef}); - state.store->assertStorePath(sourceInfo.storePath); + state.store->parseStorePath(sourceInfo.storePath); if (state.allowedPaths) state.allowedPaths->insert(state.store->toRealPath(sourceInfo.storePath)); @@ -334,7 +334,7 @@ static SourceInfo getNonFlake(EvalState & state, const FlakeRef & originalRef, refMap.push_back({originalRef, resolvedRef}); refMap.push_back({flakeRef, resolvedRef}); - state.store->assertStorePath(sourceInfo.storePath); + state.store->parseStorePath(sourceInfo.storePath); if (state.allowedPaths) state.allowedPaths->insert(sourceInfo.storePath); @@ -490,7 +490,7 @@ void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateL static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs) { auto & path = sourceInfo.storePath; - assert(state.store->isValidPath(path)); + assert(state.store->isValidPath(state.store->parseStorePath(path))); mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); if (sourceInfo.resolvedRef.rev) { @@ -542,7 +542,7 @@ static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, V state.mkAttrs(v, 8); - assert(state.store->isValidPath(sourceInfo.storePath)); + assert(state.store->isValidPath(state.store->parseStorePath(sourceInfo.storePath))); mkString(*state.allocAttr(v, state.sOutPath), sourceInfo.storePath, {sourceInfo.storePath}); diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 8e90e5989..ff7c725cb 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -161,7 +161,7 @@ FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) } while (true) { if (pathExists(d.path + "/.git")) break; - subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir); + subdir = std::string(baseNameOf(d.path)) + (subdir.empty() ? "" : "/" + subdir); d.path = dirOf(d.path); if (d.path == "/") throw MissingFlake("path '%s' is not a flake (because it does not reference a Git repository)", uri); diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 5693e57dc..93d4ae946 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -26,7 +26,7 @@ nlohmann::json LockedInput::toJson() const Path LockedInput::computeStorePath(Store & store) const { - return store.makeFixedOutputPath(true, narHash, "source"); + return store.printStorePath(store.makeFixedOutputPath(true, narHash, "source")); } LockedInputs::LockedInputs(const nlohmann::json & json) diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 4e22a1d77..a3412eab9 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -19,27 +19,27 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs) : state(&state), attrs(nullptr), attrPath("") { - auto spec = parseDrvPathWithOutputs(drvPathWithOutputs); + auto [drvPath, selectedOutputs] = store->parseDrvPathWithOutputs(drvPathWithOutputs); - drvPath = spec.first; + this->drvPath = store->printStorePath(drvPath); auto drv = store->derivationFromPath(drvPath); - name = storePathToName(drvPath); + name = drvPath.name(); - if (spec.second.size() > 1) + if (selectedOutputs.size() > 1) throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs); outputName = - spec.second.empty() - ? get(drv.env, "outputName", "out") - : *spec.second.begin(); + selectedOutputs.empty() + ? get(drv.env, "outputName").value_or("out") + : *selectedOutputs.begin(); auto i = drv.outputs.find(outputName); if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have output '%s'", drvPath, outputName); + throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); - outPath = i->second.path; + outPath = store->printStorePath(i->second.path); } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index a9cb6b7b6..c2225383f 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -11,7 +11,7 @@ libexpr_SOURCES := \ $(d)/lexer-tab.cc \ $(d)/parser-tab.cc -libexpr_LIBS = libutil libstore +libexpr_LIBS = libutil libstore libnixrust libexpr_LDFLAGS = ifneq ($(OS), FreeBSD) diff --git a/src/libexpr/names.cc b/src/libexpr/names.cc index 382088c78..d1c8a6101 100644 --- a/src/libexpr/names.cc +++ b/src/libexpr/names.cc @@ -16,14 +16,14 @@ DrvName::DrvName() a letter. The `version' part is the rest (excluding the separating dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', '2.0.48'). */ -DrvName::DrvName(const string & s) : hits(0) +DrvName::DrvName(std::string_view s) : hits(0) { - name = fullName = s; + name = fullName = std::string(s); for (unsigned int i = 0; i < s.size(); ++i) { /* !!! isalpha/isdigit are affected by the locale. */ if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) { - name = string(s, 0, i); - version = string(s, i + 1); + name = s.substr(0, i); + version = s.substr(i + 1); break; } } diff --git a/src/libexpr/names.hh b/src/libexpr/names.hh index 13c3093e7..00e14b8c7 100644 --- a/src/libexpr/names.hh +++ b/src/libexpr/names.hh @@ -15,7 +15,7 @@ struct DrvName unsigned int hits; DrvName(); - DrvName(const string & s); + DrvName(std::string_view s); bool matches(DrvName & n); private: diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 021204f08..2df9277c9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -44,19 +44,19 @@ std::pair<string, string> decodeContext(const string & s) InvalidPathError::InvalidPathError(const Path & path) : - EvalError(format("path '%1%' is not valid") % path), path(path) {} + EvalError("path '%s' is not valid", path), path(path) {} void EvalState::realiseContext(const PathSet & context) { - PathSet drvs; + std::vector<StorePathWithOutputs> drvs; for (auto & i : context) { - auto [ctx, outputName] = decodeContext(i); - assert(store->isStorePath(ctx)); + auto [ctxS, outputName] = decodeContext(i); + auto ctx = store->parseStorePath(ctxS); if (!store->isValidPath(ctx)) - throw InvalidPathError(ctx); - if (!outputName.empty() && nix::isDerivation(ctx)) { - drvs.insert(ctx + "!" + outputName); + throw InvalidPathError(store->printStorePath(ctx)); + if (!outputName.empty() && ctx.isDerivation()) { + drvs.push_back(StorePathWithOutputs{ctx.clone(), {outputName}}); /* Add the output of this derivation to the allowed paths. */ @@ -64,8 +64,8 @@ void EvalState::realiseContext(const PathSet & context) auto drv = store->derivationFromPath(ctx); DerivationOutputs::iterator i = drv.outputs.find(outputName); if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named '%s'", ctx, outputName); - allowedPaths->insert(i->second.path); + throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName); + allowedPaths->insert(store->printStorePath(i->second.path)); } } } @@ -73,10 +73,11 @@ void EvalState::realiseContext(const PathSet & context) if (drvs.empty()) return; if (!evalSettings.enableImportFromDerivation) - throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin())); + throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false", + store->printStorePath(drvs.begin()->path)); /* For performance, prefetch all substitute info. */ - PathSet willBuild, willSubstitute, unknown; + StorePathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -100,8 +101,9 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args Path realPath = state.checkSourcePath(state.toRealPath(path, context)); - if (state.store->isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) { - Derivation drv = readDerivation(realPath); + // FIXME + if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) { + Derivation drv = readDerivation(*state.store, realPath); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); @@ -115,7 +117,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args for (const auto & o : drv.outputs) { v2 = state.allocAttr(w, state.symbols.create(o.first)); - mkString(*v2, o.second.path, {"!" + o.first + "!" + path}); + mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path}); outputsVal->listElems()[outputs_index] = state.allocValue(); mkString(*(outputsVal->listElems()[outputs_index++]), o.first); } @@ -676,24 +678,24 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * runs. */ if (path.at(0) == '=') { /* !!! This doesn't work if readOnlyMode is set. */ - PathSet refs; - state.store->computeFSClosure(string(path, 1), refs); + StorePathSet refs; + state.store->computeFSClosure(state.store->parseStorePath(std::string_view(path).substr(1)), refs); for (auto & j : refs) { - drv.inputSrcs.insert(j); - if (isDerivation(j)) - drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j); + drv.inputSrcs.insert(j.clone()); + if (j.isDerivation()) + drv.inputDrvs[j.clone()] = state.store->queryDerivationOutputNames(j); } } /* Handle derivation outputs of the form ‘!<name>!<path>’. */ else if (path.at(0) == '!') { std::pair<string, string> ctx = decodeContext(path); - drv.inputDrvs[ctx.first].insert(ctx.second); + drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second); } /* Otherwise it's a source file. */ else - drv.inputSrcs.insert(path); + drv.inputSrcs.insert(state.store->parseStorePath(path)); } /* Do we have all required attributes? */ @@ -703,10 +705,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName); /* Check whether the derivation name is valid. */ - checkStoreName(drvName); if (isDerivation(drvName)) - throw EvalError(format("derivation names are not allowed to end in '%1%', at %2%") - % drvExtension % posDrvName); + throw EvalError("derivation names are not allowed to end in '%s', at %s", drvExtension, posDrvName); if (outputHash) { /* Handle fixed-output derivations. */ @@ -716,52 +716,51 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); Hash h(*outputHash, ht); - Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); - if (!jsonObject) drv.env["out"] = outPath; - drv.outputs["out"] = DerivationOutput(outPath, - (outputHashRecursive ? "r:" : "") + printHashType(h.type), - h.to_string(Base16, false)); + auto outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); + if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign("out", DerivationOutput(std::move(outPath), + (outputHashRecursive ? "r:" : "") + printHashType(h.type), + h.to_string(Base16, false))); } else { - /* Construct the "masked" store derivation, which is the final - one except that in the list of outputs, the output paths - are empty, and the corresponding environment variables have - an empty value. This ensures that changes in the set of - output names do get reflected in the hash. */ + /* Compute a hash over the "masked" store derivation, which is + the final one except that in the list of outputs, the + output paths are empty strings, and the corresponding + environment variables have an empty value. This ensures + that changes in the set of output names do get reflected in + the hash. */ for (auto & i : outputs) { if (!jsonObject) drv.env[i] = ""; - drv.outputs[i] = DerivationOutput("", "", ""); } - /* Use the masked derivation expression to compute the output - path. */ - Hash h = hashDerivationModulo(*state.store, drv); + Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); - for (auto & i : drv.outputs) - if (i.second.path == "") { - Path outPath = state.store->makeOutputPath(i.first, h, drvName); - if (!jsonObject) drv.env[i.first] = outPath; - i.second.path = outPath; - } + for (auto & i : outputs) { + auto outPath = state.store->makeOutputPath(i, h, drvName); + if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign(i, + DerivationOutput(std::move(outPath), "", "")); + } } /* Write the resulting term into the Nix store directory. */ - Path drvPath = writeDerivation(state.store, drv, drvName, state.repair); + auto drvPath = writeDerivation(state.store, drv, drvName, state.repair); + auto drvPathS = state.store->printStorePath(drvPath); - printMsg(lvlChatty, format("instantiated '%1%' -> '%2%'") - % drvName % drvPath); + printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS); /* Optimisation, but required in read-only mode! because in that case we don't actually write store derivations, so we can't read them later. */ - drvHashes[drvPath] = hashDerivationModulo(*state.store, drv); + drvHashes.insert_or_assign(drvPath.clone(), + hashDerivationModulo(*state.store, Derivation(drv), false)); state.mkAttrs(v, 1 + drv.outputs.size()); - mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath}); + mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); for (auto & i : drv.outputs) { mkString(*state.allocAttr(v, state.symbols.create(i.first)), - i.second.path, {"!" + i.first + "!" + drvPath}); + state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS}); } v.attrs->sort(); } @@ -814,7 +813,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos); Path path2 = state.store->toStorePath(path); if (!settings.readOnlyMode) - state.store->ensurePath(path2); + state.store->ensurePath(state.store->parseStorePath(path2)); context.insert(path2); mkString(v, path, context); } @@ -1010,17 +1009,17 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu string name = state.forceStringNoCtx(*args[0], pos); string contents = state.forceString(*args[1], context, pos); - PathSet refs; + StorePathSet refs; for (auto path : context) { if (path.at(0) != '/') throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); - refs.insert(path); + refs.insert(state.store->parseStorePath(path)); } - Path storePath = settings.readOnlyMode + auto storePath = state.store->printStorePath(settings.readOnlyMode ? state.store->computeStorePathForText(name, contents, refs) - : state.store->addTextToStore(name, contents, refs, state.repair); + : state.store->addTextToStore(name, contents, refs, state.repair)); /* Note: we don't need to add `context' to the context of the result, since `storePath' itself has references to the paths @@ -1060,21 +1059,18 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con return state.forceBool(res, pos); }) : defaultPathFilter; - Path expectedStorePath; - if (expectedHash) { - expectedStorePath = - state.store->makeFixedOutputPath(recursive, expectedHash, name); - } + std::optional<StorePath> expectedStorePath; + if (expectedHash) + expectedStorePath = state.store->makeFixedOutputPath(recursive, expectedHash, name); Path dstPath; - if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { - dstPath = settings.readOnlyMode + if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { + dstPath = state.store->printStorePath(settings.readOnlyMode ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first - : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair); - if (expectedHash && expectedStorePath != dstPath) { - throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path); - } + : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair)); + if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) + throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); } else - dstPath = expectedStorePath; + dstPath = state.store->printStorePath(*expectedStorePath); mkString(v, dstPath, {dstPath}); } @@ -1091,7 +1087,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args if (args[0]->type != tLambda) throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); - addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], true, Hash(), v); } static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -2151,7 +2147,7 @@ void EvalState::createBaseEnv() } if (!evalSettings.pureEval) { - mkString(v, settings.thisSystem); + mkString(v, settings.thisSystem.get()); addConstant("__currentSystem", v); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 2d79739ea..94fa0158c 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -148,7 +148,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg if (!state.store->isStorePath(i.name)) throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); if (!settings.readOnlyMode) - state.store->ensurePath(i.name); + state.store->ensurePath(state.store->parseStorePath(i.name)); state.forceAttrs(*i.value, *i.pos); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index a4c4b0943..80588f54f 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -56,7 +56,7 @@ static std::optional<GitInfo> lookupGitInfo( Path storePath = json["storePath"]; - if (store->isValidPath(storePath)) { + if (store->isValidPath(store->parseStorePath(storePath))) { GitInfo gitInfo; gitInfo.storePath = storePath; gitInfo.rev = rev; @@ -145,7 +145,7 @@ GitInfo exportGit(ref<Store> store, std::string uri, return files.count(file); }; - gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); + gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); gitInfo.revCount = haveCommits ? std::stoull(runProgram("git", true, { "-C", uri, "rev-list", "--count", "HEAD" })) : 0; // FIXME: maybe we should use the timestamp of the last // modified dirty file? @@ -265,7 +265,7 @@ GitInfo exportGit(ref<Store> store, std::string uri, unpackTarfile(*source, tmpDir); - gitInfo.storePath = store->addToStore(name, tmpDir); + gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", gitInfo.rev.gitRev() })); gitInfo.lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", gitInfo.rev.gitRev() })); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 40082894f..290cdb0b2 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -64,7 +64,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, return files.count(file); }; - hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); + hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); return hgInfo; } @@ -135,7 +135,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, hgInfo.storePath = json["storePath"]; - if (store->isValidPath(hgInfo.storePath)) { + if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) { printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath); return hgInfo; } @@ -151,7 +151,7 @@ HgInfo exportMercurial(ref<Store> store, const std::string & uri, deletePath(tmpDir + "/.hg_archival.txt"); - hgInfo.storePath = store->addToStore(name, tmpDir); + hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); nlohmann::json json; json["storePath"] = hgInfo.storePath; diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 91faea122..7ba5e1c14 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -38,7 +38,12 @@ public: return s < s2.s; } - operator const string & () const + operator const std::string & () const + { + return *s; + } + + operator const std::string_view () const { return *s; } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 81fb6b05a..d41e772e9 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -33,25 +33,25 @@ void printGCWarning() } -void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl) +void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl) { unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; + StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl); } -void printMissing(ref<Store> store, const PathSet & willBuild, - const PathSet & willSubstitute, const PathSet & unknown, +void printMissing(ref<Store> store, const StorePathSet & willBuild, + const StorePathSet & willSubstitute, const StorePathSet & unknown, unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) { if (!willBuild.empty()) { printMsg(lvl, "these derivations will be built:"); - Paths sorted = store->topoSortPaths(willBuild); + auto sorted = store->topoSortPaths(willBuild); reverse(sorted.begin(), sorted.end()); for (auto & i : sorted) - printMsg(lvl, fmt(" %s", i)); + printMsg(lvl, fmt(" %s", store->printStorePath(i))); } if (!willSubstitute.empty()) { @@ -59,14 +59,14 @@ void printMissing(ref<Store> store, const PathSet & willBuild, downloadSize / (1024.0 * 1024.0), narSize / (1024.0 * 1024.0))); for (auto & i : willSubstitute) - printMsg(lvl, fmt(" %s", i)); + printMsg(lvl, fmt(" %s", store->printStorePath(i))); } if (!unknown.empty()) { printMsg(lvl, fmt("don't know how to build these paths%s:", (settings.readOnlyMode ? " (may be caused by read-only store access)" : ""))); for (auto & i : unknown) - printMsg(lvl, fmt(" %s", i)); + printMsg(lvl, fmt(" %s", store->printStorePath(i))); } } @@ -237,7 +237,7 @@ bool LegacyArgs::processArgs(const Strings & args, bool finish) void parseCmdLine(int argc, char * * argv, std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg) { - parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg); + parseCmdLine(std::string(baseNameOf(argv[0])), argvToStrings(argc, argv), parseArg); } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 8e4861232..b49574652 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -3,6 +3,7 @@ #include "util.hh" #include "args.hh" #include "common-args.hh" +#include "path.hh" #include <signal.h> @@ -37,11 +38,15 @@ void printVersion(const string & programName); void printGCWarning(); class Store; +struct StorePathWithOutputs; -void printMissing(ref<Store> store, const PathSet & paths, Verbosity lvl = lvlInfo); +void printMissing( + ref<Store> store, + const std::vector<StorePathWithOutputs> & paths, + Verbosity lvl = lvlInfo); -void printMissing(ref<Store> store, const PathSet & willBuild, - const PathSet & willSubstitute, const PathSet & unknown, +void printMissing(ref<Store> store, const StorePathSet & willBuild, + const StorePathSet & willSubstitute, const StorePathSet & unknown, unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo); string getArg(const string & opt, diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 7e3e5ff88..aaacf8281 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -91,19 +91,18 @@ std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path) return sink.s; } -Path BinaryCacheStore::narInfoFileFor(const Path & storePath) +std::string BinaryCacheStore::narInfoFileFor(const StorePath & storePath) { - assertStorePath(storePath); - return storePathToHash(storePath) + ".narinfo"; + return storePathToHash(printStorePath(storePath)) + ".narinfo"; } void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) { auto narInfoFile = narInfoFileFor(narInfo->path); - upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo"); + upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - auto hashPart = storePathToHash(narInfo->path); + auto hashPart = storePathToHash(printStorePath(narInfo->path)); { auto state_(state.lock()); @@ -126,8 +125,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str if (ref != info.path) queryPathInfo(ref); } catch (InvalidPath &) { - throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid") - % info.path % ref); + throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", + printStorePath(info.path), printStorePath(ref)); } assert(nar->compare(0, narMagic.size(), narMagic) == 0); @@ -138,14 +137,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str narInfo->narHash = hashString(htSHA256, *nar); if (info.narHash && info.narHash != narInfo->narHash) - throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path); + throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); auto narAccessor = makeNarAccessor(nar); if (accessor_) - accessor_->addToCache(info.path, *nar, narAccessor); + accessor_->addToCache(printStorePath(info.path), *nar, narAccessor); /* Optionally write a JSON file containing a listing of the contents of the NAR. */ @@ -162,7 +161,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str } } - upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json"); + upsertFile(storePathToHash(printStorePath(info.path)) + ".ls", jsonOut.str(), "application/json"); } /* Compress the NAR. */ @@ -174,10 +173,10 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str narInfo->fileSize = narCompressed->size(); auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); - printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") - % narInfo->path % narInfo->narSize - % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) - % duration); + printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printStorePath(narInfo->path), narInfo->narSize, + ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), + duration); narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : @@ -254,14 +253,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str stats.narWriteCompressionTimeMs += duration; /* Atomically write the NAR info file.*/ - if (secretKey) narInfo->sign(*secretKey); + if (secretKey) narInfo->sign(*this, *secretKey); writeNarInfo(narInfo); stats.narInfoWrite++; } -bool BinaryCacheStore::isValidPathUncached(const Path & storePath) +bool BinaryCacheStore::isValidPathUncached(const StorePath & storePath) { // FIXME: this only checks whether a .narinfo with a matching hash // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even @@ -269,7 +268,7 @@ bool BinaryCacheStore::isValidPathUncached(const Path & storePath) return fileExists(narInfoFileFor(storePath)); } -void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) +void BinaryCacheStore::narFromPath(const StorePath & storePath, Sink & sink) { auto info = queryPathInfo(storePath).cast<const NarInfo>(); @@ -295,12 +294,13 @@ void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) stats.narReadBytes += narSize; } -void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, +void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { auto uri = getUri(); + auto storePathS = printStorePath(storePath); auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo, - fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri}); + fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); PushActivity pact(act->id); auto narInfoFile = narInfoFileFor(storePath); @@ -326,7 +326,7 @@ void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, }}); } -Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, +StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { // FIXME: some cut&paste from LocalStore::addToStore(). @@ -345,20 +345,18 @@ Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, h = hashString(hashAlgo, s); } - ValidPathInfo info; - info.path = makeFixedOutputPath(recursive, h, name); + ValidPathInfo info(makeFixedOutputPath(recursive, h, name)); addToStore(info, sink.s, repair, CheckSigs, nullptr); - return info.path; + return std::move(info.path); } -Path BinaryCacheStore::addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) +StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) { - ValidPathInfo info; - info.path = computeStorePathForText(name, s, references); - info.references = references; + ValidPathInfo info(computeStorePathForText(name, s, references)); + info.references = cloneStorePathSet(references); if (repair || !isValidPath(info.path)) { StringSink sink; @@ -366,7 +364,7 @@ Path BinaryCacheStore::addTextToStore(const string & name, const string & s, addToStore(info, sink.s, repair, CheckSigs, nullptr); } - return info.path; + return std::move(info.path); } ref<FSAccessor> BinaryCacheStore::getFSAccessor() @@ -374,7 +372,7 @@ ref<FSAccessor> BinaryCacheStore::getFSAccessor() return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache); } -void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs) +void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { /* Note: this is inherently racy since there is no locking on binary caches. In particular, with S3 this unreliable, even @@ -390,24 +388,22 @@ void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & s writeNarInfo(narInfo); } -std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path & path) +std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const StorePath & path) { - Path drvPath; + auto drvPath = path.clone(); - if (isDerivation(path)) - drvPath = path; - else { + if (!path.isDerivation()) { try { auto info = queryPathInfo(path); // FIXME: add a "Log" field to .narinfo - if (info->deriver == "") return nullptr; - drvPath = info->deriver; + if (!info->deriver) return nullptr; + drvPath = info->deriver->clone(); } catch (InvalidPath &) { return nullptr; } } - auto logPath = "log/" + baseNameOf(drvPath); + auto logPath = "log/" + std::string(baseNameOf(printStorePath(drvPath))); debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 2d7cd1947..fa2200ad8 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -65,18 +65,18 @@ private: std::string narMagic; - std::string narInfoFileFor(const Path & storePath); + std::string narInfoFileFor(const StorePath & storePath); void writeNarInfo(ref<NarInfo> narInfo); public: - bool isValidPathUncached(const Path & path) override; + bool isValidPathUncached(const StorePath & path) override; - void queryPathInfoUncached(const Path & path, + void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override; - Path queryPathFromHashPart(const string & hashPart) override + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } bool wantMassQuery() override { return wantMassQuery_; } @@ -85,27 +85,27 @@ public: RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; - Path addToStore(const string & name, const Path & srcPath, + StorePath addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override; + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; - void narFromPath(const Path & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { unsupported("buildDerivation"); } - void ensurePath(const Path & path) override + void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } ref<FSAccessor> getFSAccessor() override; - void addSignatures(const Path & storePath, const StringSet & sigs) override; + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - std::shared_ptr<std::string> getBuildLog(const Path & path) override; + std::shared_ptr<std::string> getBuildLog(const StorePath & path) override; int getPriority() override { return priority; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0de2f5bd2..05c4cb621 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -15,6 +15,7 @@ #include "parsed-derivations.hh" #include "machines.hh" #include "daemon.hh" +#include "worker-protocol.hh" #include <algorithm> #include <iostream> @@ -99,7 +100,7 @@ typedef set<GoalPtr, CompareGoalPtrs> Goals; typedef list<WeakGoalPtr> WeakGoals; /* A map of paths to goals (and the other way around). */ -typedef map<Path, WeakGoalPtr> WeakGoalMap; +typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; @@ -254,7 +255,7 @@ private: steady_time_point lastWokenUp; /* Cache for pathContentsGood(). */ - std::map<Path, bool> pathContentsGoodCache; + std::map<StorePath, bool> pathContentsGoodCache; public: @@ -301,10 +302,10 @@ public: ~Worker(); /* Make a goal (with caching). */ - GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); - std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const Path & drvPath, + GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(StorePath && drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const Path & storePath, RepairFlag repair = NoRepair); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -352,9 +353,9 @@ public: /* Check whether the given valid path exists and has the right contents. */ - bool pathContentsGood(const Path & path); + bool pathContentsGood(const StorePath & path); - void markContentsGood(const Path & path); + void markContentsGood(StorePath && path); void updateProgress() { @@ -471,7 +472,10 @@ static void commonChildInit(Pipe & logPipe) close(fdDevNull); } -void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) +void handleDiffHook( + uid_t uid, uid_t gid, + const Path & tryA, const Path & tryB, + const Path & drvPath, const Path & tmpDir) { auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { @@ -500,12 +504,6 @@ void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Pa class UserLock { private: - /* POSIX locks suck. If we have a lock on a file, and we open and - close that file again (without closing the original file - descriptor), we lose the lock. So we have to be *very* careful - not to open a lock file on which we are holding a lock. */ - static Sync<PathSet> lockedPaths_; - Path fnUserLock; AutoCloseFD fdUserLock; @@ -516,7 +514,6 @@ private: public: UserLock(); - ~UserLock(); void kill(); @@ -530,9 +527,6 @@ public: }; -Sync<PathSet> UserLock::lockedPaths_; - - UserLock::UserLock() { assert(settings.buildUsersGroup != ""); @@ -569,47 +563,34 @@ UserLock::UserLock() fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); - { - auto lockedPaths(lockedPaths_.lock()); - if (!lockedPaths->insert(fnUserLock).second) - /* We already have a lock on this one. */ - continue; - } + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError(format("opening user lock '%1%'") % fnUserLock); - try { + if (lockFile(fd.get(), ltWrite, false)) { + fdUserLock = std::move(fd); + user = i; + uid = pw->pw_uid; - AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fd) - throw SysError(format("opening user lock '%1%'") % fnUserLock); - - if (lockFile(fd.get(), ltWrite, false)) { - fdUserLock = std::move(fd); - user = i; - uid = pw->pw_uid; - - /* Sanity check... */ - if (uid == getuid() || uid == geteuid()) - throw Error(format("the Nix user should not be a member of '%1%'") - % settings.buildUsersGroup); + /* Sanity check... */ + if (uid == getuid() || uid == geteuid()) + throw Error(format("the Nix user should not be a member of '%1%'") + % settings.buildUsersGroup); #if __linux__ - /* Get the list of supplementary groups of this build user. This - is usually either empty or contains a group such as "kvm". */ - supplementaryGIDs.resize(10); - int ngroups = supplementaryGIDs.size(); - int err = getgrouplist(pw->pw_name, pw->pw_gid, - supplementaryGIDs.data(), &ngroups); - if (err == -1) - throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name); - - supplementaryGIDs.resize(ngroups); + /* Get the list of supplementary groups of this build user. This + is usually either empty or contains a group such as "kvm". */ + supplementaryGIDs.resize(10); + int ngroups = supplementaryGIDs.size(); + int err = getgrouplist(pw->pw_name, pw->pw_gid, + supplementaryGIDs.data(), &ngroups); + if (err == -1) + throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name); + + supplementaryGIDs.resize(ngroups); #endif - return; - } - - } catch (...) { - lockedPaths_.lock()->erase(fnUserLock); + return; } } @@ -619,14 +600,6 @@ UserLock::UserLock() } -UserLock::~UserLock() -{ - auto lockedPaths(lockedPaths_.lock()); - auto erased = lockedPaths->erase(fnUserLock); - assert(erased); -} - - void UserLock::kill() { killUser(uid); @@ -694,7 +667,7 @@ HookInstance::HookInstance() throw SysError("dupping builder's stdout/stderr"); Strings args = { - baseNameOf(settings.buildHook), + std::string(baseNameOf(settings.buildHook.get())), std::to_string(verbosity), }; @@ -741,7 +714,7 @@ private: bool useDerivation; /* The path of the derivation. */ - Path drvPath; + StorePath drvPath; /* The specific outputs that we need to build. Empty means all of them. */ @@ -766,14 +739,14 @@ private: /* All input paths (that is, the union of FS closures of the immediate input paths). */ - PathSet inputPaths; + StorePathSet inputPaths; /* Outputs that are already valid. If we're repairing, these are the outputs that are valid *and* not corrupt. */ - PathSet validPaths; + StorePathSet validPaths; /* Outputs that are corrupt or not valid. */ - PathSet missingPaths; + StorePathSet missingPaths; /* User selected for running the builder. */ std::unique_ptr<UserLock> buildUser; @@ -853,7 +826,7 @@ private: /* Hash rewriting. */ StringMap inputRewrites, outputRewrites; - typedef map<Path, Path> RedirectedOutputs; + typedef map<StorePath, StorePath> RedirectedOutputs; RedirectedOutputs redirectedOutputs; BuildMode buildMode; @@ -861,7 +834,7 @@ private: /* If we're repairing without a chroot, there may be outputs that are valid but corrupt. So we redirect these outputs to temporary paths. */ - PathSet redirectedBadOutputs; + StorePathSet redirectedBadOutputs; BuildResult result; @@ -900,23 +873,23 @@ private: std::vector<std::thread> daemonWorkerThreads; /* Paths that were added via recursive Nix calls. */ - PathSet addedPaths; + StorePathSet addedPaths; /* Recursive Nix calls are only allowed to build or realize paths in the original input closure or added via a recursive Nix call (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where /nix/store/<bla> is some arbitrary path in a binary cache). */ - bool isAllowed(const Path & path) + bool isAllowed(const StorePath & path) { return inputPaths.count(path) || addedPaths.count(path); } - friend class RestrictedStore; + friend struct RestrictedStore; public: - DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, + DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode = bmNormal); - DerivationGoal(const Path & drvPath, const BasicDerivation & drv, + DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode = bmNormal); ~DerivationGoal(); @@ -931,14 +904,14 @@ public: i.e. a derivation named "aardvark" always comes before "baboon". And substitution goals always happen before derivation goals (due to "b$"). */ - return "b$" + storePathToName(drvPath) + "$" + drvPath; + return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); } void work() override; - Path getDrvPath() + StorePath getDrvPath() { - return drvPath; + return drvPath.clone(); } /* Add wanted outputs to an already existing derivation goal. */ @@ -978,7 +951,7 @@ private: /* Add 'path' to the set of paths that may be referenced by the outputs, and make it appear in the sandbox. */ - void addDependency(const Path & path); + void addDependency(const StorePath & path); /* Make a file owned by the builder. */ void chownToBuilder(const Path & path); @@ -1012,15 +985,12 @@ private: void flushLine(); /* Return the set of (in)valid paths. */ - PathSet checkPathValidity(bool returnValid, bool checkHash); - - /* Abort the goal if `path' failed to build. */ - bool pathFailed(const Path & path); + StorePathSet checkPathValidity(bool returnValid, bool checkHash); /* Forcibly kill the child process, if any. */ void killChild(); - Path addHashRewrite(const Path & path); + void addHashRewrite(const StorePath & path); void repairClosure(); @@ -1031,23 +1001,23 @@ private: void done(BuildResult::Status status, const string & msg = ""); - PathSet exportReferences(PathSet storePaths); + StorePathSet exportReferences(const StorePathSet & storePaths); }; const Path DerivationGoal::homeDir = "/homeless-shelter"; -DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, +DerivationGoal::DerivationGoal(StorePath && drvPath, const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(true) - , drvPath(drvPath) + , drvPath(std::move(drvPath)) , wantedOutputs(wantedOutputs) , buildMode(buildMode) { state = &DerivationGoal::getDerivation; - name = (format("building of '%1%'") % drvPath).str(); + name = fmt("building of '%s'", worker.store.printStorePath(drvPath)); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -1055,16 +1025,16 @@ DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOut } -DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv, +DerivationGoal::DerivationGoal(StorePath && drvPath, const BasicDerivation & drv, Worker & worker, BuildMode buildMode) : Goal(worker) , useDerivation(false) - , drvPath(drvPath) + , drvPath(std::move(drvPath)) , buildMode(buildMode) { - this->drv = std::unique_ptr<BasicDerivation>(new BasicDerivation(drv)); + this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv)); state = &DerivationGoal::haveDerivation; - name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); + name = fmt("building of %s", worker.store.showPaths(drv.outputPaths())); trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); @@ -1174,7 +1144,7 @@ void DerivationGoal::loadDerivation() trace("loading derivation"); if (nrFailed != 0) { - printError(format("cannot build missing derivation '%1%'") % drvPath); + printError("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath)); done(BuildResult::MiscFailure); return; } @@ -1203,7 +1173,7 @@ void DerivationGoal::haveDerivation() worker.store.addTempRoot(i.second.path); /* Check what outputs paths are not already valid. */ - PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); + auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair); /* If they are all valid, then we're done. */ if (invalidOutputs.size() == 0 && buildMode == bmNormal) { @@ -1211,7 +1181,7 @@ void DerivationGoal::haveDerivation() return; } - parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + parsedDrv = std::make_unique<ParsedDerivation>(drvPath.clone(), *drv); /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build @@ -1232,7 +1202,9 @@ void DerivationGoal::outputsSubstituted() trace("all outputs substituted (maybe)"); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, (format("some substitutes for the outputs of derivation '%1%' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ") % drvPath).str()); + done(BuildResult::TransientFailure, + fmt("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", + worker.store.printStorePath(drvPath))); return; } @@ -1259,14 +1231,15 @@ void DerivationGoal::outputsSubstituted() return; } if (buildMode == bmCheck && nrInvalid > 0) - throw Error(format("some outputs of '%1%' are not valid, so checking is not possible") % drvPath); + throw Error("some outputs of '%s' are not valid, so checking is not possible", + worker.store.printStorePath(drvPath)); /* Otherwise, at least one of the output paths could not be produced using a substitute. So we have to build instead. */ /* Make sure checkPathValidity() from now on checks all outputs. */ - wantedOutputs = PathSet(); + wantedOutputs.clear(); /* The inputs must be built before we can build this goal. */ if (useDerivation) @@ -1276,8 +1249,8 @@ void DerivationGoal::outputsSubstituted() for (auto & i : drv->inputSrcs) { if (worker.store.isValidPath(i)) continue; if (!settings.useSubstitutes) - throw Error(format("dependency '%1%' of '%2%' does not exist, and substitution is disabled") - % i % drvPath); + throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); addWaitee(worker.makeSubstitutionGoal(i)); } @@ -1296,7 +1269,7 @@ void DerivationGoal::repairClosure() that produced those outputs. */ /* Get the output closure. */ - PathSet outputClosure; + StorePathSet outputClosure; for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; worker.store.computeFSClosure(i.second.path, outputClosure); @@ -1309,26 +1282,26 @@ void DerivationGoal::repairClosure() /* Get all dependencies of this derivation so that we know which derivation is responsible for which path in the output closure. */ - PathSet inputClosure; + StorePathSet inputClosure; if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); - std::map<Path, Path> outputsToDrv; + std::map<StorePath, StorePath> outputsToDrv; for (auto & i : inputClosure) - if (isDerivation(i)) { + if (i.isDerivation()) { Derivation drv = worker.store.derivationFromPath(i); for (auto & j : drv.outputs) - outputsToDrv[j.second.path] = i; + outputsToDrv.insert_or_assign(j.second.path.clone(), i.clone()); } /* Check each path (slow!). */ - PathSet broken; for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; - printError(format("found corrupted or missing path '%1%' in the output closure of '%2%'") % i % drvPath); - Path drvPath2 = outputsToDrv[i]; - if (drvPath2 == "") + printError("found corrupted or missing path '%s' in the output closure of '%s'", + worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); + auto drvPath2 = outputsToDrv.find(i); + if (drvPath2 == outputsToDrv.end()) addWaitee(worker.makeSubstitutionGoal(i, Repair)); else - addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); + addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair)); } if (waitees.empty()) { @@ -1344,7 +1317,8 @@ void DerivationGoal::closureRepaired() { trace("closure repaired"); if (nrFailed > 0) - throw Error(format("some paths in the output closure of derivation '%1%' could not be repaired") % drvPath); + throw Error("some paths in the output closure of derivation '%s' could not be repaired", + worker.store.printStorePath(drvPath)); done(BuildResult::AlreadyValid); } @@ -1355,10 +1329,9 @@ void DerivationGoal::inputsRealised() if (nrFailed != 0) { if (!useDerivation) - throw Error(format("some dependencies of '%1%' are missing") % drvPath); - printError( - format("cannot build derivation '%1%': %2% dependencies couldn't be built") - % drvPath % nrFailed); + throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); + printError("cannot build derivation '%s': %s dependencies couldn't be built", + worker.store.printStorePath(drvPath), nrFailed); done(BuildResult::DependencyFailed); return; } @@ -1381,19 +1354,21 @@ void DerivationGoal::inputsRealised() that are specified as inputs. */ assert(worker.store.isValidPath(i.first)); Derivation inDrv = worker.store.derivationFromPath(i.first); - for (auto & j : i.second) - if (inDrv.outputs.find(j) != inDrv.outputs.end()) - worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); + for (auto & j : i.second) { + auto k = inDrv.outputs.find(j); + if (k != inDrv.outputs.end()) + worker.store.computeFSClosure(k->second.path, inputPaths); else throw Error( - format("derivation '%1%' requires non-existent output '%2%' from input derivation '%3%'") - % drvPath % j % i.first); + "derivation '%s' requires non-existent output '%s' from input derivation '%s'", + worker.store.printStorePath(drvPath), j, worker.store.printStorePath(i.first)); + } } /* Second, the input sources. */ worker.store.computeFSClosure(drv->inputSrcs, inputPaths); - debug(format("added input paths %1%") % showPaths(inputPaths)); + debug("added input paths %s", worker.store.showPaths(inputPaths)); /* Is this a fixed-output derivation? */ fixedOutput = drv->isFixedOutput(); @@ -1423,7 +1398,7 @@ void DerivationGoal::tryToBuild() few seconds and then retry this goal. */ PathSet lockFiles; for (auto & outPath : drv->outputPaths()) - lockFiles.insert(worker.store.toRealPath(outPath)); + lockFiles.insert(worker.store.toRealPath(worker.store.printStorePath(outPath))); if (!outputLocks.lockPaths(lockFiles, "", false)) { worker.waitForAWhile(shared_from_this()); @@ -1439,23 +1414,22 @@ void DerivationGoal::tryToBuild() build this derivation, so no further checks are necessary. */ validPaths = checkPathValidity(true, buildMode == bmRepair); if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { - debug(format("skipping build of derivation '%1%', someone beat us to it") % drvPath); + debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); done(BuildResult::AlreadyValid); return; } - missingPaths = drv->outputPaths(); + missingPaths = cloneStorePathSet(drv->outputPaths()); if (buildMode != bmCheck) for (auto & i : validPaths) missingPaths.erase(i); /* If any of the outputs already exist but are not valid, delete them. */ for (auto & i : drv->outputs) { - Path path = i.second.path; - if (worker.store.isValidPath(path)) continue; - debug(format("removing invalid path '%1%'") % path); - deletePath(worker.store.toRealPath(path)); + if (worker.store.isValidPath(i.second.path)) continue; + debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path)); + deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second.path))); } /* Don't do a remote build if the derivation has the attribute @@ -1468,11 +1442,11 @@ void DerivationGoal::tryToBuild() buildMode == bmRepair ? "repairing outputs of '%s'" : buildMode == bmCheck ? "checking outputs of '%s'" : nrRounds > 1 ? "building '%s' (round %d/%d)" : - "building '%s'", drvPath, curRound, nrRounds); - fmt("building '%s'", drvPath); + "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + fmt("building '%s'", worker.store.printStorePath(drvPath)); if (hook) msg += fmt(" on '%s'", machineName); act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, - Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds}); + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); worker.updateProgress(); }; @@ -1541,7 +1515,7 @@ void replaceValidPath(const Path & storePath, const Path tmpPath) if (pathExists(storePath)) rename(storePath.c_str(), oldPath.c_str()); if (rename(tmpPath.c_str(), storePath.c_str()) == -1) - throw SysError(format("moving '%1%' to '%2%'") % tmpPath % storePath); + throw SysError("moving '%s' to '%s'", tmpPath, storePath); deletePath(oldPath); } @@ -1566,7 +1540,7 @@ void DerivationGoal::buildDone() kill it. */ int status = hook ? hook->pid.kill() : pid.kill(); - debug(format("builder process for '%1%' finished") % drvPath); + debug("builder process for '%s' finished", worker.store.printStorePath(drvPath)); result.timesBuilt++; result.stopTime = time(0); @@ -1622,12 +1596,15 @@ void DerivationGoal::buildDone() /* Move paths out of the chroot for easier debugging of build failures. */ if (useChroot && buildMode == bmNormal) - for (auto & i : missingPaths) - if (pathExists(chrootRootDir + i)) - rename((chrootRootDir + i).c_str(), i.c_str()); + for (auto & i : missingPaths) { + auto p = worker.store.printStorePath(i); + if (pathExists(chrootRootDir + p)) + rename((chrootRootDir + p).c_str(), p.c_str()); + } - std::string msg = (format("builder for '%1%' %2%") - % drvPath % statusToString(status)).str(); + auto msg = fmt("builder for '%s' %s", + worker.store.printStorePath(drvPath), + statusToString(status)); if (!settings.verboseBuild && !logTail.empty()) { msg += (format("; last %d log lines:") % logTail.size()).str(); @@ -1648,13 +1625,13 @@ void DerivationGoal::buildDone() if (settings.postBuildHook != "") { Activity act(*logger, lvlInfo, actPostBuildHook, fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{drvPath}); + Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); auto outputPaths = drv->outputPaths(); std::map<std::string, std::string> hookEnvironment = getEnv(); - hookEnvironment.emplace("DRV_PATH", drvPath); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); + hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); RunOptions opts(settings.postBuildHook, {}); opts.environment = hookEnvironment; @@ -1707,7 +1684,7 @@ void DerivationGoal::buildDone() /* Delete unused redirected outputs (when doing hash rewriting). */ for (auto & i : redirectedOutputs) - deletePath(i.second); + deletePath(worker.store.toRealPath(worker.store.printStorePath(i.second))); /* Delete the chroot (if we were using one). */ autoDelChroot.reset(); /* this runs the destructor */ @@ -1772,7 +1749,7 @@ HookReply DerivationGoal::tryBuildHook() << "try" << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) << drv->platform - << drvPath + << worker.store.printStorePath(drvPath) << parsedDrv->getRequiredSystemFeatures(); worker.hook->sink.flush(); @@ -1805,7 +1782,7 @@ HookReply DerivationGoal::tryBuildHook() else if (reply == "postpone") return rpPostpone; else if (reply != "accept") - throw Error(format("bad hook reply '%1%'") % reply); + throw Error("bad hook reply '%s'", reply); } catch (SysError & e) { if (e.errNo == EPIPE) { @@ -1823,11 +1800,11 @@ HookReply DerivationGoal::tryBuildHook() /* Tell the hook all the inputs that have to be copied to the remote system. */ - hook->sink << inputPaths; + writeStorePaths(worker.store, hook->sink, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ - hook->sink << missingPaths; + writeStorePaths(worker.store, hook->sink, missingPaths); hook->sink = FdSink(); hook->toHook.writeSide = -1; @@ -1844,10 +1821,10 @@ HookReply DerivationGoal::tryBuildHook() } -void chmod_(const Path & path, mode_t mode) +static void chmod_(const Path & path, mode_t mode) { if (chmod(path.c_str(), mode) == -1) - throw SysError(format("setting permissions on '%1%'") % path); + throw SysError("setting permissions on '%s'", path); } @@ -1858,33 +1835,25 @@ int childEntry(void * arg) } -PathSet DerivationGoal::exportReferences(PathSet storePaths) +StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) { - PathSet paths; - - for (auto storePath : storePaths) { - - /* Check that the store path is valid. */ - if (!worker.store.isInStore(storePath)) - throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") - % storePath); - - storePath = worker.store.toStorePath(storePath); + StorePathSet paths; + for (auto & storePath : storePaths) { if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", storePath); + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - worker.store.computeFSClosure(storePath, paths); + worker.store.computeFSClosure(singleton(storePath), paths); } /* If there are derivations in the graph, then include their outputs as well. This is useful if you want to do things like passing all build-time dependencies of some path to a derivation that builds a NixOS DVD image. */ - PathSet paths2(paths); + auto paths2 = cloneStorePathSet(paths); for (auto & j : paths2) { - if (isDerivation(j)) { + if (j.isDerivation()) { Derivation drv = worker.store.derivationFromPath(j); for (auto & k : drv.outputs) worker.store.computeFSClosure(k.second.path, paths); @@ -1932,7 +1901,7 @@ void DerivationGoal::startBuilder() throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", drv->platform, concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), - drvPath, + worker.store.printStorePath(drvPath), settings.thisSystem, concatStringsSep<StringSet>(", ", settings.systemFeatures)); @@ -1948,12 +1917,12 @@ void DerivationGoal::startBuilder() auto noChroot = parsedDrv->getBoolAttr("__noChroot"); if (settings.sandboxMode == smEnabled) { if (noChroot) - throw Error(format("derivation '%1%' has '__noChroot' set, " - "but that's not allowed when 'sandbox' is 'true'") % drvPath); + throw Error("derivation '%s' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); #if __APPLE__ if (additionalSandboxProfile != "") - throw Error(format("derivation '%1%' specifies a sandbox profile, " - "but this is only allowed when 'sandbox' is 'relaxed'") % drvPath); + throw Error("derivation '%s' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); #endif useChroot = true; } @@ -1989,14 +1958,13 @@ void DerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - auto drvName = storePathToName(drvPath); - tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); + tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); chownToBuilder(tmpDir); /* Substitute output placeholders with the actual output paths. */ for (auto & output : drv->outputs) - inputRewrites[hashPlaceholder(output.first)] = output.second.path; + inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path); /* Construct the environment passed to the builder. */ initEnv(); @@ -2012,19 +1980,22 @@ void DerivationGoal::startBuilder() temporary build directory. The text files have the format used by `nix-store --register-validity'. However, the deriver fields are left empty. */ - string s = get(drv->env, "exportReferencesGraph"); + string s = get(drv->env, "exportReferencesGraph").value_or(""); Strings ss = tokenizeString<Strings>(s); if (ss.size() % 2 != 0) throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); for (Strings::iterator i = ss.begin(); i != ss.end(); ) { string fileName = *i++; - checkStoreName(fileName); /* !!! abuse of this function */ - Path storePath = *i++; + static std::regex regex("[A-Za-z_][A-Za-z0-9_.]*"); + if (!std::regex_match(fileName, regex)) + throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName); + + auto storePath = worker.store.parseStorePath(*i++); /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); + exportReferences(singleton(storePath)), false, false)); } } @@ -2054,17 +2025,19 @@ void DerivationGoal::startBuilder() dirsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ - PathSet closure; + StorePathSet closure; for (auto & i : dirsInChroot) try { if (worker.store.isInStore(i.second.source)) - worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure); + worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure); } catch (InvalidPath & e) { } catch (Error & e) { throw Error(format("while processing 'sandbox-paths': %s") % e.what()); } - for (auto & i : closure) - dirsInChroot[i] = i; + for (auto & i : closure) { + auto p = worker.store.printStorePath(i); + dirsInChroot.insert_or_assign(p, p); + } PathSet allowedPaths = settings.allowedImpureHostPrefixes; @@ -2086,7 +2059,8 @@ void DerivationGoal::startBuilder() } } if (!found) - throw Error(format("derivation '%1%' requested impure path '%2%', but it was not in allowed-impure-host-deps") % drvPath % i); + throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", + worker.store.printStorePath(drvPath), i); dirsInChroot[i] = i; } @@ -2096,7 +2070,7 @@ void DerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store to ensure that we can create hard-links to non-directory inputs in the fake Nix store in the chroot (see below). */ - chrootRootDir = worker.store.toRealPath(drvPath) + ".chroot"; + chrootRootDir = worker.store.toRealPath(worker.store.printStorePath(drvPath)) + ".chroot"; deletePath(chrootRootDir); /* Clean up the chroot directory automatically. */ @@ -2155,14 +2129,15 @@ void DerivationGoal::startBuilder() throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir); for (auto & i : inputPaths) { - Path r = worker.store.toRealPath(i); + auto p = worker.store.printStorePath(i); + Path r = worker.store.toRealPath(p); struct stat st; if (lstat(r.c_str(), &st)) - throw SysError(format("getting attributes of path '%1%'") % i); + throw SysError("getting attributes of path '%s'", p); if (S_ISDIR(st.st_mode)) - dirsInChroot[i] = r; + dirsInChroot.insert_or_assign(p, r); else - linkOrCopy(r, chrootRootDir + i); + linkOrCopy(r, chrootRootDir + p); } /* If we're repairing, checking or rebuilding part of a @@ -2171,7 +2146,7 @@ void DerivationGoal::startBuilder() (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputs) - dirsInChroot.erase(i.second.path); + dirsInChroot.erase(worker.store.printStorePath(i.second.path)); #elif __APPLE__ /* We don't really have any parent prep work to do (yet?) @@ -2203,17 +2178,17 @@ void DerivationGoal::startBuilder() corrupt outputs in advance. So rewrite them as well. */ if (buildMode == bmRepair) for (auto & i : missingPaths) - if (worker.store.isValidPath(i) && pathExists(i)) { + if (worker.store.isValidPath(i) && pathExists(worker.store.printStorePath(i))) { addHashRewrite(i); - redirectedBadOutputs.insert(i); + redirectedBadOutputs.insert(i.clone()); } } if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) { printMsg(lvlChatty, format("executing pre-build hook '%1%'") % settings.preBuildHook); - auto args = useChroot ? Strings({drvPath, chrootRootDir}) : - Strings({ drvPath }); + auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : + Strings({ worker.store.printStorePath(drvPath) }); enum BuildHookState { stBegin, stExtraChrootDirs @@ -2480,7 +2455,7 @@ void DerivationGoal::initTmpDir() { there is no size constraint). */ if (!parsedDrv->getStructuredAttrs()) { - StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile")); + StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or("")); int fileNr = 0; for (auto & i : drv->env) { if (passAsFile.find(i.first) == passAsFile.end()) { @@ -2510,6 +2485,7 @@ void DerivationGoal::initTmpDir() { env["PWD"] = tmpDirInSandbox; } + void DerivationGoal::initEnv() { env.clear(); @@ -2581,7 +2557,7 @@ void DerivationGoal::writeStructuredAttrs() /* Add an "outputs" object containing the output paths. */ nlohmann::json outputs; for (auto & i : drv->outputs) - outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites); json["outputs"] = outputs; /* Handle exportReferencesGraph. */ @@ -2591,9 +2567,9 @@ void DerivationGoal::writeStructuredAttrs() std::ostringstream str; { JSONPlaceholder jsonRoot(str, true); - PathSet storePaths; + StorePathSet storePaths; for (auto & p : *i) - storePaths.insert(p.get<std::string>()); + storePaths.insert(worker.store.parseStorePath(p.get<std::string>())); worker.store.pathInfoToJSON(jsonRoot, exportReferences(storePaths), false, true); } @@ -2692,22 +2668,22 @@ struct RestrictedStore : public LocalFSStore std::string getUri() override { return next->getUri(); } - PathSet queryAllValidPaths() override + StorePathSet queryAllValidPaths() override { - PathSet paths; - for (auto & p : goal.inputPaths) paths.insert(p); - for (auto & p : goal.addedPaths) paths.insert(p); + StorePathSet paths; + for (auto & p : goal.inputPaths) paths.insert(p.clone()); + for (auto & p : goal.addedPaths) paths.insert(p.clone()); return paths; } - void queryPathInfoUncached(const Path & path, + void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override { if (goal.isAllowed(path)) { try { /* Censor impure information. */ auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path)); - info->deriver.clear(); + info->deriver.reset(); info->registrationTime = 0; info->ultimate = false; info->sigs.clear(); @@ -2719,19 +2695,19 @@ struct RestrictedStore : public LocalFSStore callback(nullptr); }; - void queryReferrers(const Path & path, PathSet & referrers) override + void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - PathSet queryDerivationOutputs(const Path & path) override + StorePathSet queryDerivationOutputs(const StorePath & path) override { throw Error("queryDerivationOutputs"); } - StringSet queryDerivationOutputNames(const Path & path) override + StringSet queryDerivationOutputNames(const StorePath & path) override { throw Error("queryDerivationOutputNames"); } - Path queryPathFromHashPart(const string & hashPart) override + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } - Path addToStore(const string & name, const Path & srcPath, + StorePath addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override { throw Error("addToStore"); } @@ -2744,7 +2720,7 @@ struct RestrictedStore : public LocalFSStore goal.addDependency(info.path); } - Path addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(const string & dump, const string & name, bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, recursive, hashAlgo, repair); @@ -2752,89 +2728,87 @@ struct RestrictedStore : public LocalFSStore return path; } - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair = NoRepair) override + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair = NoRepair) override { auto path = next->addTextToStore(name, s, references, repair); goal.addDependency(path); return path; } - void narFromPath(const Path & path, Sink & sink) override + void narFromPath(const StorePath & path, Sink & sink) override { if (!goal.isAllowed(path)) - throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", path); + throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path)); LocalFSStore::narFromPath(path, sink); } - void ensurePath(const Path & path) override + void ensurePath(const StorePath & path) override { if (!goal.isAllowed(path)) - throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", path); + throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path)); /* Nothing to be done; 'path' must already be valid. */ } - void buildPaths(const PathSet & paths, BuildMode buildMode) override + void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); - PathSet newPaths; + StorePathSet newPaths; for (auto & path : paths) { - DrvPathWithOutputs i = parseDrvPathWithOutputs(path); - if (isDerivation(i.first)) { - if (!goal.isAllowed(i.first)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", i.first); - auto drv = derivationFromPath(i.first); + if (path.path.isDerivation()) { + if (!goal.isAllowed(path.path)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); + auto drv = derivationFromPath(path.path); for (auto & output : drv.outputs) - if (wantOutput(output.first, i.second)) - newPaths.insert(output.second.path); - } else if (!goal.isAllowed(path)) - throw InvalidPath("cannot build unknown path '%s' in recursive Nix", path); + if (wantOutput(output.first, path.outputs)) + newPaths.insert(output.second.path.clone()); + } else if (!goal.isAllowed(path.path)) + throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path)); } next->buildPaths(paths, buildMode); - PathSet closure; + StorePathSet closure; next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) override { unsupported("buildDerivation"); } - void addTempRoot(const Path & path) + void addTempRoot(const StorePath & path) override { } - void addIndirectRoot(const Path & path) + void addIndirectRoot(const Path & path) override { } - Roots findRoots() + Roots findRoots(bool censor) override { return Roots(); } - void collectGarbage(const GCOptions & options, GCResults & results) + void collectGarbage(const GCOptions & options, GCResults & results) override { } - void addSignatures(const Path & storePath, const StringSet & sigs) + void addSignatures(const StorePath & storePath, const StringSet & sigs) override { unsupported("addSignatures"); } - void queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) + void queryMissing(const std::vector<StorePathWithOutputs> & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) override { /* This is slightly impure since it leaks information to the client about what paths will be built/substituted or are already present. Probably not a big deal. */ - PathSet allowed; + std::vector<StorePathWithOutputs> allowed; for (auto & path : targets) { - DrvPathWithOutputs i = parseDrvPathWithOutputs(path); - if (goal.isAllowed(i.first)) - allowed.insert(i.first); + if (goal.isAllowed(path.path)) + allowed.emplace_back(path); else - unknown.insert(i.first); + unknown.insert(path.path.clone()); } next->queryMissing(allowed, willBuild, willSubstitute, @@ -2887,7 +2861,7 @@ void DerivationGoal::startDaemon() debug("received daemon connection"); - auto workerThread = std::thread([this, store, remote{std::move(remote)}]() { + auto workerThread = std::thread([store, remote{std::move(remote)}]() { FdSource from(remote.get()); FdSink to(remote.get()); try { @@ -2925,28 +2899,26 @@ void DerivationGoal::stopDaemon() } -void DerivationGoal::addDependency(const Path & path) +void DerivationGoal::addDependency(const StorePath & path) { - worker.store.assertStorePath(path); - if (isAllowed(path)) return; - addedPaths.insert(path); + addedPaths.insert(path.clone()); /* If we're doing a sandbox build, then we have to make the path appear in the sandbox. */ if (useChroot) { - debug("materialising '%s' in the sandbox", path); + debug("materialising '%s' in the sandbox", worker.store.printStorePath(path)); #if __linux__ - Path source = worker.store.toRealPath(path); - Path target = chrootRootDir + path; + Path source = worker.store.toRealPath(worker.store.printStorePath(path)); + Path target = chrootRootDir + worker.store.printStorePath(path); debug("bind-mounting %s -> %s", target, source); if (pathExists(target)) - throw Error("store path '%s' already exists in the sandbox", path); + throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); struct stat st; if (lstat(source.c_str(), &st)) @@ -2973,13 +2945,14 @@ void DerivationGoal::addDependency(const Path & path) int status = child.wait(); if (status != 0) - throw Error("could not add path '%s' to sandbox", path); + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); } else linkOrCopy(source, target); #else - throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", path); + throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", + worker.store.printStorePath(path)); #endif } @@ -3395,8 +3368,10 @@ void DerivationGoal::runChild() } /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) - dirsInChroot[i] = i; + for (auto & i : inputPaths) { + auto p = worker.store.printStorePath(i); + dirsInChroot[p] = p; + } /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ if (settings.darwinLogSandboxViolations) { @@ -3412,13 +3387,13 @@ void DerivationGoal::runChild() /* Our rwx outputs */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : missingPaths) { - sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); - } + for (auto & i : missingPaths) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i)); + /* Also add redirected outputs to the chroot */ - for (auto & i : redirectedOutputs) { - sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str(); - } + for (auto & i : redirectedOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(i.second)); + sandboxProfile += ")\n"; /* Our inputs (transitive dependencies and any impurities computed above) @@ -3437,19 +3412,19 @@ void DerivationGoal::runChild() if (lstat(path.c_str(), &st)) { if (i.second.optional && errno == ENOENT) continue; - throw SysError(format("getting attributes of path '%1%'") % path); + throw SysError("getting attributes of path '%s", path); } if (S_ISDIR(st.st_mode)) - sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str(); + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); else - sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str(); + sandboxProfile += fmt("\t(literal \"%s\")\n", path); } sandboxProfile += ")\n"; /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ sandboxProfile += "(allow file-read*\n"; for (auto & i : ancestry) { - sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); + sandboxProfile += fmt("\t(literal \"%s\")\n", i); } sandboxProfile += ")\n"; @@ -3490,8 +3465,7 @@ void DerivationGoal::runChild() #endif else { builder = drv->builder.c_str(); - string builderBasename = baseNameOf(drv->builder); - args.push_back(builderBasename); + args.push_back(std::string(baseNameOf(drv->builder))); } for (auto & i : drv->args) @@ -3505,7 +3479,7 @@ void DerivationGoal::runChild() try { logger = makeJSONLogger(*logger); - BasicDerivation drv2(*drv); + BasicDerivation & drv2(*drv); for (auto & e : drv2.env) e.second = rewriteStrings(e.second, inputRewrites); @@ -3538,16 +3512,15 @@ void DerivationGoal::runChild() /* Parse a list of reference specifiers. Each element must either be a store path, or the symbolic name of the output of the derivation (such as `out'). */ -PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) +StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) { - PathSet result; + StorePathSet result; for (auto & i : paths) { if (store.isStorePath(i)) - result.insert(i); - else if (drv.outputs.find(i) != drv.outputs.end()) - result.insert(drv.outputs.find(i)->second.path); - else throw BuildError( - format("derivation contains an illegal reference specifier '%1%'") % i); + result.insert(store.parseStorePath(i)); + else if (drv.outputs.count(i)) + result.insert(drv.outputs.find(i)->second.path.clone()); + else throw BuildError("derivation contains an illegal reference specifier '%s'", i); } return result; } @@ -3580,19 +3553,17 @@ void DerivationGoal::registerOutputs() /* The paths that can be referenced are the input closures, the output paths, and any paths that have been built via recursive Nix calls. */ - PathSet referenceablePaths; - for (auto & p : inputPaths) referenceablePaths.insert(p); - for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path); - for (auto & p : addedPaths) referenceablePaths.insert(p); + StorePathSet referenceablePaths; + for (auto & p : inputPaths) referenceablePaths.insert(p.clone()); + for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path.clone()); + for (auto & p : addedPaths) referenceablePaths.insert(p.clone()); /* Check whether the output paths were created, and grep each output path to determine what other paths it references. Also make all output paths read-only. */ for (auto & i : drv->outputs) { - Path path = i.second.path; - if (missingPaths.find(path) == missingPaths.end()) continue; - - ValidPathInfo info; + auto path = worker.store.printStorePath(i.second.path); + if (!missingPaths.count(i.second.path)) continue; Path actualPath = path; if (useChroot) { @@ -3609,22 +3580,25 @@ void DerivationGoal::registerOutputs() } if (needsHashRewrite()) { - Path redirected = redirectedOutputs[path]; - if (buildMode == bmRepair - && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() - && pathExists(redirected)) - replaceValidPath(path, redirected); - if (buildMode == bmCheck && redirected != "") - actualPath = redirected; + auto r = redirectedOutputs.find(i.second.path); + if (r != redirectedOutputs.end()) { + auto redirected = worker.store.toRealPath(worker.store.printStorePath(r->second)); + if (buildMode == bmRepair + && redirectedBadOutputs.count(i.second.path) + && pathExists(redirected)) + replaceValidPath(path, redirected); + if (buildMode == bmCheck) + actualPath = redirected; + } } struct stat st; if (lstat(actualPath.c_str(), &st) == -1) { if (errno == ENOENT) throw BuildError( - format("builder for '%1%' failed to produce output path '%2%'") - % drvPath % path); - throw SysError(format("getting attributes of path '%1%'") % actualPath); + "builder for '%s' failed to produce output path '%s'", + worker.store.printStorePath(drvPath), path); + throw SysError("getting attributes of path '%s'", actualPath); } #ifndef __CYGWIN__ @@ -3661,6 +3635,8 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ + std::string ca; + if (fixedOutput) { bool recursive; Hash h; @@ -3677,7 +3653,7 @@ void DerivationGoal::registerOutputs() the derivation to its content-addressed location. */ Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); - Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path)); + auto dest = worker.store.makeFixedOutputPath(recursive, h2, i.second.path.name()); if (h != h2) { @@ -3686,9 +3662,9 @@ void DerivationGoal::registerOutputs() worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - dest, h.to_string(SRI), h2.to_string(SRI))); + worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); - Path actualDest = worker.store.toRealPath(dest); + Path actualDest = worker.store.toRealPath(worker.store.printStorePath(dest)); if (worker.store.isValidPath(dest)) std::rethrow_exception(delayedException); @@ -3697,16 +3673,16 @@ void DerivationGoal::registerOutputs() PathLocks outputLocks({actualDest}); deletePath(actualDest); if (rename(actualPath.c_str(), actualDest.c_str()) == -1) - throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest); + throw SysError("moving '%s' to '%s'", actualPath, worker.store.printStorePath(dest)); } - path = dest; + path = worker.store.printStorePath(dest); actualPath = actualDest; } else - assert(path == dest); + assert(worker.store.parseStorePath(path) == dest); - info.ca = makeFixedOutputCA(recursive, h2); + ca = makeFixedOutputCA(recursive, h2); } /* Get rid of all weird permissions. This also checks that @@ -3720,11 +3696,11 @@ void DerivationGoal::registerOutputs() verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); HashResult hash; - PathSet references = scanForReferences(actualPath, referenceablePaths, hash); + auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); if (buildMode == bmCheck) { - if (!worker.store.isValidPath(path)) continue; - auto info = *worker.store.queryPathInfo(path); + if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; + ValidPathInfo info(*worker.store.queryPathInfo(worker.store.parseStorePath(path))); if (hash.first != info.narHash) { worker.checkMismatch = true; if (settings.runDiffHook || settings.keepFailed) { @@ -3736,20 +3712,22 @@ void DerivationGoal::registerOutputs() handleDiffHook( buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), - path, dst, drvPath, tmpDir); + path, dst, worker.store.printStorePath(drvPath), tmpDir); - throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") - % drvPath % path % dst); + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'", + worker.store.printStorePath(drvPath), path, dst); } else - throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs") - % drvPath % path); + throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs", + worker.store.printStorePath(drvPath), path); } /* Since we verified the build, it's now ultimately trusted. */ if (!info.ultimate) { info.ultimate = true; worker.store.signPathInfo(info); - worker.store.registerValidPaths({info}); + ValidPathInfos infos; + infos.push_back(std::move(info)); + worker.store.registerValidPaths(infos); } continue; @@ -3757,29 +3735,30 @@ void DerivationGoal::registerOutputs() /* For debugging, print out the referenced and unreferenced paths. */ for (auto & i : inputPaths) { - PathSet::iterator j = references.find(i); + auto j = references.find(i); if (j == references.end()) - debug(format("unreferenced input: '%1%'") % i); + debug("unreferenced input: '%1%'", worker.store.printStorePath(i)); else - debug(format("referenced input: '%1%'") % i); + debug("referenced input: '%1%'", worker.store.printStorePath(i)); } if (curRound == nrRounds) { worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() - worker.markContentsGood(path); + worker.markContentsGood(worker.store.parseStorePath(path)); } - info.path = path; + ValidPathInfo info(worker.store.parseStorePath(path)); info.narHash = hash.first; info.narSize = hash.second; - info.references = references; - info.deriver = drvPath; + info.references = std::move(references); + info.deriver = drvPath.clone(); info.ultimate = true; + info.ca = ca; worker.store.signPathInfo(info); if (!info.references.empty()) info.ca.clear(); - infos[i.first] = info; + infos.emplace(i.first, std::move(info)); } if (buildMode == bmCheck) return; @@ -3794,16 +3773,19 @@ void DerivationGoal::registerOutputs() for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) if (!(*i == *j)) { result.isNonDeterministic = true; - Path prev = i->second.path + checkSuffix; + Path prev = worker.store.printStorePath(i->second.path) + checkSuffix; bool prevExists = keepPreviousRound && pathExists(prev); auto msg = prevExists - ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) - : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); + ? fmt("output '%s' of '%s' differs from '%s' from previous round", + worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev) + : fmt("output '%s' of '%s' differs from previous round", + worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath)); handleDiffHook( buildUser ? buildUser->getUID() : getuid(), buildUser ? buildUser->getGID() : getgid(), - prev, i->second.path, drvPath, tmpDir); + prev, worker.store.printStorePath(i->second.path), + worker.store.printStorePath(drvPath), tmpDir); if (settings.enforceDeterminism) throw NotDeterministic(msg); @@ -3816,16 +3798,17 @@ void DerivationGoal::registerOutputs() /* If this is the first round of several, then move the output out of the way. */ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { for (auto & i : drv->outputs) { - Path prev = i.second.path + checkSuffix; + auto path = worker.store.printStorePath(i.second.path); + Path prev = path + checkSuffix; deletePath(prev); - Path dst = i.second.path + checkSuffix; - if (rename(i.second.path.c_str(), dst.c_str())) - throw SysError(format("renaming '%1%' to '%2%'") % i.second.path % dst); + Path dst = path + checkSuffix; + if (rename(path.c_str(), dst.c_str())) + throw SysError("renaming '%s' to '%s'", path, dst); } } if (curRound < nrRounds) { - prevInfos = infos; + prevInfos = std::move(infos); return; } @@ -3833,7 +3816,7 @@ void DerivationGoal::registerOutputs() if the result was not determistic? */ if (curRound == nrRounds) { for (auto & i : drv->outputs) { - Path prev = i.second.path + checkSuffix; + Path prev = worker.store.printStorePath(i.second.path) + checkSuffix; deletePath(prev); } } @@ -3858,7 +3841,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) { std::map<Path, const ValidPathInfo &> outputsByPath; for (auto & output : outputs) - outputsByPath.emplace(output.second.path, output.second); + outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second); for (auto & output : outputs) { auto & outputName = output.first; @@ -3874,76 +3857,77 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) /* Compute the closure and closure size of some output. This is slightly tricky because some of its references (namely other outputs) may not be valid yet. */ - auto getClosure = [&](const Path & path) + auto getClosure = [&](const StorePath & path) { uint64_t closureSize = 0; - PathSet pathsDone; - std::queue<Path> pathsLeft; - pathsLeft.push(path); + StorePathSet pathsDone; + std::queue<StorePath> pathsLeft; + pathsLeft.push(path.clone()); while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); + auto path = pathsLeft.front().clone(); pathsLeft.pop(); - if (!pathsDone.insert(path).second) continue; + if (!pathsDone.insert(path.clone()).second) continue; - auto i = outputsByPath.find(path); + auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; for (auto & ref : i->second.references) - pathsLeft.push(ref); + pathsLeft.push(ref.clone()); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; for (auto & ref : info->references) - pathsLeft.push(ref); + pathsLeft.push(ref.clone()); } } - return std::make_pair(pathsDone, closureSize); + return std::make_pair(std::move(pathsDone), closureSize); }; auto applyChecks = [&](const Checks & checks) { if (checks.maxSize && info.narSize > *checks.maxSize) throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", - info.path, info.narSize, *checks.maxSize); + worker.store.printStorePath(info.path), info.narSize, *checks.maxSize); if (checks.maxClosureSize) { uint64_t closureSize = getClosure(info.path).second; if (closureSize > *checks.maxClosureSize) throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", - info.path, closureSize, *checks.maxClosureSize); + worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize); } auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive) { if (!value) return; - PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); + auto spec = parseReferenceSpecifiers(worker.store, *drv, *value); - PathSet used = recursive ? getClosure(info.path).first : info.references; + auto used = recursive ? cloneStorePathSet(getClosure(info.path).first) : cloneStorePathSet(info.references); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); - PathSet badPaths; + StorePathSet badPaths; for (auto & i : used) if (allowed) { if (!spec.count(i)) - badPaths.insert(i); + badPaths.insert(i.clone()); } else { if (spec.count(i)) - badPaths.insert(i); + badPaths.insert(i.clone()); } if (!badPaths.empty()) { string badPathsStr; for (auto & i : badPaths) { badPathsStr += "\n "; - badPathsStr += i; + badPathsStr += worker.store.printStorePath(i); } - throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr); + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", + worker.store.printStorePath(info.path), badPathsStr); } }; @@ -3975,7 +3959,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) Strings res; for (auto j = i->begin(); j != i->end(); ++j) { if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath)); res.push_back(j->get<std::string>()); } checks.disallowedRequisites = res; @@ -4012,7 +3996,7 @@ Path DerivationGoal::openLogFile() if (!settings.keepLog) return ""; - string baseName = baseNameOf(drvPath); + auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath))); /* Create a log file. */ Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); @@ -4051,9 +4035,7 @@ void DerivationGoal::deleteTmpDir(bool force) /* Don't keep temporary directories for builtins because they might have privileged stuff (like a copy of netrc). */ if (settings.keepFailed && !force && !drv->isBuiltin()) { - printError( - format("note: keeping build directory '%2%'") - % drvPath % tmpDir); + printError("note: keeping build directory '%s'", tmpDir); chmod(tmpDir.c_str(), 0755); } else @@ -4132,31 +4114,31 @@ void DerivationGoal::flushLine() } -PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) +StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { - PathSet result; + StorePathSet result; for (auto & i : drv->outputs) { if (!wantOutput(i.first, wantedOutputs)) continue; bool good = worker.store.isValidPath(i.second.path) && (!checkHash || worker.pathContentsGood(i.second.path)); - if (good == returnValid) result.insert(i.second.path); + if (good == returnValid) result.insert(i.second.path.clone()); } return result; } -Path DerivationGoal::addHashRewrite(const Path & path) +void DerivationGoal::addHashRewrite(const StorePath & path) { - string h1 = string(path, worker.store.storeDir.size() + 1, 32); - string h2 = string(hashString(htSHA256, "rewrite:" + drvPath + ":" + path).to_string(Base32, false), 0, 32); - Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33); - deletePath(p); - assert(path.size() == p.size()); + auto h1 = std::string(((std::string_view) path.to_string()).substr(0, 32)); + auto p = worker.store.makeStorePath( + "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), + Hash(htSHA256), path.name()); + auto h2 = std::string(((std::string_view) p.to_string()).substr(0, 32)); + deletePath(worker.store.printStorePath(p)); inputRewrites[h1] = h2; outputRewrites[h2] = h1; - redirectedOutputs[path] = p; - return p; + redirectedOutputs.insert_or_assign(path.clone(), std::move(p)); } @@ -4194,7 +4176,7 @@ class SubstitutionGoal : public Goal private: /* The store path that should be realised through a substitute. */ - Path storePath; + StorePath storePath; /* The remaining substituters. */ std::list<ref<Store>> subs; @@ -4230,7 +4212,7 @@ private: GoalState state; public: - SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair = NoRepair); + SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair = NoRepair); ~SubstitutionGoal(); void timedOut() override { abort(); }; @@ -4239,7 +4221,7 @@ public: { /* "a$" ensures substitution goals happen before derivation goals. */ - return "a$" + storePathToName(storePath) + "$" + storePath; + return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } void work() override; @@ -4256,7 +4238,7 @@ public: void handleChildOutput(int fd, const string & data) override; void handleEOF(int fd) override; - Path getStorePath() { return storePath; } + StorePath getStorePath() { return storePath.clone(); } void amDone(ExitCode result) override { @@ -4265,13 +4247,13 @@ public: }; -SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair) +SubstitutionGoal::SubstitutionGoal(StorePath && storePath, Worker & worker, RepairFlag repair) : Goal(worker) + , storePath(std::move(storePath)) , repair(repair) { - this->storePath = storePath; state = &SubstitutionGoal::init; - name = (format("substitution of '%1%'") % storePath).str(); + name = fmt("substitution of '%s'", worker.store.printStorePath(storePath)); trace("created"); maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions); } @@ -4310,7 +4292,7 @@ void SubstitutionGoal::init() } if (settings.readOnlyMode) - throw Error(format("cannot substitute path '%1%' - no write access to the Nix store") % storePath); + throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>(); @@ -4325,7 +4307,7 @@ void SubstitutionGoal::tryNext() if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ - debug(format("path '%1%' is required, but there is no substituter that can build it") % storePath); + debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a @@ -4389,7 +4371,7 @@ void SubstitutionGoal::tryNext() && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) { printError("warning: substituter '%s' does not have a valid signature for path '%s'", - sub->getUri(), storePath); + sub->getUri(), worker.store.printStorePath(storePath)); tryNext(); return; } @@ -4412,7 +4394,7 @@ void SubstitutionGoal::referencesValid() trace("all references realised"); if (nrFailed > 0) { - debug(format("some references of path '%1%' could not be realised") % storePath); + debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); return; } @@ -4451,7 +4433,7 @@ void SubstitutionGoal::tryToRun() /* Wake up the worker loop when we're done. */ Finally updateStats([this]() { outPipe.writeSide = -1; }); - Activity act(*logger, actSubstitute, Logger::Fields{storePath, sub->getUri()}); + Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), @@ -4498,10 +4480,9 @@ void SubstitutionGoal::finished() return; } - worker.markContentsGood(storePath); + worker.markContentsGood(storePath.clone()); - printMsg(lvlChatty, - format("substitution of path '%1%' succeeded") % storePath); + printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); maintainRunningSubstitutions.reset(); @@ -4567,13 +4548,13 @@ Worker::~Worker() } -GoalPtr Worker::makeDerivationGoal(const Path & path, +GoalPtr Worker::makeDerivationGoal(const StorePath & path, const StringSet & wantedOutputs, BuildMode buildMode) { - GoalPtr goal = derivationGoals[path].lock(); + GoalPtr goal = derivationGoals[path.clone()].lock(); // FIXME if (!goal) { - goal = std::make_shared<DerivationGoal>(path, wantedOutputs, *this, buildMode); - derivationGoals[path] = goal; + goal = std::make_shared<DerivationGoal>(path.clone(), wantedOutputs, *this, buildMode); + derivationGoals.insert_or_assign(path.clone(), goal); wakeUp(goal); } else (dynamic_cast<DerivationGoal *>(goal.get()))->addWantedOutputs(wantedOutputs); @@ -4581,21 +4562,21 @@ GoalPtr Worker::makeDerivationGoal(const Path & path, } -std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const Path & drvPath, +std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(StorePath && drvPath, const BasicDerivation & drv, BuildMode buildMode) { - auto goal = std::make_shared<DerivationGoal>(drvPath, drv, *this, buildMode); + auto goal = std::make_shared<DerivationGoal>(std::move(drvPath), drv, *this, buildMode); wakeUp(goal); return goal; } -GoalPtr Worker::makeSubstitutionGoal(const Path & path, RepairFlag repair) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair) { - GoalPtr goal = substitutionGoals[path].lock(); + GoalPtr goal = substitutionGoals[path.clone()].lock(); // FIXME if (!goal) { - goal = std::make_shared<SubstitutionGoal>(path, *this, repair); - substitutionGoals[path] = goal; + goal = std::make_shared<SubstitutionGoal>(path.clone(), *this, repair); + substitutionGoals.insert_or_assign(path.clone(), goal); wakeUp(goal); } return goal; @@ -4934,38 +4915,38 @@ unsigned int Worker::exitStatus() } -bool Worker::pathContentsGood(const Path & path) +bool Worker::pathContentsGood(const StorePath & path) { - std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path); + auto i = pathContentsGoodCache.find(path); if (i != pathContentsGoodCache.end()) return i->second; - printInfo(format("checking path '%1%'...") % path); + printInfo("checking path '%s'...", store.printStorePath(path)); auto info = store.queryPathInfo(path); bool res; - if (!pathExists(path)) + if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.type, path); + HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } - pathContentsGoodCache[path] = res; - if (!res) printError(format("path '%1%' is corrupted or missing!") % path); + pathContentsGoodCache.insert_or_assign(path.clone(), res); + if (!res) printError("path '%s' is corrupted or missing!", store.printStorePath(path)); return res; } -void Worker::markContentsGood(const Path & path) +void Worker::markContentsGood(StorePath && path) { - pathContentsGoodCache[path] = true; + pathContentsGoodCache.insert_or_assign(std::move(path), true); } ////////////////////////////////////////////////////////////////////// -static void primeCache(Store & store, const PathSet & paths) +static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths) { - PathSet willBuild, willSubstitute, unknown; + StorePathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -4976,24 +4957,23 @@ static void primeCache(Store & store, const PathSet & paths) } -void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) +void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) { Worker worker(*this); primeCache(*this, drvPaths); Goals goals; - for (auto & i : drvPaths) { - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); - if (isDerivation(i2.first)) - goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode)); + for (auto & path : drvPaths) { + if (path.path.isDerivation()) + goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode)); else - goals.insert(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); + goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair)); } worker.run(goals); - PathSet failed; + StorePathSet failed; for (auto & i : goals) { if (i->getExitCode() != Goal::ecSuccess) { DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); @@ -5007,11 +4987,11 @@ void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) } -BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, +BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { Worker worker(*this); - auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + auto goal = worker.makeBasicDerivationGoal(drvPath.clone(), drv, buildMode); BuildResult result; @@ -5027,12 +5007,12 @@ BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivat } -void LocalStore::ensurePath(const Path & path) +void LocalStore::ensurePath(const StorePath & path) { /* If the path is already valid, we're done. */ if (isValidPath(path)) return; - primeCache(*this, {path}); + primeCache(*this, {StorePathWithOutputs(path)}); Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path); @@ -5041,11 +5021,11 @@ void LocalStore::ensurePath(const Path & path) worker.run(goals); if (goal->getExitCode() != Goal::ecSuccess) - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", path); + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } -void LocalStore::repairPath(const Path & path) +void LocalStore::repairPath(const StorePath & path) { Worker worker(*this); GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); @@ -5056,13 +5036,13 @@ void LocalStore::repairPath(const Path & path) if (goal->getExitCode() != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ - auto deriver = queryPathInfo(path)->deriver; - if (deriver != "" && isValidPath(deriver)) { + auto info = queryPathInfo(path); + if (info->deriver && isValidPath(*info->deriver)) { goals.clear(); - goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); + goals.insert(worker.makeDerivationGoal(*info->deriver, StringSet(), bmRepair)); worker.run(goals); } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", path); + throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); } } diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0a37459b0..73c0f5f7f 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -9,7 +9,7 @@ struct Package { Path path; bool active; int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} + Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; typedef std::vector<Package> Packages; diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index b1af3b4fc..f6ae5d2e6 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -24,7 +24,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) Path storePath = getAttr("out"); auto mainUrl = getAttr("url"); - bool unpack = get(drv.env, "unpack", "") == "1"; + bool unpack = get(drv.env, "unpack").value_or("") == "1"; /* Note: have to use a fresh downloader here because we're in a forked process. */ diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index d3c17e772..c53dfdd56 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -260,14 +260,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, switch (op) { case wopIsValidPath: { - /* 'readStorePath' could raise an error leading to the connection - being closed. To be able to recover from an invalid path error, - call 'startWork' early, and do 'assertStorePath' afterwards so - that the 'Error' exception handler doesn't close the - connection. */ - Path path = readString(from); - logger->startWork(); - store->assertStorePath(path); + auto path = store->parseStorePath(readString(from)); + logger->startWork(); bool result = store->isValidPath(path); logger->stopWork(); to << result; @@ -275,34 +269,36 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQueryValidPaths: { - PathSet paths = readStorePaths<PathSet>(*store, from); + auto paths = readStorePaths<StorePathSet>(*store, from); logger->startWork(); - PathSet res = store->queryValidPaths(paths); + auto res = store->queryValidPaths(paths); logger->stopWork(); - to << res; + writeStorePaths(*store, to, res); break; } case wopHasSubstitutes: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); - PathSet res = store->querySubstitutablePaths({path}); + StorePathSet paths; // FIXME + paths.insert(path.clone()); + auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - to << (res.find(path) != res.end()); + to << (res.count(path) != 0); break; } case wopQuerySubstitutablePaths: { - PathSet paths = readStorePaths<PathSet>(*store, from); + auto paths = readStorePaths<StorePathSet>(*store, from); logger->startWork(); - PathSet res = store->querySubstitutablePaths(paths); + auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - to << res; + writeStorePaths(*store, to, res); break; } case wopQueryPathHash: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); @@ -314,23 +310,24 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopQueryReferrers: case wopQueryValidDerivers: case wopQueryDerivationOutputs: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); - PathSet paths; + StorePathSet paths; if (op == wopQueryReferences) - paths = store->queryPathInfo(path)->references; + for (auto & i : store->queryPathInfo(path)->references) + paths.insert(i.clone()); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); else if (op == wopQueryValidDerivers) paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - to << paths; + writeStorePaths(*store, to, paths); break; } case wopQueryDerivationOutputNames: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); StringSet names; names = store->queryDerivationOutputNames(path); @@ -340,20 +337,20 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQueryDeriver: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); - auto deriver = store->queryPathInfo(path)->deriver; + auto info = store->queryPathInfo(path); logger->stopWork(); - to << deriver; + to << (info->deriver ? store->printStorePath(*info->deriver) : ""); break; } case wopQueryPathFromHashPart: { - string hashPart = readString(from); + auto hashPart = readString(from); logger->startWork(); - Path path = store->queryPathFromHashPart(hashPart); + auto path = store->queryPathFromHashPart(hashPart); logger->stopWork(); - to << path; + to << (path ? store->printStorePath(*path) : ""); break; } @@ -383,26 +380,26 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); if (!savedRegular.regular) throw Error("regular file expected"); - Path path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + auto path = store->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); logger->stopWork(); - to << path; + to << store->printStorePath(path); break; } case wopAddTextToStore: { string suffix = readString(from); string s = readString(from); - PathSet refs = readStorePaths<PathSet>(*store, from); + auto refs = readStorePaths<StorePathSet>(*store, from); logger->startWork(); - Path path = store->addTextToStore(suffix, s, refs, NoRepair); + auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); - to << path; + to << store->printStorePath(path); break; } case wopExportPath: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); readInt(from); // obsolete logger->startWork(); TunnelSink sink(to); @@ -415,15 +412,19 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopImportPaths: { logger->startWork(); TunnelSource source(from, to); - Paths paths = store->importPaths(source, nullptr, + auto paths = store->importPaths(source, nullptr, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); - to << paths; + Strings paths2; + for (auto & i : paths) paths2.push_back(store->printStorePath(i)); + to << paths2; break; } case wopBuildPaths: { - PathSet drvs = readStorePaths<PathSet>(*store, from); + std::vector<StorePathWithOutputs> drvs; + for (auto & s : readStrings<Strings>(from)) + drvs.push_back(store->parseDrvPathWithOutputs(s)); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -441,7 +442,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopBuildDerivation: { - Path drvPath = readStorePath(*store, from); + auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; readDerivation(from, *store, drv); BuildMode buildMode = (BuildMode) readInt(from); @@ -455,7 +456,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopEnsurePath: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); store->ensurePath(path); logger->stopWork(); @@ -464,7 +465,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddTempRoot: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); store->addTempRoot(path); logger->stopWork(); @@ -502,7 +503,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, for (auto & [target, links] : roots) for (auto & link : links) - to << link << target; + to << link << store->printStorePath(target); break; } @@ -510,7 +511,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopCollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = readStorePaths<PathSet>(*store, from); + options.pathsToDelete = readStorePaths<StorePathSet>(*store, from); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -568,44 +569,52 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQuerySubstitutablePathInfo: { - Path path = absPath(readString(from)); + auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; - store->querySubstitutablePathInfos({path}, infos); + StorePathSet paths; + paths.insert(path.clone()); // FIXME + store->querySubstitutablePathInfos(paths, infos); logger->stopWork(); - SubstitutablePathInfos::iterator i = infos.find(path); + auto i = infos.find(path); if (i == infos.end()) to << 0; else { - to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize; + to << 1 + << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); + writeStorePaths(*store, to, i->second.references); + to << i->second.downloadSize + << i->second.narSize; } break; } case wopQuerySubstitutablePathInfos: { - PathSet paths = readStorePaths<PathSet>(*store, from); + auto paths = readStorePaths<StorePathSet>(*store, from); logger->startWork(); SubstitutablePathInfos infos; store->querySubstitutablePathInfos(paths, infos); logger->stopWork(); to << infos.size(); for (auto & i : infos) { - to << i.first << i.second.deriver << i.second.references - << i.second.downloadSize << i.second.narSize; + to << store->printStorePath(i.first) + << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); + writeStorePaths(*store, to, i.second.references); + to << i.second.downloadSize << i.second.narSize; } break; } case wopQueryAllValidPaths: { logger->startWork(); - PathSet paths = store->queryAllValidPaths(); + auto paths = store->queryAllValidPaths(); logger->stopWork(); - to << paths; + writeStorePaths(*store, to, paths); break; } case wopQueryPathInfo: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); std::shared_ptr<const ValidPathInfo> info; logger->startWork(); try { @@ -617,8 +626,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - to << info->deriver << info->narHash.to_string(Base16, false) << info->references - << info->registrationTime << info->narSize; + to << (info->deriver ? store->printStorePath(*info->deriver) : "") + << info->narHash.to_string(Base16, false); + writeStorePaths(*store, to, info->references); + to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs @@ -651,7 +662,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopAddSignatures: { - Path path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); StringSet sigs = readStrings<StringSet>(from); logger->startWork(); if (!trusted) @@ -663,22 +674,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopNarFromPath: { - auto path = readStorePath(*store, from); + auto path = store->parseStorePath(readString(from)); logger->startWork(); logger->stopWork(); - dumpPath(path, to); + dumpPath(store->printStorePath(path), to); break; } case wopAddToStoreNar: { bool repair, dontCheckSigs; - ValidPathInfo info; - info.path = readStorePath(*store, from); - from >> info.deriver; - if (!info.deriver.empty()) - store->assertStorePath(info.deriver); + ValidPathInfo info(store->parseStorePath(readString(from))); + auto deriver = readString(from); + if (deriver != "") + info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(from), htSHA256); - info.references = readStorePaths<PathSet>(*store, from); + info.references = readStorePaths<StorePathSet>(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); from >> info.ca >> repair >> dontCheckSigs; @@ -709,13 +719,18 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } case wopQueryMissing: { - PathSet targets = readStorePaths<PathSet>(*store, from); + std::vector<StorePathWithOutputs> targets; + for (auto & s : readStrings<Strings>(from)) + targets.push_back(store->parseDrvPathWithOutputs(s)); logger->startWork(); - PathSet willBuild, willSubstitute, unknown; + StorePathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - to << willBuild << willSubstitute << unknown << downloadSize << narSize; + writeStorePaths(*store, to, willBuild); + writeStorePaths(*store, to, willSubstitute); + writeStorePaths(*store, to, unknown); + to << downloadSize << narSize; break; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 23fcfb281..726e34479 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -21,17 +21,39 @@ void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const HashType hashType = parseHashType(algo); if (hashType == htUnknown) - throw Error(format("unknown hash algorithm '%1%'") % algo); + throw Error("unknown hash algorithm '%s'", algo); hash = Hash(this->hash, hashType); } -Path BasicDerivation::findOutput(const string & id) const +BasicDerivation::BasicDerivation(const BasicDerivation & other) + : platform(other.platform) + , builder(other.builder) + , args(other.args) + , env(other.env) +{ + for (auto & i : other.outputs) + outputs.insert_or_assign(i.first, + DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); + for (auto & i : other.inputSrcs) + inputSrcs.insert(i.clone()); +} + + +Derivation::Derivation(const Derivation & other) + : BasicDerivation(other) +{ + for (auto & i : other.inputDrvs) + inputDrvs.insert_or_assign(i.first.clone(), i.second); +} + + +const StorePath & BasicDerivation::findOutput(const string & id) const { auto i = outputs.find(id); if (i == outputs.end()) - throw Error(format("derivation has no output '%1%'") % id); + throw Error("derivation has no output '%s'", id); return i->second.path; } @@ -42,18 +64,17 @@ bool BasicDerivation::isBuiltin() const } -Path writeDerivation(ref<Store> store, +StorePath writeDerivation(ref<Store> store, const Derivation & drv, const string & name, RepairFlag repair) { - PathSet references; - references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); + auto references = cloneStorePathSet(drv.inputSrcs); for (auto & i : drv.inputDrvs) - references.insert(i.first); + references.insert(i.first.clone()); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be held during a garbage collection). */ string suffix = name + drvExtension; - string contents = drv.unparse(); + string contents = drv.unparse(*store, false); return settings.readOnlyMode ? store->computeStorePathForText(suffix, contents, references) : store->addTextToStore(suffix, contents, references, repair); @@ -121,7 +142,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } -static Derivation parseDerivation(const string & s) +static Derivation parseDerivation(const Store & store, const string & s) { Derivation drv; istringstream_nocopy str(s); @@ -129,13 +150,12 @@ static Derivation parseDerivation(const string & s) /* Parse the list of outputs. */ while (!endOfList(str)) { - DerivationOutput out; - expect(str, "("); string id = parseString(str); - expect(str, ","); out.path = parsePath(str); - expect(str, ","); out.hashAlgo = parseString(str); - expect(str, ","); out.hash = parseString(str); + expect(str, "("); std::string id = parseString(str); + expect(str, ","); auto path = store.parseStorePath(parsePath(str)); + expect(str, ","); auto hashAlgo = parseString(str); + expect(str, ","); auto hash = parseString(str); expect(str, ")"); - drv.outputs[id] = out; + drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); } /* Parse the list of input derivations. */ @@ -144,11 +164,11 @@ static Derivation parseDerivation(const string & s) expect(str, "("); Path drvPath = parsePath(str); expect(str, ",["); - drv.inputDrvs[drvPath] = parseStrings(str, false); + drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false)); expect(str, ")"); } - expect(str, ",["); drv.inputSrcs = parseStrings(str, true); + expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); expect(str, ","); drv.platform = parseString(str); expect(str, ","); drv.builder = parseString(str); @@ -171,25 +191,24 @@ static Derivation parseDerivation(const string & s) } -Derivation readDerivation(const Path & drvPath) +Derivation readDerivation(const Store & store, const Path & drvPath) { try { - return parseDerivation(readFile(drvPath)); + return parseDerivation(store, readFile(drvPath)); } catch (FormatError & e) { throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); } } -Derivation Store::derivationFromPath(const Path & drvPath) +Derivation Store::derivationFromPath(const StorePath & drvPath) { - assertStorePath(drvPath); ensurePath(drvPath); auto accessor = getFSAccessor(); try { - return parseDerivation(accessor->readFile(drvPath)); + return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); } catch (FormatError & e) { - throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); + throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); } } @@ -220,33 +239,56 @@ static void printStrings(string & res, ForwardIterator i, ForwardIterator j) } -string Derivation::unparse() const +string Derivation::unparse(const Store & store, bool maskOutputs, + std::map<std::string, StringSet> * actualInputs) const { string s; s.reserve(65536); s += "Derive(["; - bool first = true; - for (auto & i : outputs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i.first); - s += ','; printString(s, i.second.path); - s += ','; printString(s, i.second.hashAlgo); - s += ','; printString(s, i.second.hash); - s += ')'; + StringSet maskedOutputs; + + if (maskOutputs) { + bool first = true; + maskedOutputs = tokenizeString<StringSet>(get(env, "outputs").value_or("out"), " "); + for (auto & i : maskedOutputs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i); + s += ",\"\",\"\",\"\")"; + } + } else { + bool first = true; + for (auto & i : outputs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i.first); + s += ','; printString(s, store.printStorePath(i.second.path)); + s += ','; printString(s, i.second.hashAlgo); + s += ','; printString(s, i.second.hash); + s += ')'; + } } s += "],["; - first = true; - for (auto & i : inputDrvs) { - if (first) first = false; else s += ','; - s += '('; printString(s, i.first); - s += ','; printStrings(s, i.second.begin(), i.second.end()); - s += ')'; + bool first = true; + if (actualInputs) { + for (auto & i : *actualInputs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i.first); + s += ','; printStrings(s, i.second.begin(), i.second.end()); + s += ')'; + } + } else { + for (auto & i : inputDrvs) { + if (first) first = false; else s += ','; + s += '('; printString(s, store.printStorePath(i.first)); + s += ','; printStrings(s, i.second.begin(), i.second.end()); + s += ')'; + } } s += "],"; - printStrings(s, inputSrcs.begin(), inputSrcs.end()); + auto paths = store.printStorePathSet(inputSrcs); // FIXME: slow + printStrings(s, paths.begin(), paths.end()); s += ','; printString(s, platform); s += ','; printString(s, builder); @@ -257,7 +299,7 @@ string Derivation::unparse() const for (auto & i : env) { if (first) first = false; else s += ','; s += '('; printString(s, i.first); - s += ','; printString(s, i.second); + s += ','; printString(s, maskOutputs && maskedOutputs.count(i.first) ? "" : i.second); s += ')'; } @@ -267,6 +309,7 @@ string Derivation::unparse() const } +// FIXME: remove bool isDerivation(const string & fileName) { return hasSuffix(fileName, drvExtension); @@ -304,7 +347,7 @@ DrvHashes drvHashes; paths have been replaced by the result of a recursive call to this function, and that for fixed-output derivations we return a hash of its output path. */ -Hash hashDerivationModulo(Store & store, Derivation drv) +Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { @@ -312,42 +355,41 @@ Hash hashDerivationModulo(Store & store, Derivation drv) return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" + i->second.hash + ":" - + i->second.path); + + store.printStorePath(i->second.path)); } /* For other derivations, replace the inputs paths with recursive calls to this function.*/ - DerivationInputs inputs2; + std::map<std::string, StringSet> inputs2; for (auto & i : drv.inputDrvs) { - Hash h = drvHashes[i.first]; - if (!h) { + auto h = drvHashes.find(i.first); + if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); - Derivation drv2 = readDerivation(store.toRealPath(i.first)); - h = hashDerivationModulo(store, drv2); - drvHashes[i.first] = h; + h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, + readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first; } - inputs2[h.to_string(Base16, false)] = i.second; + inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } - drv.inputDrvs = inputs2; - return hashString(htSHA256, drv.unparse()); + return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); } -DrvPathWithOutputs parseDrvPathWithOutputs(const string & s) +StorePathWithOutputs Store::parseDrvPathWithOutputs(const std::string & s) { size_t n = s.find("!"); return n == s.npos - ? DrvPathWithOutputs(s, std::set<string>()) - : DrvPathWithOutputs(string(s, 0, n), tokenizeString<std::set<string> >(string(s, n + 1), ",")); + ? StorePathWithOutputs{parseStorePath(s), std::set<string>()} + : StorePathWithOutputs{parseStorePath(std::string_view(s.data(), n)), + tokenizeString<std::set<string>>(string(s, n + 1), ",")}; } -Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs) +std::string StorePathWithOutputs::to_string(const Store & store) const { return outputs.empty() - ? drvPath - : drvPath + "!" + concatStringsSep(",", outputs); + ? store.printStorePath(path) + : store.printStorePath(path) + "!" + concatStringsSep(",", outputs); } @@ -357,28 +399,28 @@ bool wantOutput(const string & output, const std::set<string> & wanted) } -PathSet BasicDerivation::outputPaths() const +StorePathSet BasicDerivation::outputPaths() const { - PathSet paths; + StorePathSet paths; for (auto & i : outputs) - paths.insert(i.second.path); + paths.insert(i.second.path.clone()); return paths; } -Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) { drv.outputs.clear(); auto nr = readNum<size_t>(in); for (size_t n = 0; n < nr; n++) { auto name = readString(in); - DerivationOutput o; - in >> o.path >> o.hashAlgo >> o.hash; - store.assertStorePath(o.path); - drv.outputs[name] = o; + auto path = store.parseStorePath(readString(in)); + auto hashAlgo = readString(in); + auto hash = readString(in); + drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); } - drv.inputSrcs = readStorePaths<PathSet>(store, in); + drv.inputSrcs = readStorePaths<StorePathSet>(store, in); in >> drv.platform >> drv.builder; drv.args = readStrings<Strings>(in); @@ -393,16 +435,16 @@ Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) } -Sink & operator << (Sink & out, const BasicDerivation & drv) +void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) { out << drv.outputs.size(); for (auto & i : drv.outputs) - out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; - out << drv.inputSrcs << drv.platform << drv.builder << drv.args; + out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash; + writeStorePaths(store, out, drv.inputSrcs); + out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) out << i.first << i.second; - return out; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 8e02c9bc5..c2df66229 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -10,26 +10,18 @@ namespace nix { -/* Extension of derivations in the Nix store. */ -const string drvExtension = ".drv"; - - /* Abstract syntax of derivations. */ struct DerivationOutput { - Path path; - string hashAlgo; /* hash used for expected hash computation */ - string hash; /* expected hash, may be null */ - DerivationOutput() - { - } - DerivationOutput(Path path, string hashAlgo, string hash) - { - this->path = path; - this->hashAlgo = hashAlgo; - this->hash = hash; - } + StorePath path; + std::string hashAlgo; /* hash used for expected hash computation */ + std::string hash; /* expected hash, may be null */ + DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash) + : path(std::move(path)) + , hashAlgo(std::move(hashAlgo)) + , hash(std::move(hash)) + { } void parseHashInfo(bool & recursive, Hash & hash) const; }; @@ -37,24 +29,26 @@ typedef std::map<string, DerivationOutput> DerivationOutputs; /* For inputs that are sub-derivations, we specify exactly which output IDs we are interested in. */ -typedef std::map<Path, StringSet> DerivationInputs; +typedef std::map<StorePath, StringSet> DerivationInputs; typedef std::map<string, string> StringPairs; struct BasicDerivation { DerivationOutputs outputs; /* keyed on symbolic IDs */ - PathSet inputSrcs; /* inputs that are sources */ + StorePathSet inputSrcs; /* inputs that are sources */ string platform; Path builder; Strings args; StringPairs env; + BasicDerivation() { } + explicit BasicDerivation(const BasicDerivation & other); virtual ~BasicDerivation() { }; /* Return the path corresponding to the output identifier `id' in the given derivation. */ - Path findOutput(const string & id) const; + const StorePath & findOutput(const std::string & id) const; bool isBuiltin() const; @@ -62,7 +56,7 @@ struct BasicDerivation bool isFixedOutput() const; /* Return the output paths of a derivation. */ - PathSet outputPaths() const; + StorePathSet outputPaths() const; }; @@ -71,7 +65,12 @@ struct Derivation : BasicDerivation DerivationInputs inputDrvs; /* inputs that are sub-derivations */ /* Print a derivation. */ - std::string unparse() const; + std::string unparse(const Store & store, bool maskOutputs, + std::map<std::string, StringSet> * actualInputs = nullptr) const; + + Derivation() { } + Derivation(Derivation && other) = default; + explicit Derivation(const Derivation & other); }; @@ -79,38 +78,29 @@ class Store; /* Write a derivation to the Nix store, and return its path. */ -Path writeDerivation(ref<Store> store, +StorePath writeDerivation(ref<Store> store, const Derivation & drv, const string & name, RepairFlag repair = NoRepair); /* Read a derivation from a file. */ -Derivation readDerivation(const Path & drvPath); +Derivation readDerivation(const Store & store, const Path & drvPath); -/* Check whether a file name ends with the extension for - derivations. */ +// FIXME: remove bool isDerivation(const string & fileName); -Hash hashDerivationModulo(Store & store, Derivation drv); +Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /* Memoisation of hashDerivationModulo(). */ -typedef std::map<Path, Hash> DrvHashes; +typedef std::map<StorePath, Hash> DrvHashes; extern DrvHashes drvHashes; // FIXME: global, not thread-safe -/* Split a string specifying a derivation and a set of outputs - (/nix/store/hash-foo!out1,out2,...) into the derivation path and - the outputs. */ -typedef std::pair<string, std::set<string> > DrvPathWithOutputs; -DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); - -Path makeDrvPathWithOutputs(const Path & drvPath, const std::set<string> & outputs); - bool wantOutput(const string & output, const std::set<string> & wanted); struct Source; struct Sink; -Source & readDerivation(Source & in, Store & store, BasicDerivation & drv); -Sink & operator << (Sink & out, const BasicDerivation & drv); +Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv); +void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); std::string hashPlaceholder(const std::string & outputName); diff --git a/src/libstore/download.cc b/src/libstore/download.cc index 38faa1bfd..2eb39af3a 100644 --- a/src/libstore/download.cc +++ b/src/libstore/download.cc @@ -650,10 +650,10 @@ struct CurlDownloader : public Downloader #ifdef ENABLE_S3 auto [bucketName, key, params] = parseS3Uri(request.uri); - std::string profile = get(params, "profile", ""); - std::string region = get(params, "region", Aws::Region::US_EAST_1); - std::string scheme = get(params, "scheme", ""); - std::string endpoint = get(params, "endpoint", ""); + std::string profile = get(params, "profile").value_or(""); + std::string region = get(params, "region").value_or(Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme").value_or(""); + std::string endpoint = get(params, "endpoint").value_or(""); S3Helper s3Helper(profile, region, scheme, endpoint); @@ -811,13 +811,13 @@ CachedDownloadResult Downloader::downloadCached( if (p != string::npos) name = string(url, p + 1); } - Path expectedStorePath; + std::optional<StorePath> expectedStorePath; if (request.expectedHash) { expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); - if (store->isValidPath(expectedStorePath)) { + if (store->isValidPath(*expectedStorePath)) { CachedDownloadResult result; - result.storePath = expectedStorePath; - result.path = store->toRealPath(expectedStorePath); + result.storePath = store->printStorePath(*expectedStorePath); + result.path = store->toRealPath(result.storePath); assert(!request.getLastModified); // FIXME return result; } @@ -833,7 +833,7 @@ CachedDownloadResult Downloader::downloadCached( PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); - Path storePath; + std::optional<StorePath> storePath; string expectedETag; @@ -842,9 +842,10 @@ CachedDownloadResult Downloader::downloadCached( CachedDownloadResult result; if (pathExists(fileLink) && pathExists(dataFile)) { - storePath = readLink(fileLink); - store->addTempRoot(storePath); - if (store->isValidPath(storePath)) { + storePath = store->parseStorePath(readLink(fileLink)); + // FIXME + store->addTempRoot(*storePath); + if (store->isValidPath(*storePath)) { auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n"); if (ss.size() >= 3 && ss[0] == url) { time_t lastChecked; @@ -858,7 +859,7 @@ CachedDownloadResult Downloader::downloadCached( } } } else - storePath = ""; + storePath.reset(); } if (!skip) { @@ -871,46 +872,45 @@ CachedDownloadResult Downloader::downloadCached( result.etag = res.etag; if (!res.cached) { - ValidPathInfo info; StringSink sink; dumpString(*res.data, sink); Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); - info.path = store->makeFixedOutputPath(false, hash, name); + ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); info.ca = makeFixedOutputCA(false, hash); store->addToStore(info, sink.s, NoRepair, NoCheckSigs); - storePath = info.path; + storePath = info.path.clone(); } - assert(!storePath.empty()); - replaceSymlink(storePath, fileLink); + assert(storePath); + replaceSymlink(store->printStorePath(*storePath), fileLink); writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); } catch (DownloadError & e) { - if (storePath.empty()) throw; + if (!storePath) throw; warn("warning: %s; using cached result", e.msg()); result.etag = expectedETag; } } if (request.unpack) { - Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; + Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked"; PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); - Path unpackedStorePath; + std::optional<StorePath> unpackedStorePath; if (pathExists(unpackedLink)) { - unpackedStorePath = readLink(unpackedLink); - store->addTempRoot(unpackedStorePath); - if (!store->isValidPath(unpackedStorePath)) - unpackedStorePath = ""; + unpackedStorePath = store->parseStorePath(readLink(unpackedLink)); + store->addTempRoot(*unpackedStorePath); + if (!store->isValidPath(*unpackedStorePath)) + unpackedStorePath.reset(); else result.lastModified = lstat(unpackedLink).st_mtime; } - if (unpackedStorePath.empty()) { + if (!unpackedStorePath) { printInfo("unpacking '%s'...", url); Path tmpDir = createTempDir(); AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(storePath), tmpDir, baseNameOf(url)); + unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir, std::string(baseNameOf(url))); auto members = readDirectory(tmpDir); if (members.size() != 1) throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); @@ -921,15 +921,15 @@ CachedDownloadResult Downloader::downloadCached( // Store the last-modified date of the tarball in the symlink // mtime. This saves us from having to store it somewhere // else. - replaceSymlink(unpackedStorePath, unpackedLink, result.lastModified); - storePath = unpackedStorePath; + replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink, result.lastModified); + storePath = std::move(*unpackedStorePath); } - if (expectedStorePath != "" && storePath != expectedStorePath) { + if (expectedStorePath && *storePath != *expectedStorePath) { unsigned int statusCode = 102; Hash gotHash = request.unpack - ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first - : hashFile(request.expectedHash.type, store->toRealPath(storePath)); + ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first + : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))); throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", url, request.expectedHash.to_string(), gotHash.to_string()); } @@ -937,8 +937,8 @@ CachedDownloadResult Downloader::downloadCached( if (request.gcRoot) store->addIndirectRoot(fileLink); - result.storePath = storePath; - result.path = store->toRealPath(storePath); + result.storePath = store->printStorePath(*storePath); + result.path = store->toRealPath(result.storePath); return result; } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index ef3fea7c8..4692d1a7b 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -24,9 +24,9 @@ struct HashAndWriteSink : Sink } }; -void Store::exportPaths(const Paths & paths, Sink & sink) +void Store::exportPaths(const StorePathSet & paths, Sink & sink) { - Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end())); + auto sorted = topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); std::string doneLabel("paths exported"); @@ -42,7 +42,7 @@ void Store::exportPaths(const Paths & paths, Sink & sink) sink << 0; } -void Store::exportPath(const Path & path, Sink & sink) +void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); @@ -55,15 +55,21 @@ void Store::exportPath(const Path & path, Sink & sink) Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) - throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path - % info->narHash.to_string() % hash.to_string()); - - hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0; + throw Error("hash of path '%s' has changed from '%s' to '%s'!", + printStorePath(path), info->narHash.to_string(), hash.to_string()); + + hashAndWriteSink + << exportMagic + << printStorePath(path); + writeStorePaths(*this, hashAndWriteSink, info->references); + hashAndWriteSink + << (info->deriver ? printStorePath(*info->deriver) : "") + << 0; } -Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) +StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs) { - Paths res; + StorePaths res; while (true) { auto n = readNum<uint64_t>(source); if (n == 0) break; @@ -77,16 +83,15 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, if (magic != exportMagic) throw Error("Nix archive cannot be imported; wrong format"); - ValidPathInfo info; - - info.path = readStorePath(*this, source); + ValidPathInfo info(parseStorePath(readString(source))); //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); - info.references = readStorePaths<PathSet>(*this, source); + info.references = readStorePaths<StorePathSet>(*this, source); - info.deriver = readString(source); - if (info.deriver != "") assertStorePath(info.deriver); + auto deriver = readString(source); + if (deriver != "") + info.deriver = parseStorePath(deriver); info.narHash = hashString(htSHA256, *tee.source.data); info.narSize = tee.source.data->size(); @@ -97,7 +102,7 @@ Paths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); - res.push_back(info.path); + res.push_back(info.path.clone()); } return res; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 842980fb7..ed81186af 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -85,12 +85,10 @@ void LocalStore::addIndirectRoot(const Path & path) } -Path LocalFSStore::addPermRoot(const Path & _storePath, +Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) { - Path storePath(canonPath(_storePath)); Path gcRoot(canonPath(_gcRoot)); - assertStorePath(storePath); if (isInStore(gcRoot)) throw Error(format( @@ -102,7 +100,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot); - makeSymlink(gcRoot, storePath); + makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); } @@ -117,10 +115,10 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, % gcRoot % rootsDir); } - if (baseNameOf(gcRoot) == baseNameOf(storePath)) + if (baseNameOf(gcRoot) == std::string(storePath.to_string())) writeFile(gcRoot, ""); else - makeSymlink(gcRoot, storePath); + makeSymlink(gcRoot, printStorePath(storePath)); } /* Check that the root can be found by the garbage collector. @@ -129,13 +127,12 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, check if the root is in a directory in or linked from the gcroots directory. */ if (settings.checkRootReachability) { - Roots roots = findRoots(false); - if (roots[storePath].count(gcRoot) == 0) + auto roots = findRoots(false); + if (roots[storePath.clone()].count(gcRoot) == 0) printError( - format( - "warning: '%1%' is not in a directory where the garbage collector looks for roots; " - "therefore, '%2%' might be removed by the garbage collector") - % gcRoot % storePath); + "warning: '%1%' is not in a directory where the garbage collector looks for roots; " + "therefore, '%2%' might be removed by the garbage collector", + gcRoot, printStorePath(storePath)); } /* Grab the global GC root, causing us to block while a GC is in @@ -147,7 +144,7 @@ Path LocalFSStore::addPermRoot(const Path & _storePath, } -void LocalStore::addTempRoot(const Path & path) +void LocalStore::addTempRoot(const StorePath & path) { auto state(_state.lock()); @@ -188,7 +185,7 @@ void LocalStore::addTempRoot(const Path & path) debug(format("acquiring write lock on '%1%'") % fnTempRoots); lockFile(state->fdTempRoots.get(), ltWrite, true); - string s = path + '\0'; + string s = printStorePath(path) + '\0'; writeFull(state->fdTempRoots.get(), s); /* Downgrade to a read lock. */ @@ -246,8 +243,7 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) while ((end = contents.find((char) 0, pos)) != string::npos) { Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); - assertStorePath(root); - tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); + tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } @@ -260,10 +256,11 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) { auto foundRoot = [&](const Path & path, const Path & target) { Path storePath = toStorePath(target); - if (isStorePath(storePath) && isValidPath(storePath)) - roots[storePath].emplace(path); + // FIXME + if (isStorePath(storePath) && isValidPath(parseStorePath(storePath))) + roots[parseStorePath(storePath)].emplace(path); else - printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath); + printInfo("skipping invalid root from '%1%' to '%2%'", path, storePath); }; try { @@ -299,9 +296,10 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) } else if (type == DT_REG) { - Path storePath = storeDir + "/" + baseNameOf(path); - if (isStorePath(storePath) && isValidPath(storePath)) - roots[storePath].emplace(path); + Path storePath = storeDir + "/" + std::string(baseNameOf(path)); + // FIXME + if (isStorePath(storePath) && isValidPath(parseStorePath(storePath))) + roots[parseStorePath(storePath)].emplace(path); } } @@ -309,7 +307,7 @@ void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) catch (SysError & e) { /* We only ignore permanent failures. */ if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) - printInfo(format("cannot read potential root '%1%'") % path); + printInfo("cannot read potential root '%1%'", path); else throw; } @@ -340,7 +338,9 @@ Roots LocalStore::findRoots(bool censor) return roots; } -static void readProcLink(const string & file, Roots & roots) +typedef std::unordered_map<Path, std::unordered_set<std::string>> UncheckedRoots; + +static void readProcLink(const string & file, UncheckedRoots & roots) { /* 64 is the starting buffer size gnu readlink uses... */ auto bufsiz = ssize_t{64}; @@ -369,7 +369,7 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } -static void readFileRoots(const char * path, Roots & roots) +static void readFileRoots(const char * path, UncheckedRoots & roots) { try { roots[readFile(path)].emplace(path); @@ -381,7 +381,7 @@ static void readFileRoots(const char * path, Roots & roots) void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { - Roots unchecked; + UncheckedRoots unchecked; auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { @@ -466,16 +466,16 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) #endif for (auto & [target, links] : unchecked) { - if (isInStore(target)) { - Path path = toStorePath(target); - if (isStorePath(path) && isValidPath(path)) { - debug(format("got additional root '%1%'") % path); - if (censor) - roots[path].insert(censored); - else - roots[path].insert(links.begin(), links.end()); - } - } + if (!isInStore(target)) continue; + Path pathS = toStorePath(target); + if (!isStorePath(pathS)) continue; + auto path = parseStorePath(pathS); + if (!isValidPath(path)) continue; + debug("got additional root '%1%'", pathS); + if (censor) + roots[path.clone()].insert(censored); + else + roots[path.clone()].insert(links.begin(), links.end()); } } @@ -485,18 +485,19 @@ struct GCLimitReached { }; struct LocalStore::GCState { - GCOptions options; + const GCOptions & options; GCResults & results; - PathSet roots; - PathSet tempRoots; - PathSet dead; - PathSet alive; + StorePathSet roots; + StorePathSet tempRoots; + StorePathSet dead; + StorePathSet alive; bool gcKeepOutputs; bool gcKeepDerivations; unsigned long long bytesInvalidated; bool moveToTrash = true; bool shouldDelete; - GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { } + GCState(const GCOptions & options, GCResults & results) + : options(options), results(results), bytesInvalidated(0) { } }; @@ -504,7 +505,7 @@ bool LocalStore::isActiveTempFile(const GCState & state, const Path & path, const string & suffix) { return hasSuffix(path, suffix) - && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end(); + && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size()))); } @@ -522,16 +523,17 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) unsigned long long size = 0; - if (isStorePath(path) && isValidPath(path)) { - PathSet referrers; - queryReferrers(path, referrers); + // FIXME + if (isStorePath(path) && isValidPath(parseStorePath(path))) { + StorePathSet referrers; + queryReferrers(parseStorePath(path), referrers); for (auto & i : referrers) - if (i != path) deletePathRecursive(state, i); - size = queryPathInfo(path)->narSize; - invalidatePathChecked(path); + if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i)); + size = queryPathInfo(parseStorePath(path))->narSize; + invalidatePathChecked(parseStorePath(path)); } - Path realPath = realStoreDir + "/" + baseNameOf(path); + Path realPath = realStoreDir + "/" + std::string(baseNameOf(path)); struct stat st; if (lstat(realPath.c_str(), &st)) { @@ -555,7 +557,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) try { if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) throw SysError(format("making '%1%' writable") % realPath); - Path tmp = trashDir + "/" + baseNameOf(path); + Path tmp = trashDir + "/" + std::string(baseNameOf(path)); if (rename(realPath.c_str(), tmp.c_str())) throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp); state.bytesInvalidated += size; @@ -575,7 +577,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path) } -bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path) +bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path) { if (visited.count(path)) return false; @@ -584,41 +586,41 @@ bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & p if (state.dead.count(path)) return false; if (state.roots.count(path)) { - debug(format("cannot delete '%1%' because it's a root") % path); - state.alive.insert(path); + debug("cannot delete '%1%' because it's a root", printStorePath(path)); + state.alive.insert(path.clone()); return true; } - visited.insert(path); + visited.insert(path.clone()); - if (!isStorePath(path) || !isValidPath(path)) return false; + //FIXME + if (!isStorePath(printStorePath(path)) || !isValidPath(path)) return false; - PathSet incoming; + StorePathSet incoming; /* Don't delete this path if any of its referrers are alive. */ queryReferrers(path, incoming); /* If keep-derivations is set and this is a derivation, then don't delete the derivation if any of the outputs are alive. */ - if (state.gcKeepDerivations && isDerivation(path)) { - PathSet outputs = queryDerivationOutputs(path); - for (auto & i : outputs) + if (state.gcKeepDerivations && path.isDerivation()) { + for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - incoming.insert(i); + incoming.insert(i.clone()); } /* If keep-outputs is set, then don't delete this path if there are derivers of this path that are not garbage. */ if (state.gcKeepOutputs) { - PathSet derivers = queryValidDerivers(path); + auto derivers = queryValidDerivers(path); for (auto & i : derivers) - incoming.insert(i); + incoming.insert(i.clone()); } for (auto & i : incoming) if (i != path) if (canReachRoot(state, visited, i)) { - state.alive.insert(path); + state.alive.insert(path.clone()); return true; } @@ -630,12 +632,13 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) { checkInterrupt(); - auto realPath = realStoreDir + "/" + baseNameOf(path); + auto realPath = realStoreDir + "/" + std::string(baseNameOf(path)); if (realPath == linksDir || realPath == trashDir) return; //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); - if (!isStorePath(path) || !isValidPath(path)) { + // FIXME + if (!isStorePath(path) || !isValidPath(parseStorePath(path))) { /* A lock file belonging to a path that we're building right now isn't garbage. */ if (isActiveTempFile(state, path, ".lock")) return; @@ -650,16 +653,17 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) if (isActiveTempFile(state, path, ".check")) return; } - PathSet visited; + StorePathSet visited; - if (canReachRoot(state, visited, path)) { - debug(format("cannot delete '%1%' because it's still reachable") % path); + if (canReachRoot(state, visited, parseStorePath(path))) { + debug("cannot delete '%s' because it's still reachable", path); } else { /* No path we visited was a root, so everything is garbage. But we only delete ‘path’ and its referrers here so that ‘nix-store --delete’ doesn't have the unexpected effect of recursing into derivations and outputs. */ - state.dead.insert(visited.begin(), visited.end()); + for (auto & i : visited) + state.dead.insert(i.clone()); if (state.shouldDelete) deletePathRecursive(state, path); } @@ -715,8 +719,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { - GCState state(results); - state.options = options; + GCState state(options, results); state.gcKeepOutputs = settings.gcKeepOutputs; state.gcKeepDerivations = settings.gcKeepDerivations; @@ -741,12 +744,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ - printError(format("finding garbage collector roots...")); + printError("finding garbage collector roots..."); Roots rootMap; if (!options.ignoreLiveness) findRootsNoTemp(rootMap, true); - for (auto & i : rootMap) state.roots.insert(i.first); + for (auto & i : rootMap) state.roots.insert(i.first.clone()); /* Read the temporary roots. This acquires read locks on all per-process temporary root files. So after this point no paths @@ -754,9 +757,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) FDs fds; Roots tempRoots; findTempRoots(fds, tempRoots, true); - for (auto & root : tempRoots) - state.tempRoots.insert(root.first); - state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); + for (auto & root : tempRoots) { + state.tempRoots.insert(root.first.clone()); + state.roots.insert(root.first.clone()); + } /* After this point the set of roots or temporary roots cannot increase, since we hold locks on everything. So everything @@ -768,7 +772,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) createDirs(trashDir); } catch (SysError & e) { if (e.errNo == ENOSPC) { - printInfo(format("note: can't create trash directory: %1%") % e.msg()); + printInfo("note: can't create trash directory: %s", e.msg()); state.moveToTrash = false; } } @@ -780,22 +784,21 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (options.action == GCOptions::gcDeleteSpecific) { for (auto & i : options.pathsToDelete) { - assertStorePath(i); - tryToDelete(state, i); + tryToDelete(state, printStorePath(i)); if (state.dead.find(i) == state.dead.end()) - throw Error(format( + throw Error( "cannot delete path '%1%' since it is still alive. " "To find out why use: " - "nix-store --query --roots" - ) % i); + "nix-store --query --roots", + printStorePath(i)); } } else if (options.maxFreed > 0) { if (state.shouldDelete) - printError(format("deleting garbage...")); + printError("deleting garbage..."); else - printError(format("determining live/dead paths...")); + printError("determining live/dead paths..."); try { @@ -815,7 +818,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) string name = dirent->d_name; if (name == "." || name == "..") continue; Path path = storeDir + "/" + name; - if (isStorePath(path) && isValidPath(path)) + // FIXME + if (isStorePath(path) && isValidPath(parseStorePath(path))) entries.push_back(path); else tryToDelete(state, path); @@ -840,12 +844,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } if (state.options.action == GCOptions::gcReturnLive) { - state.results.paths = state.alive; + for (auto & i : state.alive) + state.results.paths.insert(printStorePath(i)); return; } if (state.options.action == GCOptions::gcReturnDead) { - state.results.paths = state.dead; + for (auto & i : state.dead) + state.results.paths.insert(printStorePath(i)); return; } @@ -859,7 +865,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Clean up the links directory. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { - printError(format("deleting unused links...")); + printError("deleting unused links..."); removeUnusedLinks(state); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1b8b5908c..458266be0 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -87,25 +87,27 @@ struct LegacySSHStore : public Store return uriScheme + host; } - void queryPathInfoUncached(const Path & path, + void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override { try { auto conn(connections->get()); - debug("querying remote host '%s' for info on '%s'", host, path); + debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); - conn->to << cmdQueryPathInfos << PathSet{path}; + conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)}; conn->to.flush(); - auto info = std::make_shared<ValidPathInfo>(); - conn->from >> info->path; - if (info->path.empty()) return callback(nullptr); + auto p = readString(conn->from); + if (p.empty()) return callback(nullptr); + auto info = std::make_shared<ValidPathInfo>(parseStorePath(p)); assert(path == info->path); PathSet references; - conn->from >> info->deriver; - info->references = readStorePaths<PathSet>(*this, conn->from); + auto deriver = readString(conn->from); + if (deriver != "") + info->deriver = parseStorePath(deriver); + info->references = readStorePaths<StorePathSet>(*this, conn->from); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -127,7 +129,7 @@ struct LegacySSHStore : public Store RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override { - debug("adding path '%s' to remote host '%s'", info.path, host); + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); auto conn(connections->get()); @@ -135,10 +137,11 @@ struct LegacySSHStore : public Store conn->to << cmdAddToStoreNar - << info.path - << info.deriver - << info.narHash.to_string(Base16, false) - << info.references + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(Base16, false); + writeStorePaths(*this, conn->to, info.references); + conn->to << info.registrationTime << info.narSize << info.ultimate @@ -165,9 +168,10 @@ struct LegacySSHStore : public Store } conn->to << exportMagic - << info.path - << info.references - << info.deriver + << printStorePath(info.path); + writeStorePaths(*this, conn->to, info.references); + conn->to + << (info.deriver ? printStorePath(*info.deriver) : "") << 0 << 0; conn->to.flush(); @@ -175,39 +179,40 @@ struct LegacySSHStore : public Store } if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s', info.path, host"); + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } - void narFromPath(const Path & path, Sink & sink) override + void narFromPath(const StorePath & path, Sink & sink) override { auto conn(connections->get()); - conn->to << cmdDumpStorePath << path; + conn->to << cmdDumpStorePath << printStorePath(path); conn->to.flush(); copyNAR(conn->from, sink); } - Path queryPathFromHashPart(const string & hashPart) override + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } - Path addToStore(const string & name, const Path & srcPath, + StorePath addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override { unsupported("addToStore"); } - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override { unsupported("addTextToStore"); } - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override { auto conn(connections->get()); conn->to << cmdBuildDerivation - << drvPath - << drv + << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + conn->to << settings.maxSilentTime << settings.buildTimeout; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) @@ -230,11 +235,11 @@ struct LegacySSHStore : public Store return status; } - void ensurePath(const Path & path) override + void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } - void computeFSClosure(const PathSet & paths, - PathSet & out, bool flipDirection = false, + void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override { if (flipDirection || includeDerivers) { @@ -246,16 +251,15 @@ struct LegacySSHStore : public Store conn->to << cmdQueryClosure - << includeOutputs - << paths; + << includeOutputs; + writeStorePaths(*this, conn->to, paths); conn->to.flush(); - auto res = readStorePaths<PathSet>(*this, conn->from); - - out.insert(res.begin(), res.end()); + for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) + out.insert(i.clone()); } - PathSet queryValidPaths(const PathSet & paths, + StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override { auto conn(connections->get()); @@ -263,11 +267,11 @@ struct LegacySSHStore : public Store conn->to << cmdQueryValidPaths << false // lock - << maybeSubstitute - << paths; + << maybeSubstitute; + writeStorePaths(*this, conn->to, paths); conn->to.flush(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } void connect() override diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 9f99ee76d..363be1443 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -44,15 +44,15 @@ protected: } } - PathSet queryAllValidPaths() override + StorePathSet queryAllValidPaths() override { - PathSet paths; + StorePathSet paths; for (auto & entry : readDirectory(binaryCacheDir)) { if (entry.name.size() != 40 || !hasSuffix(entry.name, ".narinfo")) continue; - paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)); + paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8))); } return paths; diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 642e4070d..aa5abd835 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -21,7 +21,7 @@ struct LocalStoreAccessor : public FSAccessor Path toRealPath(const Path & path) { Path storePath = store->toStorePath(path); - if (!store->isValidPath(storePath)) + if (!store->isValidPath(store->parseStorePath(storePath))) throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -77,34 +77,32 @@ ref<FSAccessor> LocalFSStore::getFSAccessor() std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); } -void LocalFSStore::narFromPath(const Path & path, Sink & sink) +void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) { if (!isValidPath(path)) - throw Error(format("path '%s' is not valid") % path); - dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink); + throw Error("path '%s' is not valid", printStorePath(path)); + dumpPath(getRealStoreDir() + std::string(printStorePath(path), storeDir.size()), sink); } const string LocalFSStore::drvsLogDir = "drvs"; -std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path & path_) +std::shared_ptr<std::string> LocalFSStore::getBuildLog(const StorePath & path_) { - auto path(path_); + auto path = path_.clone(); - assertStorePath(path); - - - if (!isDerivation(path)) { + if (!path.isDerivation()) { try { - path = queryPathInfo(path)->deriver; + auto info = queryPathInfo(path); + if (!info->deriver) return nullptr; + path = info->deriver->clone(); } catch (InvalidPath &) { return nullptr; } - if (path == "") return nullptr; } - string baseName = baseNameOf(path); + auto baseName = std::string(baseNameOf(printStorePath(path))); for (int j = 0; j < 2; j++) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 023c90cd8..47b055bc1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -534,42 +534,36 @@ void canonicalisePathMetaData(const Path & path, uid_t fromUid) } -void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv) +void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv) { - string drvName = storePathToName(drvPath); - assert(isDerivation(drvName)); + assert(drvPath.isDerivation()); + std::string drvName(drvPath.name()); drvName = string(drvName, 0, drvName.size() - drvExtension.size()); if (drv.isFixedOutput()) { DerivationOutputs::const_iterator out = drv.outputs.find("out"); if (out == drv.outputs.end()) - throw Error(format("derivation '%1%' does not have an output named 'out'") % drvPath); + throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); bool recursive; Hash h; out->second.parseHashInfo(recursive, h); - Path outPath = makeFixedOutputPath(recursive, h, drvName); + auto outPath = makeFixedOutputPath(recursive, h, drvName); StringPairs::const_iterator j = drv.env.find("out"); - if (out->second.path != outPath || j == drv.env.end() || j->second != outPath) - throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'") - % drvPath % out->second.path % outPath); + if (out->second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + printStorePath(drvPath), printStorePath(out->second.path), printStorePath(outPath)); } else { - Derivation drvCopy(drv); - for (auto & i : drvCopy.outputs) { - i.second.path = ""; - drvCopy.env[i.first] = ""; - } - - Hash h = hashDerivationModulo(*this, drvCopy); + Hash h = hashDerivationModulo(*this, drv, true); for (auto & i : drv.outputs) { - Path outPath = makeOutputPath(i.first, h, drvName); + auto outPath = makeOutputPath(i.first, h, drvName); StringPairs::const_iterator j = drv.env.find(i.first); - if (i.second.path != outPath || j == drv.env.end() || j->second != outPath) - throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'") - % drvPath % i.second.path % outPath); + if (i.second.path != outPath || j == drv.env.end() || parseStorePath(j->second) != outPath) + throw Error("derivation '%s' has incorrect output '%s', should be '%s'", + printStorePath(drvPath), printStorePath(i.second.path), printStorePath(outPath)); } } } @@ -578,16 +572,15 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs) { - checkStoreName(storePathToName(info.path)); - if (info.ca != "" && !info.isContentAddressed(*this)) - throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path); + throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", + printStorePath(info.path)); state.stmtRegisterValidPath.use() - (info.path) + (printStorePath(info.path)) (info.narHash.to_string(Base16)) (info.registrationTime == 0 ? time(0) : info.registrationTime) - (info.deriver, info.deriver != "") + (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) @@ -599,8 +592,8 @@ uint64_t LocalStore::addValidPath(State & state, the database. This is useful for the garbage collector: it can efficiently query whether a path is an output of some derivation. */ - if (isDerivation(info.path)) { - Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path)); + if (info.path.isDerivation()) { + auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string())); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -613,34 +606,31 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtAddDerivationOutput.use() (id) (i.first) - (i.second.path) + (printStorePath(i.second.path)) .exec(); } } { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared<ValidPathInfo>(info)); + state_->pathInfoCache.upsert(storePathToHash(printStorePath(info.path)), std::make_shared<ValidPathInfo>(info)); } return id; } -void LocalStore::queryPathInfoUncached(const Path & path, +void LocalStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { - auto info = std::make_shared<ValidPathInfo>(); - info->path = path; - - assertStorePath(path); + auto info = std::make_shared<ValidPathInfo>(path.clone()); callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { auto state(_state.lock()); /* Get the path info. */ - auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(info->path))); if (!useQueryPathInfo.next()) return std::shared_ptr<ValidPathInfo>(); @@ -650,13 +640,13 @@ void LocalStore::queryPathInfoUncached(const Path & path, try { info->narHash = Hash(useQueryPathInfo.getStr(1)); } catch (BadHash & e) { - throw Error("in valid-path entry for '%s': %s", path, e.what()); + throw Error("in valid-path entry for '%s': %s", printStorePath(path), e.what()); } info->registrationTime = useQueryPathInfo.getInt(2); auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); - if (s) info->deriver = s; + if (s) info->deriver = parseStorePath(s); /* Note that narSize = NULL yields 0. */ info->narSize = useQueryPathInfo.getInt(4); @@ -673,7 +663,7 @@ void LocalStore::queryPathInfoUncached(const Path & path, auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->references.insert(useQueryReferences.getStr(0)); + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); return info; })); @@ -691,27 +681,27 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) - (info.path) + (printStorePath(info.path)) .exec(); } -uint64_t LocalStore::queryValidPathId(State & state, const Path & path) +uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path) { - auto use(state.stmtQueryPathInfo.use()(path)); + auto use(state.stmtQueryPathInfo.use()(printStorePath(path))); if (!use.next()) - throw Error(format("path '%1%' is not valid") % path); + throw Error("path '%s' is not valid", printStorePath(path)); return use.getInt(0); } -bool LocalStore::isValidPath_(State & state, const Path & path) +bool LocalStore::isValidPath_(State & state, const StorePath & path) { - return state.stmtQueryPathInfo.use()(path).next(); + return state.stmtQueryPathInfo.use()(printStorePath(path)).next(); } -bool LocalStore::isValidPathUncached(const Path & path) +bool LocalStore::isValidPathUncached(const StorePath & path) { return retrySQLite<bool>([&]() { auto state(_state.lock()); @@ -720,39 +710,38 @@ bool LocalStore::isValidPathUncached(const Path & path) } -PathSet LocalStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) +StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { - PathSet res; + StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(i)) res.insert(i.clone()); return res; } -PathSet LocalStore::queryAllValidPaths() +StorePathSet LocalStore::queryAllValidPaths() { - return retrySQLite<PathSet>([&]() { + return retrySQLite<StorePathSet>([&]() { auto state(_state.lock()); auto use(state->stmtQueryValidPaths.use()); - PathSet res; - while (use.next()) res.insert(use.getStr(0)); + StorePathSet res; + while (use.next()) res.insert(parseStorePath(use.getStr(0))); return res; }); } -void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers) { - auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); + auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path))); while (useQueryReferrers.next()) - referrers.insert(useQueryReferrers.getStr(0)); + referrers.insert(parseStorePath(useQueryReferrers.getStr(0))); } -void LocalStore::queryReferrers(const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(const StorePath & path, StorePathSet & referrers) { - assertStorePath(path); return retrySQLite<void>([&]() { auto state(_state.lock()); queryReferrers(*state, path, referrers); @@ -760,42 +749,40 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) } -PathSet LocalStore::queryValidDerivers(const Path & path) +StorePathSet LocalStore::queryValidDerivers(const StorePath & path) { - assertStorePath(path); - - return retrySQLite<PathSet>([&]() { + return retrySQLite<StorePathSet>([&]() { auto state(_state.lock()); - auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path))); - PathSet derivers; + StorePathSet derivers; while (useQueryValidDerivers.next()) - derivers.insert(useQueryValidDerivers.getStr(1)); + derivers.insert(parseStorePath(useQueryValidDerivers.getStr(1))); return derivers; }); } -PathSet LocalStore::queryDerivationOutputs(const Path & path) +StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) { - return retrySQLite<PathSet>([&]() { + return retrySQLite<StorePathSet>([&]() { auto state(_state.lock()); auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() (queryValidPathId(*state, path))); - PathSet outputs; + StorePathSet outputs; while (useQueryDerivationOutputs.next()) - outputs.insert(useQueryDerivationOutputs.getStr(1)); + outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1))); return outputs; }); } -StringSet LocalStore::queryDerivationOutputNames(const Path & path) +StringSet LocalStore::queryDerivationOutputNames(const StorePath & path) { return retrySQLite<StringSet>([&]() { auto state(_state.lock()); @@ -812,31 +799,36 @@ StringSet LocalStore::queryDerivationOutputNames(const Path & path) } -Path LocalStore::queryPathFromHashPart(const string & hashPart) +std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart) { if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); Path prefix = storeDir + "/" + hashPart; - return retrySQLite<Path>([&]() -> std::string { + return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> { auto state(_state.lock()); auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); - if (!useQueryPathFromHashPart.next()) return ""; + if (!useQueryPathFromHashPart.next()) return {}; const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); - return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; + if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0) + return parseStorePath(s); + return {}; }); } -PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) +StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) { - if (!settings.useSubstitutes) return PathSet(); + if (!settings.useSubstitutes) return StorePathSet(); - auto remaining = paths; - PathSet res; + StorePathSet remaining; + for (auto & i : paths) + remaining.insert(i.clone()); + + StorePathSet res; for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; @@ -845,12 +837,12 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) auto valid = sub->queryValidPaths(remaining); - PathSet remaining2; + StorePathSet remaining2; for (auto & path : remaining) if (valid.count(path)) - res.insert(path); + res.insert(path.clone()); else - remaining2.insert(path); + remaining2.insert(path.clone()); std::swap(remaining, remaining2); } @@ -859,7 +851,7 @@ PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) } -void LocalStore::querySubstitutablePathInfos(const PathSet & paths, +void LocalStore::querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) { if (!settings.useSubstitutes) return; @@ -867,17 +859,16 @@ void LocalStore::querySubstitutablePathInfos(const PathSet & paths, if (sub->storeDir != storeDir) continue; for (auto & path : paths) { if (infos.count(path)) continue; - debug(format("checking substituter '%s' for path '%s'") - % sub->getUri() % path); + debug("checking substituter '%s' for path '%s'", sub->getUri(), printStorePath(path)); try { auto info = sub->queryPathInfo(path); auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); - infos[path] = SubstitutablePathInfo{ - info->deriver, - info->references, + infos.insert_or_assign(path.clone(), SubstitutablePathInfo{ + info->deriver ? info->deriver->clone() : std::optional<StorePath>(), + cloneStorePathSet(info->references), narInfo ? narInfo->fileSize : 0, - info->narSize}; + info->narSize}); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { @@ -911,7 +902,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) auto state(_state.lock()); SQLiteTxn txn(state->db); - PathSet paths; + StorePathSet paths; for (auto & i : infos) { assert(i.narHash.type == htSHA256); @@ -919,7 +910,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) updatePathInfo(*state, i); else addValidPath(*state, i, false); - paths.insert(i.path); + paths.insert(i.path.clone()); } for (auto & i : infos) { @@ -932,10 +923,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) this in addValidPath() above, because the references might not be valid yet. */ for (auto & i : infos) - if (isDerivation(i.path)) { + if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(i.path)); - checkDerivationOutputs(i.path, drv); + checkDerivationOutputs(i.path, + readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string()))); } /* Do a topological sort of the paths. This will throw an @@ -951,18 +942,18 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) /* Invalidate a path. The caller is responsible for checking that there are no referrers. */ -void LocalStore::invalidatePath(State & state, const Path & path) +void LocalStore::invalidatePath(State & state, const StorePath & path) { - debug(format("invalidating path '%1%'") % path); + debug("invalidating path '%s'", printStorePath(path)); - state.stmtInvalidatePath.use()(path).exec(); + state.stmtInvalidatePath.use()(printStorePath(path)).exec(); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(storePathToHash(path)); + state_->pathInfoCache.erase(storePathToHash(printStorePath(path))); } } @@ -980,10 +971,10 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) { if (!info.narHash) - throw Error("cannot add path '%s' because it lacks a hash", info.path); + throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path)); if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) - throw Error("cannot add path '%s' because it lacks a valid signature", info.path); + throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path)); addTempRoot(info.path); @@ -991,12 +982,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, PathLocks outputLock; - Path realPath = realStoreDir + "/" + baseNameOf(info.path); + Path realPath = realStoreDir + "/" + std::string(info.path.to_string()); /* Lock the output path. But don't lock if we're being called from a build hook (whose parent process already acquired a lock on this path). */ - if (!locksHeld.count(info.path)) + if (!locksHeld.count(printStorePath(info.path))) outputLock.lockPaths({realPath}); if (repair || !isValidPath(info.path)) { @@ -1011,7 +1002,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, else { if (!info.references.empty()) settings.requireExperimentalFeature("ca-references"); - hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(info.path)); + hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path))); } LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { @@ -1026,11 +1017,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - info.path, info.narHash.to_string(), hashResult.first.to_string()); + printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string()); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", - info.path, info.narSize, hashResult.second); + printStorePath(info.path), info.narSize, hashResult.second); autoGC(); @@ -1046,12 +1037,12 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, } -Path LocalStore::addToStoreFromDump(const string & dump, const string & name, +StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name, bool recursive, HashType hashAlgo, RepairFlag repair) { Hash h = hashString(hashAlgo, dump); - Path dstPath = makeFixedOutputPath(recursive, h, name); + auto dstPath = makeFixedOutputPath(recursive, h, name); addTempRoot(dstPath); @@ -1060,7 +1051,8 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, /* The first check above is an optimisation to prevent unnecessary lock acquisition. */ - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + Path realPath = realStoreDir + "/"; + realPath += dstPath.to_string(); PathLocks outputLock({realPath}); @@ -1091,8 +1083,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, optimisePath(realPath); // FIXME: combine with hashPath() - ValidPathInfo info; - info.path = dstPath; + ValidPathInfo info(dstPath.clone()); info.narHash = hash.first; info.narSize = hash.second; info.ca = makeFixedOutputCA(recursive, h); @@ -1106,7 +1097,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, } -Path LocalStore::addToStore(const string & name, const Path & _srcPath, +StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { Path srcPath(absPath(_srcPath)); @@ -1124,8 +1115,8 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath, } -Path LocalStore::addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) +StorePath LocalStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); auto dstPath = makeTextPath(name, hash, references); @@ -1134,7 +1125,8 @@ Path LocalStore::addTextToStore(const string & name, const string & s, if (repair || !isValidPath(dstPath)) { - Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + Path realPath = realStoreDir + "/"; + realPath += dstPath.to_string(); PathLocks outputLock({realPath}); @@ -1154,11 +1146,10 @@ Path LocalStore::addTextToStore(const string & name, const string & s, optimisePath(realPath); - ValidPathInfo info; - info.path = dstPath; + ValidPathInfo info(dstPath.clone()); info.narHash = narHash; info.narSize = sink.s->size(); - info.references = references; + info.references = cloneStorePathSet(references); info.ca = "text:" + hash.to_string(); registerValidPath(info); } @@ -1180,27 +1171,25 @@ Path LocalStore::createTempDirInStore() the GC between createTempDir() and addTempRoot(), so repeat until `tmpDir' exists. */ tmpDir = createTempDir(realStoreDir); - addTempRoot(tmpDir); + addTempRoot(parseStorePath(tmpDir)); } while (!pathExists(tmpDir)); return tmpDir; } -void LocalStore::invalidatePathChecked(const Path & path) +void LocalStore::invalidatePathChecked(const StorePath & path) { - assertStorePath(path); - retrySQLite<void>([&]() { auto state(_state.lock()); SQLiteTxn txn(state->db); if (isValidPath_(*state, path)) { - PathSet referrers; queryReferrers(*state, path, referrers); + StorePathSet referrers; queryReferrers(*state, path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) - throw PathInUse(format("cannot delete path '%1%' because it is in use by %2%") - % path % showPaths(referrers)); + throw PathInUse("cannot delete path '%s' because it is in use by %s", + printStorePath(path), showPaths(referrers)); invalidatePath(*state, path); } @@ -1219,18 +1208,19 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) existing and valid paths. */ AutoCloseFD fdGCLock = openGCLock(ltWrite); - PathSet store; + StringSet store; for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); /* Check whether all valid paths actually exist. */ printInfo("checking path existence..."); - PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + StorePathSet validPaths; + PathSet done; fdGCLock = -1; - for (auto & i : validPaths2) - verifyPath(i, store, done, validPaths, repair, errors); + for (auto & i : queryAllValidPaths()) + verifyPath(printStorePath(i), store, done, validPaths, repair, errors); /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1265,21 +1255,20 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, format("checking contents of '%1%'") % i); + printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); std::unique_ptr<AbstractHashSink> hashSink; if (info->ca == "") hashSink = std::make_unique<HashSink>(info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path)); + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path))); - dumpPath(toRealPath(i), *hashSink); + dumpPath(toRealPath(printStorePath(i)), *hashSink); auto current = hashSink->finish(); if (info->narHash != nullHash && info->narHash != current.first) { - printError(format("path '%1%' was modified! " - "expected hash '%2%', got '%3%'") - % i % info->narHash.to_string() % current.first.to_string()); + printError("path '%s' was modified! expected hash '%s', got '%s'", + printStorePath(i), info->narHash.to_string(), current.first.to_string()); if (repair) repairPath(i); else errors = true; } else { @@ -1287,14 +1276,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Fill in missing hashes. */ if (info->narHash == nullHash) { - printError(format("fixing missing hash on '%1%'") % i); + printError("fixing missing hash on '%s'", printStorePath(i)); info->narHash = current.first; update = true; } /* Fill in missing narSize fields (from old stores). */ if (info->narSize == 0) { - printError(format("updating size field on '%1%' to %2%") % i % current.second); + printError("updating size field on '%s' to %s", printStorePath(i), current.second); info->narSize = current.second; update = true; } @@ -1310,9 +1299,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* It's possible that the path got GC'ed, so ignore errors on invalid paths. */ if (isValidPath(i)) - printError(format("error: %1%") % e.msg()); + printError("error: %s", e.msg()); else - printError(format("warning: %1%") % e.msg()); + warn(e.msg()); errors = true; } } @@ -1322,43 +1311,43 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors) +void LocalStore::verifyPath(const Path & pathS, const StringSet & store, + PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (!done.insert(path).second) return; + if (!done.insert(pathS).second) return; - if (!isStorePath(path)) { - printError(format("path '%1%' is not in the Nix store") % path); - auto state(_state.lock()); - invalidatePath(*state, path); + if (!isStorePath(pathS)) { + printError("path '%s' is not in the Nix store", pathS); return; } - if (store.find(baseNameOf(path)) == store.end()) { + auto path = parseStorePath(pathS); + + if (!store.count(std::string(path.to_string()))) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; - PathSet referrers; queryReferrers(path, referrers); + StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(i, store, done, validPaths, repair, errors); - if (validPaths.find(i) != validPaths.end()) + verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + if (validPaths.count(i)) canInvalidate = false; } if (canInvalidate) { - printError(format("path '%1%' disappeared, removing from database...") % path); + printError("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); invalidatePath(*state, path); } else { - printError(format("path '%1%' disappeared, but it still has valid referrers!") % path); + printError("path '%s' disappeared, but it still has valid referrers!", pathS); if (repair) try { repairPath(path); } catch (Error & e) { - printError(format("warning: %1%") % e.msg()); + warn(e.msg()); errors = true; } else errors = true; @@ -1367,7 +1356,7 @@ void LocalStore::verifyPath(const Path & path, const PathSet & store, return; } - validPaths.insert(path); + validPaths.insert(std::move(path)); } @@ -1436,7 +1425,7 @@ void LocalStore::vacuumDB() } -void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) +void LocalStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { retrySQLite<void>([&]() { auto state(_state.lock()); @@ -1462,7 +1451,7 @@ void LocalStore::signPathInfo(ValidPathInfo & info) for (auto & secretKeyFile : secretKeyFiles.get()) { SecretKey secretKey(readFile(secretKeyFile)); - info.sign(secretKey); + info.sign(*this, secretKey); } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1d4b88898..16aeab0ad 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -119,36 +119,36 @@ public: std::string getUri() override; - bool isValidPathUncached(const Path & path) override; + bool isValidPathUncached(const StorePath & path) override; - PathSet queryValidPaths(const PathSet & paths, + StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; - PathSet queryAllValidPaths() override; + StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const Path & path, + void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override; - void queryReferrers(const Path & path, PathSet & referrers) override; + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; - PathSet queryValidDerivers(const Path & path) override; + StorePathSet queryValidDerivers(const StorePath & path) override; - PathSet queryDerivationOutputs(const Path & path) override; + StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const Path & path) override; + StringSet queryDerivationOutputNames(const StorePath & path) override; - Path queryPathFromHashPart(const string & hashPart) override; + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; - PathSet querySubstitutablePaths(const PathSet & paths) override; + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const PathSet & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; - Path addToStore(const string & name, const Path & srcPath, + StorePath addToStore(const string & name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) override; @@ -156,20 +156,22 @@ public: in `dump', which is either a NAR serialisation (if recursive == true) or simply the contents of a regular file (if recursive == false). */ - Path addToStoreFromDump(const string & dump, const string & name, + StorePath addToStoreFromDump(const string & dump, const string & name, bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override; + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override; + void buildPaths( + const std::vector<StorePathWithOutputs> & paths, + BuildMode buildMode) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const Path & path) override; + void ensurePath(const StorePath & path) override; - void addTempRoot(const Path & path) override; + void addTempRoot(const StorePath & path) override; void addIndirectRoot(const Path & path) override; @@ -215,9 +217,9 @@ public: /* Repair the contents of the given path by redownloading it using a substituter (if available). */ - void repairPath(const Path & path); + void repairPath(const StorePath & path); - void addSignatures(const Path & storePath, const StringSet & sigs) override; + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; /* If free disk space in /nix/store if below minFree, delete garbage until it exceeds maxFree. */ @@ -231,17 +233,17 @@ private: void makeStoreWritable(); - uint64_t queryValidPathId(State & state, const Path & path); + uint64_t queryValidPathId(State & state, const StorePath & path); uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); - void invalidatePath(State & state, const Path & path); + void invalidatePath(State & state, const StorePath & path); /* Delete a path from the Nix store. */ - void invalidatePathChecked(const Path & path); + void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const PathSet & store, - PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors); + void verifyPath(const Path & path, const StringSet & store, + PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); void updatePathInfo(State & state, const ValidPathInfo & info); @@ -256,7 +258,7 @@ private: void tryToDelete(GCState & state, const Path & path); - bool canReachRoot(GCState & state, PathSet & visited, const Path & path); + bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path); void deletePathRecursive(GCState & state, const Path & path); @@ -275,7 +277,7 @@ private: Path createTempDirInStore(); - void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); typedef std::unordered_set<ino_t> InodeHash; @@ -284,8 +286,8 @@ private: void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash); // Internal versions that are not wrapped in retry_sqlite. - bool isValidPath_(State & state, const Path & path); - void queryReferrers(State & state, const Path & path, PathSet & referrers); + bool isValidPath_(State & state, const StorePath & path); + void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers); /* Add signatures to a ValidPathInfo using the secret keys specified by the ‘secret-key-files’ option. */ diff --git a/src/libstore/local.mk b/src/libstore/local.mk index d690fea28..d3254554d 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -6,7 +6,7 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) -libstore_LIBS = libutil +libstore_LIBS = libutil libnixrust libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread ifneq ($(OS), FreeBSD) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 2fceb9b9a..9c47fe524 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -9,13 +9,13 @@ namespace nix { -void Store::computeFSClosure(const PathSet & startPaths, - PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) +void Store::computeFSClosure(const StorePathSet & startPaths, + StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { struct State { size_t pending; - PathSet & paths; + StorePathSet & paths; std::exception_ptr exc; }; @@ -29,45 +29,47 @@ void Store::computeFSClosure(const PathSet & startPaths, { auto state(state_.lock()); if (state->exc) return; - if (!state->paths.insert(path).second) return; + if (!state->paths.insert(parseStorePath(path)).second) return; state->pending++; } - queryPathInfo(path, {[&, path](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future<ref<const ValidPathInfo>> fut) { // FIXME: calls to isValidPath() should be async try { auto info = fut.get(); + auto path = parseStorePath(pathS); + if (flipDirection) { - PathSet referrers; + StorePathSet referrers; queryReferrers(path, referrers); for (auto & ref : referrers) if (ref != path) - enqueue(ref); + enqueue(printStorePath(ref)); if (includeOutputs) for (auto & i : queryValidDerivers(path)) - enqueue(i); + enqueue(printStorePath(i)); - if (includeDerivers && isDerivation(path)) + if (includeDerivers && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) if (isValidPath(i) && queryPathInfo(i)->deriver == path) - enqueue(i); + enqueue(printStorePath(i)); } else { for (auto & ref : info->references) if (ref != path) - enqueue(ref); + enqueue(printStorePath(ref)); - if (includeOutputs && isDerivation(path)) + if (includeOutputs && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) - if (isValidPath(i)) enqueue(i); + if (isValidPath(i)) enqueue(printStorePath(i)); - if (includeDerivers && isValidPath(info->deriver)) - enqueue(info->deriver); + if (includeDerivers && info->deriver && isValidPath(*info->deriver)) + enqueue(printStorePath(*info->deriver)); } @@ -87,7 +89,7 @@ void Store::computeFSClosure(const PathSet & startPaths, }; for (auto & startPath : startPaths) - enqueue(startPath); + enqueue(printStorePath(startPath)); { auto state(state_.lock()); @@ -97,15 +99,17 @@ void Store::computeFSClosure(const PathSet & startPaths, } -void Store::computeFSClosure(const Path & startPath, - PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) +void Store::computeFSClosure(const StorePath & startPath, + StorePathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) { - computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers); + StorePathSet paths; + paths.insert(startPath.clone()); + computeFSClosure(paths, paths_, flipDirection, includeOutputs, includeDerivers); } -void Store::queryMissing(const PathSet & targets, - PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_, +void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, + StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, unsigned long long & downloadSize_, unsigned long long & narSize_) { Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); @@ -116,8 +120,8 @@ void Store::queryMissing(const PathSet & targets, struct State { - PathSet done; - PathSet & unknown, & willSubstitute, & willBuild; + std::unordered_set<std::string> done; + StorePathSet & unknown, & willSubstitute, & willBuild; unsigned long long & downloadSize; unsigned long long & narSize; }; @@ -126,31 +130,36 @@ void Store::queryMissing(const PathSet & targets, { size_t left; bool done = false; - PathSet outPaths; + StorePathSet outPaths; DrvState(size_t left) : left(left) { } }; - Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); + Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); - std::function<void(Path)> doPath; + std::function<void(StorePathWithOutputs)> doPath; - auto mustBuildDrv = [&](const Path & drvPath, const Derivation & drv) { + auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); - state->willBuild.insert(drvPath); + state->willBuild.insert(drvPath.clone()); } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second))); + pool.enqueue(std::bind(doPath, StorePathWithOutputs(i.first, i.second))); }; auto checkOutput = [&]( - const Path & drvPath, ref<Derivation> drv, const Path & outPath, ref<Sync<DrvState>> drvState_) + const Path & drvPathS, ref<Derivation> drv, const Path & outPathS, ref<Sync<DrvState>> drvState_) { if (drvState_->lock()->done) return; + auto drvPath = parseStorePath(drvPathS); + auto outPath = parseStorePath(outPathS); + SubstitutablePathInfos infos; - querySubstitutablePathInfos({outPath}, infos); + StorePathSet paths; // FIXME + paths.insert(outPath.clone()); + querySubstitutablePathInfos(paths, infos); if (infos.empty()) { drvState_->lock()->done = true; @@ -161,74 +170,74 @@ void Store::queryMissing(const PathSet & targets, if (drvState->done) return; assert(drvState->left); drvState->left--; - drvState->outPaths.insert(outPath); + drvState->outPaths.insert(outPath.clone()); if (!drvState->left) { for (auto & path : drvState->outPaths) - pool.enqueue(std::bind(doPath, path)); + pool.enqueue(std::bind(doPath, StorePathWithOutputs(path.clone()))); } } } }; - doPath = [&](const Path & path) { + doPath = [&](const StorePathWithOutputs & path) { { auto state(state_.lock()); - if (!state->done.insert(path).second) return; + if (!state->done.insert(path.to_string(*this)).second) return; } - DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); - - if (isDerivation(i2.first)) { - if (!isValidPath(i2.first)) { + if (path.path.isDerivation()) { + if (!isValidPath(path.path)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(path); + state->unknown.insert(path.path.clone()); return; } - Derivation drv = derivationFromPath(i2.first); - ParsedDerivation parsedDrv(i2.first, drv); + auto drv = make_ref<Derivation>(derivationFromPath(path.path)); + ParsedDerivation parsedDrv(path.path.clone(), *drv); PathSet invalid; - for (auto & j : drv.outputs) - if (wantOutput(j.first, i2.second) + for (auto & j : drv->outputs) + if (wantOutput(j.first, path.outputs) && !isValidPath(j.second.path)) - invalid.insert(j.second.path); + invalid.insert(printStorePath(j.second.path)); if (invalid.empty()) return; if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, i2.first, make_ref<Derivation>(drv), output, drvState)); + pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState)); } else - mustBuildDrv(i2.first, drv); + mustBuildDrv(path.path, *drv); } else { - if (isValidPath(path)) return; + if (isValidPath(path.path)) return; SubstitutablePathInfos infos; - querySubstitutablePathInfos({path}, infos); + StorePathSet paths; // FIXME + paths.insert(path.path.clone()); + querySubstitutablePathInfos(paths, infos); if (infos.empty()) { auto state(state_.lock()); - state->unknown.insert(path); + state->unknown.insert(path.path.clone()); return; } - auto info = infos.find(path); + auto info = infos.find(path.path); assert(info != infos.end()); { auto state(state_.lock()); - state->willSubstitute.insert(path); + state->willSubstitute.insert(path.path.clone()); state->downloadSize += info->second.downloadSize; state->narSize += info->second.narSize; } for (auto & ref : info->second.references) - pool.enqueue(std::bind(doPath, ref)); + pool.enqueue(std::bind(doPath, StorePathWithOutputs(ref))); } }; @@ -239,39 +248,42 @@ void Store::queryMissing(const PathSet & targets, } -Paths Store::topoSortPaths(const PathSet & paths) +StorePaths Store::topoSortPaths(const StorePathSet & paths) { - Paths sorted; - PathSet visited, parents; + StorePaths sorted; + StorePathSet visited, parents; - std::function<void(const Path & path, const Path * parent)> dfsVisit; + std::function<void(const StorePath & path, const StorePath * parent)> dfsVisit; - dfsVisit = [&](const Path & path, const Path * parent) { - if (parents.find(path) != parents.end()) - throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent); + dfsVisit = [&](const StorePath & path, const StorePath * parent) { + if (parents.count(path)) + throw BuildError("cycle detected in the references of '%s' from '%s'", + printStorePath(path), printStorePath(*parent)); - if (!visited.insert(path).second) return; - parents.insert(path); + if (!visited.insert(path.clone()).second) return; + parents.insert(path.clone()); - PathSet references; + StorePathSet references; try { - references = queryPathInfo(path)->references; + references = cloneStorePathSet(queryPathInfo(path)->references); } catch (InvalidPath &) { } for (auto & i : references) /* Don't traverse into paths that don't exist. That can happen due to substitutes for non-existent paths. */ - if (i != path && paths.find(i) != paths.end()) + if (i != path && paths.count(i)) dfsVisit(i, &path); - sorted.push_front(path); + sorted.push_back(path.clone()); parents.erase(path); }; for (auto & i : paths) dfsVisit(i, nullptr); + std::reverse(sorted.begin(), sorted.end()); + return sorted; } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 37c4c72e0..2bd515536 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -188,11 +188,8 @@ public: if (!queryNAR.getInt(0)) return {oInvalid, 0}; - auto narInfo = make_ref<NarInfo>(); - auto namePart = queryNAR.getStr(1); - narInfo->path = cache.storeDir + "/" + - hashPart + (namePart.empty() ? "" : "-" + namePart); + auto narInfo = make_ref<NarInfo>(StorePath::fromBaseName(hashPart + "-" + namePart)); narInfo->url = queryNAR.getStr(2); narInfo->compression = queryNAR.getStr(3); if (!queryNAR.isNull(4)) @@ -201,9 +198,9 @@ public: narInfo->narHash = Hash(queryNAR.getStr(6)); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) - narInfo->references.insert(cache.storeDir + "/" + r); + narInfo->references.insert(StorePath::fromBaseName(r)); if (!queryNAR.isNull(9)) - narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9); + narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); narInfo->ca = queryNAR.getStr(11); @@ -225,12 +222,12 @@ public: auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); - assert(hashPart == storePathToHash(info->path)); + //assert(hashPart == storePathToHash(info->path)); state->insertNAR.use() (cache.id) (hashPart) - (storePathToName(info->path)) + (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) @@ -238,7 +235,7 @@ public: (info->narHash.to_string()) (info->narSize) (concatStringsSep(" ", info->shortRefs())) - (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") + (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) (info->ca) (time(0)).exec(); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index cb568ccdc..fb02cf3fd 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,6 +4,7 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) + : ValidPathInfo(StorePath::make((unsigned char *) "xxxxxxxxxxxxxxxxxxxx", "x")) // FIXME: hack { auto corrupt = [&]() { throw Error(format("NAR info file '%1%' is corrupt") % whence); @@ -18,6 +19,8 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & } }; + bool havePath = false; + size_t pos = 0; while (pos < s.size()) { @@ -32,8 +35,8 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & std::string value(s, colon + 2, eol - colon - 2); if (name == "StorePath") { - if (!store.isStorePath(value)) corrupt(); - path = value; + path = store.parseStorePath(value); + havePath = true; } else if (name == "URL") url = value; @@ -52,18 +55,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "References") { auto refs = tokenizeString<Strings>(value, " "); if (!references.empty()) corrupt(); - for (auto & r : refs) { - auto r2 = store.storeDir + "/" + r; - if (!store.isStorePath(r2)) corrupt(); - references.insert(r2); - } + for (auto & r : refs) + references.insert(StorePath::fromBaseName(r)); } else if (name == "Deriver") { - if (value != "unknown-deriver") { - auto p = store.storeDir + "/" + value; - if (!store.isStorePath(p)) corrupt(); - deriver = p; - } + if (value != "unknown-deriver") + deriver = StorePath::fromBaseName(value); } else if (name == "System") system = value; @@ -79,13 +76,13 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt(); + if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt(); } -std::string NarInfo::to_string() const +std::string NarInfo::to_string(const Store & store) const { std::string res; - res += "StorePath: " + path + "\n"; + res += "StorePath: " + store.printStorePath(path) + "\n"; res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; @@ -98,8 +95,8 @@ std::string NarInfo::to_string() const res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; - if (!deriver.empty()) - res += "Deriver: " + baseNameOf(deriver) + "\n"; + if (deriver) + res += "Deriver: " + std::string(deriver->to_string()) + "\n"; if (!system.empty()) res += "System: " + system + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 4995061fb..373c33427 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -14,11 +14,12 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; std::string system; - NarInfo() { } + NarInfo() = delete; + NarInfo(StorePath && path) : ValidPathInfo(std::move(path)) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); - std::string to_string() const; + std::string to_string(const Store & store) const; }; } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 991512f21..8ac382e9d 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -254,7 +254,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) { Activity act(*logger, actOptimiseStore); - PathSet paths = queryAllValidPaths(); + auto paths = queryAllValidPaths(); InodeHash inodeHash = loadInodeHash(); act.progress(0, paths.size()); @@ -265,8 +265,8 @@ void LocalStore::optimiseStore(OptimiseStats & stats) addTempRoot(i); if (!isValidPath(i)) continue; /* path was GC'ed, probably */ { - Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i)); - optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash); + Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); + optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash); } done++; act.progress(done, paths.size()); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 5553dd863..45c033c66 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -4,8 +4,8 @@ namespace nix { -ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) - : drvPath(drvPath), drv(drv) +ParsedDerivation::ParsedDerivation(StorePath && drvPath, BasicDerivation & drv) + : drvPath(std::move(drvPath)), drv(drv) { /* Parse the __json attribute, if any. */ auto jsonAttr = drv.env.find("__json"); @@ -13,7 +13,7 @@ ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) try { structuredAttrs = std::make_unique<nlohmann::json>(nlohmann::json::parse(jsonAttr->second)); } catch (std::exception & e) { - throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); + throw Error("cannot process __json attribute of '%s': %s", drvPath.to_string(), e.what()); } } } @@ -28,7 +28,7 @@ std::optional<std::string> ParsedDerivation::getStringAttr(const std::string & n return {}; else { if (!i->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath); + throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath.to_string()); return i->get<std::string>(); } } else { @@ -48,7 +48,7 @@ bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const return def; else { if (!i->is_boolean()) - throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath); + throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath.to_string()); return i->get<bool>(); } } else { @@ -68,11 +68,11 @@ std::optional<Strings> ParsedDerivation::getStringsAttr(const std::string & name return {}; else { if (!i->is_array()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string()); Strings res; for (auto j = i->begin(); j != i->end(); ++j) { if (!j->is_string()) - throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath.to_string()); res.push_back(j->get<std::string>()); } return res; diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index 6e67e1665..f4df5dd54 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -6,13 +6,13 @@ namespace nix { class ParsedDerivation { - Path drvPath; + StorePath drvPath; BasicDerivation & drv; std::unique_ptr<nlohmann::json> structuredAttrs; public: - ParsedDerivation(const Path & drvPath, BasicDerivation & drv); + ParsedDerivation(StorePath && drvPath, BasicDerivation & drv); ~ParsedDerivation(); diff --git a/src/libstore/path.cc b/src/libstore/path.cc new file mode 100644 index 000000000..81ae495a1 --- /dev/null +++ b/src/libstore/path.cc @@ -0,0 +1,99 @@ +#include "store-api.hh" + +namespace nix { + +extern "C" { + rust::Result<StorePath> ffi_StorePath_new(rust::StringSlice path, rust::StringSlice storeDir); + rust::Result<StorePath> ffi_StorePath_new2(unsigned char hash[20], rust::StringSlice storeDir); + rust::Result<StorePath> ffi_StorePath_fromBaseName(rust::StringSlice baseName); + rust::String ffi_StorePath_to_string(const StorePath & _this); + StorePath ffi_StorePath_clone(const StorePath & _this); + rust::StringSlice ffi_StorePath_name(const StorePath & _this); +} + +StorePath StorePath::make(std::string_view path, std::string_view storeDir) +{ + return ffi_StorePath_new((rust::StringSlice) path, (rust::StringSlice) storeDir).unwrap(); +} + +StorePath StorePath::make(unsigned char hash[20], std::string_view name) +{ + return ffi_StorePath_new2(hash, (rust::StringSlice) name).unwrap(); +} + +StorePath StorePath::fromBaseName(std::string_view baseName) +{ + return ffi_StorePath_fromBaseName((rust::StringSlice) baseName).unwrap(); +} + +rust::String StorePath::to_string() const +{ + return ffi_StorePath_to_string(*this); +} + +StorePath StorePath::clone() const +{ + return ffi_StorePath_clone(*this); +} + +bool StorePath::isDerivation() const +{ + return hasSuffix(name(), drvExtension); +} + +std::string_view StorePath::name() const +{ + return ffi_StorePath_name(*this); +} + +StorePath Store::parseStorePath(std::string_view path) const +{ + return StorePath::make(path, storeDir); +} + + +StorePathSet Store::parseStorePathSet(const PathSet & paths) const +{ + StorePathSet res; + for (auto & i : paths) res.insert(parseStorePath(i)); + return res; +} + +std::string Store::printStorePath(const StorePath & path) const +{ + auto s = storeDir + "/"; + s += (std::string_view) path.to_string(); + return s; +} + +PathSet Store::printStorePathSet(const StorePathSet & paths) const +{ + PathSet res; + for (auto & i : paths) res.insert(printStorePath(i)); + return res; +} + +StorePathSet cloneStorePathSet(const StorePathSet & paths) +{ + StorePathSet res; + for (auto & p : paths) + res.insert(p.clone()); + return res; +} + +StorePathSet storePathsToSet(const StorePaths & paths) +{ + StorePathSet res; + for (auto & p : paths) + res.insert(p.clone()); + return res; +} + +StorePathSet singleton(const StorePath & path) +{ + StorePathSet res; + res.insert(path.clone()); + return res; +} + +} diff --git a/src/libstore/path.hh b/src/libstore/path.hh new file mode 100644 index 000000000..273808f02 --- /dev/null +++ b/src/libstore/path.hh @@ -0,0 +1,81 @@ +#pragma once + +#include "rust-ffi.hh" + +namespace nix { + +/* See path.rs. */ +struct StorePath; + +extern "C" { + void ffi_StorePath_drop(void *); + bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b); + bool ffi_StorePath_eq(const StorePath & a, const StorePath & b); + unsigned char * ffi_StorePath_hash_data(const StorePath & p); +} + +struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> +{ + static StorePath make(std::string_view path, std::string_view storeDir); + + static StorePath make(unsigned char hash[20], std::string_view name); + + static StorePath fromBaseName(std::string_view baseName); + + rust::String to_string() const; + + bool operator < (const StorePath & other) const + { + return ffi_StorePath_less_than(*this, other); + } + + bool operator == (const StorePath & other) const + { + return ffi_StorePath_eq(*this, other); + } + + bool operator != (const StorePath & other) const + { + return !(*this == other); + } + + StorePath clone() const; + + /* Check whether a file name ends with the extension for + derivations. */ + bool isDerivation() const; + + std::string_view name() const; + + unsigned char * hashData() const + { + return ffi_StorePath_hash_data(*this); + } +}; + +typedef std::set<StorePath> StorePathSet; +typedef std::vector<StorePath> StorePaths; + +StorePathSet cloneStorePathSet(const StorePathSet & paths); +StorePathSet storePathsToSet(const StorePaths & paths); + +StorePathSet singleton(const StorePath & path); + +/* Size of the hash part of store paths, in base-32 characters. */ +const size_t storePathHashLen = 32; // i.e. 160 bits + +/* Extension of derivations in the Nix store. */ +const std::string drvExtension = ".drv"; + +} + +namespace std { + +template<> struct hash<nix::StorePath> { + std::size_t operator()(const nix::StorePath & path) const noexcept + { + return * (std::size_t *) path.hashData(); + } +}; + +} diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 29f6f6c17..2bef51878 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -40,7 +40,7 @@ Generations findGenerations(Path profile, int & curGen) Generations gens; Path profileDir = dirOf(profile); - string profileName = baseNameOf(profile); + auto profileName = std::string(baseNameOf(profile)); for (auto & i : readDirectory(profileDir)) { int n; @@ -108,7 +108,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, Path outPath) user environment etc. we've just built. */ Path generation; makeName(profile, num + 1, generation); - store->addPermRoot(outPath, generation, false, true); + store->addPermRoot(store->parseStorePath(outPath), generation, false, true); return generation; } diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 605ca9815..102e15921 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -89,7 +89,7 @@ PathSet scanForReferences(const string & path, hash part of the file name. (This assumes that all references have the form `HASH-bla'). */ for (auto & i : refs) { - string baseName = baseNameOf(i); + auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); if (pos == string::npos) throw Error(format("bad reference '%1%'") % i); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 5233fb2c2..5a2d103b9 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -50,7 +50,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) auto storePath = store->toStorePath(path); std::string restPath = std::string(path, storePath.size()); - if (!store->isValidPath(storePath)) + if (!store->isValidPath(store->parseStorePath(storePath))) throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); auto i = nars.find(storePath); @@ -96,7 +96,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) } catch (SysError &) { } } - store->narFromPath(storePath, sink); + store->narFromPath(store->parseStorePath(storePath), sink); auto narAccessor = makeNarAccessor(sink.s); addToCache(storePath, *sink.s, narAccessor); return {narAccessor, restPath}; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 2a89b7c98..8c55da268 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -22,23 +22,22 @@ namespace nix { -Path readStorePath(Store & store, Source & from) +template<> StorePathSet readStorePaths(const Store & store, Source & from) { - Path path = readString(from); - store.assertStorePath(path); - return path; + StorePathSet paths; + for (auto & i : readStrings<Strings>(from)) + paths.insert(store.parseStorePath(i)); + return paths; } -template<class T> T readStorePaths(Store & store, Source & from) +void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths) { - T paths = readStrings<T>(from); - for (auto & i : paths) store.assertStorePath(i); - return paths; + out << paths.size(); + for (auto & i : paths) + out << store.printStorePath(i); } -template PathSet readStorePaths(Store & store, Source & from); -template Paths readStorePaths(Store & store, Source & from); /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -254,60 +253,62 @@ ConnectionHandle RemoteStore::getConnection() } -bool RemoteStore::isValidPathUncached(const Path & path) +bool RemoteStore::isValidPathUncached(const StorePath & path) { auto conn(getConnection()); - conn->to << wopIsValidPath << path; + conn->to << wopIsValidPath << printStorePath(path); conn.processStderr(); return readInt(conn->from); } -PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) +StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - PathSet res; + StorePathSet res; for (auto & i : paths) - if (isValidPath(i)) res.insert(i); + if (isValidPath(i)) res.insert(i.clone()); return res; } else { - conn->to << wopQueryValidPaths << paths; + conn->to << wopQueryValidPaths; + writeStorePaths(*this, conn->to, paths); conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } } -PathSet RemoteStore::queryAllValidPaths() +StorePathSet RemoteStore::queryAllValidPaths() { auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } -PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) +StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) { auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - PathSet res; + StorePathSet res; for (auto & i : paths) { - conn->to << wopHasSubstitutes << i; + conn->to << wopHasSubstitutes << printStorePath(i); conn.processStderr(); - if (readInt(conn->from)) res.insert(i); + if (readInt(conn->from)) res.insert(i.clone()); } return res; } else { - conn->to << wopQuerySubstitutablePaths << paths; + conn->to << wopQuerySubstitutablePaths; + writeStorePaths(*this, conn->to, paths); conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } } -void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, +void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) { if (paths.empty()) return; @@ -318,29 +319,31 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, for (auto & i : paths) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << i; + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; - info.deriver = readString(conn->from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(*this, conn->from); + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = readStorePaths<StorePathSet>(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos[i] = info; + infos.insert_or_assign(i.clone(), std::move(info)); } } else { - conn->to << wopQuerySubstitutablePathInfos << paths; + conn->to << wopQuerySubstitutablePathInfos; + writeStorePaths(*this, conn->to, paths); conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { - Path path = readStorePath(*this, conn->from); - SubstitutablePathInfo & info(infos[path]); - info.deriver = readString(conn->from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(*this, conn->from); + SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = readStorePaths<StorePathSet>(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -349,14 +352,14 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, } -void RemoteStore::queryPathInfoUncached(const Path & path, +void RemoteStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { std::shared_ptr<ValidPathInfo> info; { auto conn(getConnection()); - conn->to << wopQueryPathInfo << path; + conn->to << wopQueryPathInfo << printStorePath(path); try { conn.processStderr(); } catch (Error & e) { @@ -367,14 +370,13 @@ void RemoteStore::queryPathInfoUncached(const Path & path, } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { bool valid; conn->from >> valid; - if (!valid) throw InvalidPath(format("path '%s' is not valid") % path); + if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = std::make_shared<ValidPathInfo>(); - info->path = path; - info->deriver = readString(conn->from); - if (info->deriver != "") assertStorePath(info->deriver); + info = std::make_shared<ValidPathInfo>(path.clone()); + auto deriver = readString(conn->from); + if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths<PathSet>(*this, conn->from); + info->references = readStorePaths<StorePathSet>(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -387,52 +389,52 @@ void RemoteStore::queryPathInfoUncached(const Path & path, } -void RemoteStore::queryReferrers(const Path & path, - PathSet & referrers) +void RemoteStore::queryReferrers(const StorePath & path, + StorePathSet & referrers) { auto conn(getConnection()); - conn->to << wopQueryReferrers << path; + conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - PathSet referrers2 = readStorePaths<PathSet>(*this, conn->from); - referrers.insert(referrers2.begin(), referrers2.end()); + for (auto & i : readStorePaths<StorePathSet>(*this, conn->from)) + referrers.insert(i.clone()); } -PathSet RemoteStore::queryValidDerivers(const Path & path) +StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) { auto conn(getConnection()); - conn->to << wopQueryValidDerivers << path; + conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } -PathSet RemoteStore::queryDerivationOutputs(const Path & path) +StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) { auto conn(getConnection()); - conn->to << wopQueryDerivationOutputs << path; + conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return readStorePaths<PathSet>(*this, conn->from); + return readStorePaths<StorePathSet>(*this, conn->from); } -PathSet RemoteStore::queryDerivationOutputNames(const Path & path) +PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path) { auto conn(getConnection()); - conn->to << wopQueryDerivationOutputNames << path; + conn->to << wopQueryDerivationOutputNames << printStorePath(path); conn.processStderr(); return readStrings<PathSet>(conn->from); } -Path RemoteStore::queryPathFromHashPart(const string & hashPart) +std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); conn->to << wopQueryPathFromHashPart << hashPart; conn.processStderr(); Path path = readString(conn->from); - if (!path.empty()) assertStorePath(path); - return path; + if (path.empty()) return {}; + return parseStorePath(path); } @@ -450,9 +452,10 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, copyNAR(source, sink); sink << exportMagic - << info.path - << info.references - << info.deriver + << printStorePath(info.path); + writeStorePaths(*this, sink, info.references); + sink + << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature << 0 // == no path follows ; @@ -460,14 +463,17 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = readStorePaths<PathSet>(*this, conn->from); + auto importedPaths = readStorePaths<StorePathSet>(*this, conn->from); assert(importedPaths.size() <= 1); } else { conn->to << wopAddToStoreNar - << info.path << info.deriver << info.narHash.to_string(Base16, false) - << info.references << info.registrationTime << info.narSize + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(Base16, false); + writeStorePaths(*this, conn->to, info.references); + conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; @@ -477,7 +483,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } -Path RemoteStore::addToStore(const string & name, const Path & _srcPath, +StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); @@ -511,54 +517,52 @@ Path RemoteStore::addToStore(const string & name, const Path & _srcPath, throw; } - return readStorePath(*this, conn->from); + return parseStorePath(readString(conn->from)); } -Path RemoteStore::addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) +StorePath RemoteStore::addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); auto conn(getConnection()); - conn->to << wopAddTextToStore << name << s << references; + conn->to << wopAddTextToStore << name << s; + writeStorePaths(*this, conn->to, references); conn.processStderr(); - return readStorePath(*this, conn->from); + return parseStorePath(readString(conn->from)); } -void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) +void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) { auto conn(getConnection()); conn->to << wopBuildPaths; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { - conn->to << drvPaths; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) - conn->to << buildMode; - else - /* Old daemons did not take a 'buildMode' parameter, so we - need to validate it here on the client side. */ - if (buildMode != bmNormal) - throw Error("repairing or checking is not supported when building through the Nix daemon"); - } else { - /* For backwards compatibility with old daemons, strip output - identifiers. */ - PathSet drvPaths2; - for (auto & i : drvPaths) - drvPaths2.insert(string(i, 0, i.find('!'))); - conn->to << drvPaths2; - } + assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); + Strings ss; + for (auto & p : drvPaths) + ss.push_back(p.to_string(*this)); + conn->to << ss; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + conn->to << buildMode; + else + /* Old daemons did not take a 'buildMode' parameter, so we + need to validate it here on the client side. */ + if (buildMode != bmNormal) + throw Error("repairing or checking is not supported when building through the Nix daemon"); conn.processStderr(); readInt(conn->from); } -BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, +BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) { auto conn(getConnection()); - conn->to << wopBuildDerivation << drvPath << drv << buildMode; + conn->to << wopBuildDerivation << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + conn->to << buildMode; conn.processStderr(); BuildResult res; unsigned int status; @@ -568,19 +572,19 @@ BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDeriva } -void RemoteStore::ensurePath(const Path & path) +void RemoteStore::ensurePath(const StorePath & path) { auto conn(getConnection()); - conn->to << wopEnsurePath << path; + conn->to << wopEnsurePath << printStorePath(path); conn.processStderr(); readInt(conn->from); } -void RemoteStore::addTempRoot(const Path & path) +void RemoteStore::addTempRoot(const StorePath & path) { auto conn(getConnection()); - conn->to << wopAddTempRoot << path; + conn->to << wopAddTempRoot << printStorePath(path); conn.processStderr(); readInt(conn->from); } @@ -613,8 +617,8 @@ Roots RemoteStore::findRoots(bool censor) Roots result; while (count--) { Path link = readString(conn->from); - Path target = readStorePath(*this, conn->from); - result[target].emplace(link); + auto target = parseStorePath(readString(conn->from)); + result[std::move(target)].emplace(link); } return result; } @@ -625,7 +629,9 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) auto conn(getConnection()); conn->to - << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness + << wopCollectGarbage << options.action; + writeStorePaths(*this, conn->to, options.pathsToDelete); + conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ << 0 << 0 << 0; @@ -661,17 +667,17 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) } -void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) +void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { auto conn(getConnection()); - conn->to << wopAddSignatures << storePath << sigs; + conn->to << wopAddSignatures << printStorePath(storePath) << sigs; conn.processStderr(); readInt(conn->from); } -void RemoteStore::queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, +void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize) { { @@ -680,11 +686,15 @@ void RemoteStore::queryMissing(const PathSet & targets, // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; - conn->to << wopQueryMissing << targets; + conn->to << wopQueryMissing; + Strings ss; + for (auto & p : targets) + ss.push_back(p.to_string(*this)); + conn->to << ss; conn.processStderr(); - willBuild = readStorePaths<PathSet>(*this, conn->from); - willSubstitute = readStorePaths<PathSet>(*this, conn->from); - unknown = readStorePaths<PathSet>(*this, conn->from); + willBuild = readStorePaths<StorePathSet>(*this, conn->from); + willSubstitute = readStorePaths<StorePathSet>(*this, conn->from); + unknown = readStorePaths<StorePathSet>(*this, conn->from); conn->from >> downloadSize >> narSize; return; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 728df8b00..f301a97d8 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -35,50 +35,50 @@ public: /* Implementations of abstract store API methods. */ - bool isValidPathUncached(const Path & path) override; + bool isValidPathUncached(const StorePath & path) override; - PathSet queryValidPaths(const PathSet & paths, + StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute) override; - PathSet queryAllValidPaths() override; + StorePathSet queryAllValidPaths() override; - void queryPathInfoUncached(const Path & path, + void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override; - void queryReferrers(const Path & path, PathSet & referrers) override; + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; - PathSet queryValidDerivers(const Path & path) override; + StorePathSet queryValidDerivers(const StorePath & path) override; - PathSet queryDerivationOutputs(const Path & path) override; + StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const Path & path) override; + StringSet queryDerivationOutputNames(const StorePath & path) override; - Path queryPathFromHashPart(const string & hashPart) override; + std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; - PathSet querySubstitutablePaths(const PathSet & paths) override; + StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; - void querySubstitutablePathInfos(const PathSet & paths, + void querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor) override; - Path addToStore(const string & name, const Path & srcPath, + StorePath addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; - Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair) override; + StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair) override; - void buildPaths(const PathSet & paths, BuildMode buildMode) override; + void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override; - BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; - void ensurePath(const Path & path) override; + void ensurePath(const StorePath & path) override; - void addTempRoot(const Path & path) override; + void addTempRoot(const StorePath & path) override; void addIndirectRoot(const Path & path) override; @@ -92,10 +92,10 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; - void addSignatures(const Path & storePath, const StringSet & sigs) override; + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; - void queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + void queryMissing(const std::vector<StorePathWithOutputs> & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize) override; void connect() override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index cd547a964..58966a5b7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -222,7 +222,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore fetches the .narinfo file, rather than first checking for its existence via a HEAD request. Since .narinfos are small, doing a GET is unlikely to be slower than HEAD. */ - bool isValidPathUncached(const Path & storePath) override + bool isValidPathUncached(const StorePath & storePath) override { try { queryPathInfo(storePath); @@ -382,9 +382,9 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); } - PathSet queryAllValidPaths() override + StorePathSet queryAllValidPaths() override { - PathSet paths; + StorePathSet paths; std::string marker; do { @@ -405,7 +405,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore for (auto object : contents) { auto & key = object.GetKey(); if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; - paths.insert(storeDir + "/" + key.substr(0, key.size() - 8)); + paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-unknown")); } marker = res.GetNextMarker(); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index b9d0c41ea..42ee06501 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -38,7 +38,7 @@ public: bool sameMachine() override { return false; } - void narFromPath(const Path & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink) override; ref<FSAccessor> getFSAccessor() override; @@ -66,10 +66,10 @@ private: }; }; -void SSHStore::narFromPath(const Path & path, Sink & sink) +void SSHStore::narFromPath(const StorePath & path, Sink & sink) { auto conn(connections->get()); - conn->to << wopNarFromPath << path; + conn->to << wopNarFromPath << printStorePath(path); conn->processStderr(); copyNAR(conn->from, sink); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f134f7967..0411db517 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -27,13 +27,6 @@ bool Store::isStorePath(const Path & path) const } -void Store::assertStorePath(const Path & path) const -{ - if (!isStorePath(path)) - throw Error(format("path '%1%' is not in the Nix store") % path); -} - - Path Store::toStorePath(const Path & path) const { if (!isInStore(path)) @@ -60,17 +53,9 @@ Path Store::followLinksToStore(const Path & _path) const } -Path Store::followLinksToStorePath(const Path & path) const +StorePath Store::followLinksToStorePath(const Path & path) const { - return toStorePath(followLinksToStore(path)); -} - - -string storePathToName(const Path & path) -{ - auto base = baseNameOf(path); - assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-')); - return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1); + return parseStorePath(toStorePath(followLinksToStore(path))); } @@ -82,41 +67,6 @@ string storePathToHash(const Path & path) } -void checkStoreName(const string & name) -{ - string validChars = "+-._?="; - - auto baseError = format("The path name '%2%' is invalid: %3%. " - "Path names are alphanumeric and can include the symbols %1% " - "and must not begin with a period. " - "Note: If '%2%' is a source file and you cannot rename it on " - "disk, 'builtins.path { name = ... }' can be used to give it an " - "alternative name.") % validChars % name; - - if (name.empty()) - throw Error(baseError % "it is an empty string"); - - /* Disallow names starting with a dot for possible security - reasons (e.g., "." and ".."). */ - if (name[0] == '.') - throw Error(baseError % "it is illegal to start the name with a period"); - - /* Disallow names longer than 211 characters. ext4’s max is 256, - but we need extra space for the hash and .chroot extensions. */ - if (name.length() > 211) - throw Error(baseError % "name must be less than 212 characters"); - - for (auto & i : name) - if (!((i >= 'A' && i <= 'Z') || - (i >= 'a' && i <= 'z') || - (i >= '0' && i <= '9') || - validChars.find(i) != string::npos)) - { - throw Error(baseError % (format("the '%1%' character is invalid") % i)); - } -} - - /* Store paths have the following form: <store>/<h>-<name> @@ -188,43 +138,48 @@ void checkStoreName(const string & name) */ -Path Store::makeStorePath(const string & type, - const Hash & hash, const string & name) const +StorePath Store::makeStorePath(const string & type, + const Hash & hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name; - - checkStoreName(name); - - return storeDir + "/" - + compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) - + "-" + name; + string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name); + auto h = compressHash(hashString(htSHA256, s), 20); + return StorePath::make(h.hash, name); } -Path Store::makeOutputPath(const string & id, - const Hash & hash, const string & name) const +StorePath Store::makeOutputPath(const string & id, + const Hash & hash, std::string_view name) const { return makeStorePath("output:" + id, hash, - name + (id == "out" ? "" : "-" + id)); + std::string(name) + (id == "out" ? "" : "-" + id)); } -static std::string makeType(string && type, const PathSet & references) +static std::string makeType( + const Store & store, + string && type, + const StorePathSet & references, + bool hasSelfReference = false) { for (auto & i : references) { type += ":"; - type += i; + type += store.printStorePath(i); } + if (hasSelfReference) type += ":self"; return std::move(type); } -Path Store::makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name, const PathSet & references) const +StorePath Store::makeFixedOutputPath( + bool recursive, + const Hash & hash, + std::string_view name, + const StorePathSet & references, + bool hasSelfReference) const { if (hash.type == htSHA256 && recursive) { - return makeStorePath(makeType("source", references), hash, name); + return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); return makeStorePath("output:out", hashString(htSHA256, @@ -234,28 +189,27 @@ Path Store::makeFixedOutputPath(bool recursive, } -Path Store::makeTextPath(const string & name, const Hash & hash, - const PathSet & references) const +StorePath Store::makeTextPath(std::string_view name, const Hash & hash, + const StorePathSet & references) const { assert(hash.type == htSHA256); /* Stuff the references (if any) into the type. This is a bit hacky, but we can't put them in `s' since that would be ambiguous. */ - return makeStorePath(makeType("text", references), hash, name); + return makeStorePath(makeType(*this, "text", references), hash, name); } -std::pair<Path, Hash> Store::computeStorePathForPath(const string & name, +std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const { Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - Path dstPath = makeFixedOutputPath(recursive, h, name); - return std::pair<Path, Hash>(dstPath, h); + return std::make_pair(makeFixedOutputPath(recursive, h, name), h); } -Path Store::computeStorePathForText(const string & name, const string & s, - const PathSet & references) const +StorePath Store::computeStorePathForText(const string & name, const string & s, + const StorePathSet & references) const { return makeTextPath(name, hashString(htSHA256, s), references); } @@ -274,11 +228,9 @@ std::string Store::getUri() } -bool Store::isValidPath(const Path & storePath) +bool Store::isValidPath(const StorePath & storePath) { - assertStorePath(storePath); - - auto hashPart = storePathToHash(storePath); + auto hashPart = storePathToHash(printStorePath(storePath)); { auto state_(state.lock()); @@ -312,7 +264,7 @@ bool Store::isValidPath(const Path & storePath) /* Default implementation for stores that only implement queryPathInfoUncached(). */ -bool Store::isValidPathUncached(const Path & path) +bool Store::isValidPathUncached(const StorePath & path) { try { queryPathInfo(path); @@ -323,7 +275,7 @@ bool Store::isValidPathUncached(const Path & path) } -ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) +ref<const ValidPathInfo> Store::queryPathInfo(const StorePath & storePath) { std::promise<ref<const ValidPathInfo>> promise; @@ -340,22 +292,20 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath) } -void Store::queryPathInfo(const Path & storePath, +void Store::queryPathInfo(const StorePath & storePath, Callback<ref<const ValidPathInfo>> callback) noexcept { std::string hashPart; try { - assertStorePath(storePath); - - hashPart = storePathToHash(storePath); + hashPart = storePathToHash(printStorePath(storePath)); { auto res = state.lock()->pathInfoCache.get(hashPart); if (res) { stats.narInfoReadAverted++; if (!*res) - throw InvalidPath(format("path '%s' is not valid") % storePath); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); return callback(ref<const ValidPathInfo>(*res)); } } @@ -369,8 +319,8 @@ void Store::queryPathInfo(const Path & storePath, state_->pathInfoCache.upsert(hashPart, res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); if (res.first == NarInfoDiskCache::oInvalid || - (res.second->path != storePath && storePathToName(storePath) != "")) - throw InvalidPath(format("path '%s' is not valid") % storePath); + res.second->path != storePath) + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } return callback(ref<const ValidPathInfo>(res.second)); } @@ -381,7 +331,7 @@ void Store::queryPathInfo(const Path & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { + {[this, storePath{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -394,9 +344,7 @@ void Store::queryPathInfo(const Path & storePath, state_->pathInfoCache.upsert(hashPart, info); } - if (!info - || (info->path != storePath && storePathToName(storePath) != "")) - { + if (!info || info->path != parseStorePath(storePath)) { stats.narInfoMissing++; throw InvalidPath("path '%s' is not valid", storePath); } @@ -407,27 +355,27 @@ void Store::queryPathInfo(const Path & storePath, } -PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) +StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { struct State { size_t left; - PathSet valid; + StorePathSet valid; std::exception_ptr exc; }; - Sync<State> state_(State{paths.size(), PathSet()}); + Sync<State> state_(State{paths.size(), StorePathSet()}); std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path ) { + auto doQuery = [&](const Path & path) { checkInterrupt(); - queryPathInfo(path, {[path, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(path); + state->valid.insert(parseStorePath(path)); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -439,7 +387,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, path)); + pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME pool.process(); @@ -447,7 +395,7 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti auto state(state_.lock()); if (!state->left) { if (state->exc) std::rethrow_exception(state->exc); - return state->valid; + return std::move(state->valid); } state.wait(wakeup); } @@ -457,13 +405,13 @@ PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubsti /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string Store::makeValidityRegistration(const PathSet & paths, +string Store::makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash) { string s = ""; for (auto & i : paths) { - s += i + "\n"; + s += printStorePath(i) + "\n"; auto info = queryPathInfo(i); @@ -472,31 +420,30 @@ string Store::makeValidityRegistration(const PathSet & paths, s += (format("%1%\n") % info->narSize).str(); } - Path deriver = showDerivers ? info->deriver : ""; + auto deriver = showDerivers && info->deriver ? printStorePath(*info->deriver) : ""; s += deriver + "\n"; s += (format("%1%\n") % info->references.size()).str(); for (auto & j : info->references) - s += j + "\n"; + s += printStorePath(j) + "\n"; } return s; } -void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, +void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid) { auto jsonList = jsonOut.list(); - for (auto storePath : storePaths) { + for (auto & storePath : storePaths) { auto jsonPath = jsonList.object(); - jsonPath.attr("path", storePath); + jsonPath.attr("path", printStorePath(storePath)); try { auto info = queryPathInfo(storePath); - storePath = info->path; jsonPath .attr("narHash", info->narHash.to_string()) @@ -505,7 +452,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths { auto jsonRefs = jsonPath.list("references"); for (auto & ref : info->references) - jsonRefs.elem(ref); + jsonRefs.elem(printStorePath(ref)); } if (info->ca != "") @@ -514,14 +461,14 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths std::pair<uint64_t, uint64_t> closureSizes; if (showClosureSize) { - closureSizes = getClosureSize(storePath); + closureSizes = getClosureSize(info->path); jsonPath.attr("closureSize", closureSizes.first); } if (includeImpureInfo) { - if (info->deriver != "") - jsonPath.attr("deriver", info->deriver); + if (info->deriver) + jsonPath.attr("deriver", printStorePath(*info->deriver)); if (info->registrationTime) jsonPath.attr("registrationTime", info->registrationTime); @@ -557,10 +504,10 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths } -std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path & storePath) +std::pair<uint64_t, uint64_t> Store::getClosureSize(const StorePath & storePath) { uint64_t totalNarSize = 0, totalDownloadSize = 0; - PathSet closure; + StorePathSet closure; computeFSClosure(storePath, closure, false, false); for (auto & p : closure) { auto info = queryPathInfo(p); @@ -584,30 +531,34 @@ const Store::Stats & Store::getStats() } -void Store::buildPaths(const PathSet & paths, BuildMode buildMode) +void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) { - for (auto & path : paths) - if (isDerivation(path)) + StorePathSet paths2; + + for (auto & path : paths) { + if (path.path.isDerivation()) unsupported("buildPaths"); + paths2.insert(path.path.clone()); + } - if (queryValidPaths(paths).size() != paths.size()) + if (queryValidPaths(paths2).size() != paths2.size()) unsupported("buildPaths"); } void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) + const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs) { auto srcUri = srcStore->getUri(); auto dstUri = dstStore->getUri(); Activity act(*logger, lvlInfo, actCopyPath, srcUri == "local" || srcUri == "daemon" - ? fmt("copying path '%s' to '%s'", storePath, dstUri) + ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri) : dstUri == "local" || dstUri == "daemon" - ? fmt("copying path '%s' from '%s'", storePath, srcUri) - : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), - {storePath, srcUri, dstUri}); + ? fmt("copying path '%s' from '%s'", srcStore->printStorePath(storePath), srcUri) + : fmt("copying path '%s' from '%s' to '%s'", srcStore->printStorePath(storePath), srcUri, dstUri), + {srcStore->printStorePath(storePath), srcUri, dstUri}); PushActivity pact(act.id); auto info = srcStore->queryPathInfo(storePath); @@ -640,23 +591,23 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, total += len; act.progress(total, info->narSize); }); - srcStore->narFromPath({storePath}, wrapperSink); + srcStore->narFromPath(storePath, wrapperSink); }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore->printStorePath(storePath), srcStore->getUri()); }); dstStore->addToStore(*info, *source, repair, checkSigs); } -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { - PathSet valid = dstStore->queryValidPaths(storePaths, substitute); + auto valid = dstStore->queryValidPaths(storePaths, substitute); PathSet missing; for (auto & path : storePaths) - if (!valid.count(path)) missing.insert(path); + if (!valid.count(path)) missing.insert(srcStore->printStorePath(path)); if (missing.empty()) return; @@ -677,23 +628,25 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa PathSet(missing.begin(), missing.end()), [&](const Path & storePath) { - if (dstStore->isValidPath(storePath)) { + if (dstStore->isValidPath(dstStore->parseStorePath(storePath))) { nrDone++; showProgress(); return PathSet(); } - auto info = srcStore->queryPathInfo(storePath); + auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); bytesExpected += info->narSize; act.setExpected(actCopyPath, bytesExpected); - return info->references; + return srcStore->printStorePathSet(info->references); }, - [&](const Path & storePath) { + [&](const Path & storePathS) { checkInterrupt(); + auto storePath = dstStore->parseStorePath(storePathS); + if (!dstStore->isValidPath(storePath)) { MaintainCount<decltype(nrRunning)> mc(nrRunning); showProgress(); @@ -703,7 +656,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what()); + logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); showProgress(); return; } @@ -716,20 +669,36 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, + const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { - PathSet closure; - srcStore->computeFSClosure({storePaths}, closure); + StorePathSet closure; + srcStore->computeFSClosure(storePaths, closure); copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); } -ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) +ValidPathInfo::ValidPathInfo(const ValidPathInfo & other) + : path(other.path.clone()) + , deriver(other.deriver ? other.deriver->clone(): std::optional<StorePath>{}) + , narHash(other.narHash) + , references(cloneStorePathSet(other.references)) + , registrationTime(other.registrationTime) + , narSize(other.narSize) + , id(other.id) + , ultimate(other.ultimate) + , sigs(other.sigs) + , ca(other.ca) { - ValidPathInfo info; - getline(str, info.path); - if (str.eof()) { info.path = ""; return info; } +} + + +std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven) +{ + std::string path; + getline(str, path); + if (str.eof()) { return {}; } + ValidPathInfo info(store.parseStorePath(path)); if (hashGiven) { string s; getline(str, s); @@ -737,16 +706,29 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) getline(str, s); if (!string2Int(s, info.narSize)) throw Error("number expected"); } - getline(str, info.deriver); + std::string deriver; + getline(str, deriver); + if (deriver != "") info.deriver = store.parseStorePath(deriver); string s; int n; getline(str, s); if (!string2Int(s, n)) throw Error("number expected"); while (n--) { getline(str, s); - info.references.insert(s); + info.references.insert(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); - return info; + return std::optional<ValidPathInfo>(std::move(info)); +} + + +std::string Store::showPaths(const StorePathSet & paths) +{ + std::string s; + for (auto & i : paths) { + if (s.size() != 0) s += ", "; + s += "'" + printStorePath(i) + "'"; + } + return s; } @@ -756,34 +738,34 @@ string showPaths(const PathSet & paths) } -std::string ValidPathInfo::fingerprint() const +std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0 || !narHash) - throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known") - % path); + throw Error("cannot calculate fingerprint of path '%s' because its size/hash is not known", + store.printStorePath(path)); return - "1;" + path + ";" + "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", references); + + concatStringsSep(",", store.printStorePathSet(references)); } -void ValidPathInfo::sign(const SecretKey & secretKey) +void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) { - sigs.insert(secretKey.signDetached(fingerprint())); + sigs.insert(secretKey.signDetached(fingerprint(store))); } bool ValidPathInfo::isContentAddressed(const Store & store) const { auto warn = [&]() { - printError(format("warning: path '%s' claims to be content-addressed but isn't") % path); + printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); }; if (hasPrefix(ca, "text:")) { Hash hash(std::string(ca, 5)); - if (store.makeTextPath(storePathToName(path), hash, references) == path) + if (store.makeTextPath(path.name(), hash, references) == path) return true; else warn(); @@ -792,9 +774,13 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const else if (hasPrefix(ca, "fixed:")) { bool recursive = ca.compare(6, 2, "r:") == 0; Hash hash(std::string(ca, recursive ? 8 : 6)); - auto refs = references; - replaceInSet(refs, path, std::string("self")); - if (store.makeFixedOutputPath(recursive, hash, storePathToName(path), refs) == path) + auto refs = cloneStorePathSet(references); + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path) return true; else warn(); @@ -810,15 +796,15 @@ size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & pu size_t good = 0; for (auto & sig : sigs) - if (checkSignature(publicKeys, sig)) + if (checkSignature(store, publicKeys, sig)) good++; return good; } -bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const { - return verifyDetached(fingerprint(), sig, publicKeys); + return verifyDetached(fingerprint(store), sig, publicKeys); } @@ -826,7 +812,7 @@ Strings ValidPathInfo::shortRefs() const { Strings refs; for (auto & r : references) - refs.push_back(baseNameOf(r)); + refs.push_back(std::string(r.to_string())); return refs; } @@ -939,7 +925,7 @@ static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr<Store> { - switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { + switch (getStoreType(uri, get(params, "state").value_or(settings.nixStateDir))) { case tDaemon: return std::shared_ptr<Store>(std::make_shared<UDSRemoteStore>(params)); case tLocal: { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e2e720fc4..13ca3fef2 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,5 +1,6 @@ #pragma once +#include "path.hh" #include "hash.hh" #include "serialise.hh" #include "crypto.hh" @@ -43,14 +44,11 @@ enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; -/* Size of the hash part of store paths, in base-32 characters. */ -const size_t storePathHashLen = 32; // i.e. 160 bits - /* Magic header of exportPath() output (obsolete). */ const uint32_t exportMagic = 0x4558494e; -typedef std::unordered_map<Path, std::unordered_set<std::string>> Roots; +typedef std::unordered_map<StorePath, std::unordered_set<std::string>> Roots; struct GCOptions @@ -84,7 +82,7 @@ struct GCOptions bool ignoreLiveness{false}; /* For `gcDeleteSpecific', the paths to delete. */ - PathSet pathsToDelete; + StorePathSet pathsToDelete; /* Stop after at least `maxFreed' bytes have been freed. */ unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; @@ -105,21 +103,21 @@ struct GCResults struct SubstitutablePathInfo { - Path deriver; - PathSet references; + std::optional<StorePath> deriver; + StorePathSet references; unsigned long long downloadSize; /* 0 = unknown or inapplicable */ unsigned long long narSize; /* 0 = unknown */ }; -typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos; +typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; struct ValidPathInfo { - Path path; - Path deriver; + StorePath path; + std::optional<StorePath> deriver; Hash narHash; - PathSet references; + StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -144,7 +142,7 @@ struct ValidPathInfo Ideally, the content-addressability assertion would just be a Boolean, and the store path would be computed from - ‘storePathToName(path)’, ‘narHash’ and ‘references’. However, + the name component, ‘narHash’ and ‘references’. However, 1) we've accumulated several types of content-addressed paths over the years; and 2) fixed-output derivations support multiple hash algorithms and serialisation methods (flat file @@ -172,9 +170,9 @@ struct ValidPathInfo the NAR, and the sorted references. The size field is strictly speaking superfluous, but might prevent endless/excessive data attacks. */ - std::string fingerprint() const; + std::string fingerprint(const Store & store) const; - void sign(const SecretKey & secretKey); + void sign(const Store & store, const SecretKey & secretKey); /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -187,10 +185,13 @@ struct ValidPathInfo size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; /* Verify a single signature. */ - bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; + bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; Strings shortRefs() const; + ValidPathInfo(StorePath && path) : path(std::move(path)) { } + explicit ValidPathInfo(const ValidPathInfo & other); + virtual ~ValidPathInfo() { } }; @@ -241,6 +242,23 @@ struct BuildResult }; +struct StorePathWithOutputs +{ + StorePath path; + std::set<std::string> outputs; + + StorePathWithOutputs(const StorePath & path, const std::set<std::string> & outputs = {}) + : path(path.clone()), outputs(outputs) + { } + + StorePathWithOutputs(const StorePathWithOutputs & other) + : path(other.path.clone()), outputs(other.outputs) + { } + + std::string to_string(const Store & store) const; +}; + + class Store : public std::enable_shared_from_this<Store>, public Config { public: @@ -259,6 +277,7 @@ protected: struct State { + // FIXME: fix key LRUCache<std::string, std::shared_ptr<const ValidPathInfo>> pathInfoCache; }; @@ -274,6 +293,24 @@ public: virtual std::string getUri() = 0; + StorePath parseStorePath(std::string_view path) const; + + std::string printStorePath(const StorePath & path) const; + + // FIXME: remove + StorePathSet parseStorePathSet(const PathSet & paths) const; + + PathSet printStorePathSet(const StorePathSet & path) const; + + /* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path + and the outputs. */ + StorePathWithOutputs parseDrvPathWithOutputs(const string & s); + + /* Display a set of paths in human-readable form (i.e., between quotes + and separated by commas). */ + std::string showPaths(const StorePathSet & paths); + /* Return true if ‘path’ is in the Nix store (but not the Nix store itself). */ bool isInStore(const Path & path) const; @@ -282,9 +319,6 @@ public: the Nix store. */ bool isStorePath(const Path & path) const; - /* Throw an exception if ‘path’ is not a store path. */ - void assertStorePath(const Path & path) const; - /* Chop off the parts after the top-level store name, e.g., /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ Path toStorePath(const Path & path) const; @@ -294,26 +328,27 @@ public: /* Same as followLinksToStore(), but apply toStorePath() to the result. */ - Path followLinksToStorePath(const Path & path) const; + StorePath followLinksToStorePath(const Path & path) const; /* Constructs a unique store path name. */ - Path makeStorePath(const string & type, - const Hash & hash, const string & name) const; + StorePath makeStorePath(const string & type, + const Hash & hash, std::string_view name) const; - Path makeOutputPath(const string & id, - const Hash & hash, const string & name) const; + StorePath makeOutputPath(const string & id, + const Hash & hash, std::string_view name) const; - Path makeFixedOutputPath(bool recursive, - const Hash & hash, const string & name, - const PathSet & references = {}) const; + StorePath makeFixedOutputPath(bool recursive, + const Hash & hash, std::string_view name, + const StorePathSet & references = {}, + bool hasSelfReference = false) const; - Path makeTextPath(const string & name, const Hash & hash, - const PathSet & references) const; + StorePath makeTextPath(std::string_view name, const Hash & hash, + const StorePathSet & references) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store path and the cryptographic hash of the contents of srcPath. */ - std::pair<Path, Hash> computeStorePathForPath(const string & name, + std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; @@ -331,21 +366,21 @@ public: simply yield a different store path, so other users wouldn't be affected), but it has some backwards compatibility issues (the hashing scheme changes), so I'm not doing that for now. */ - Path computeStorePathForText(const string & name, const string & s, - const PathSet & references) const; + StorePath computeStorePathForText(const string & name, const string & s, + const StorePathSet & references) const; /* Check whether a path is valid. */ - bool isValidPath(const Path & path); + bool isValidPath(const StorePath & path); protected: - virtual bool isValidPathUncached(const Path & path); + virtual bool isValidPathUncached(const StorePath & path); public: /* Query which of the given paths is valid. Optionally, try to substitute missing paths. */ - virtual PathSet queryValidPaths(const PathSet & paths, + virtual StorePathSet queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute = NoSubstitute); /* Query the set of all valid paths. Note that for some store @@ -353,54 +388,54 @@ public: (i.e. you'll get /nix/store/<hash> rather than /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the full store path. */ - virtual PathSet queryAllValidPaths() + virtual StorePathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } /* Query information about a valid path. It is permitted to omit the name part of the store path. */ - ref<const ValidPathInfo> queryPathInfo(const Path & path); + ref<const ValidPathInfo> queryPathInfo(const StorePath & path); /* Asynchronous version of queryPathInfo(). */ - void queryPathInfo(const Path & path, + void queryPathInfo(const StorePath & path, Callback<ref<const ValidPathInfo>> callback) noexcept; protected: - virtual void queryPathInfoUncached(const Path & path, + virtual void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0; public: /* Queries the set of incoming FS references for a store path. The result is not cleared. */ - virtual void queryReferrers(const Path & path, PathSet & referrers) + virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) { unsupported("queryReferrers"); } /* Return all currently valid derivations that have `path' as an output. (Note that the result of `queryDeriver()' is the derivation that was actually used to produce `path', which may not exist anymore.) */ - virtual PathSet queryValidDerivers(const Path & path) { return {}; }; + virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /* Query the outputs of the derivation denoted by `path'. */ - virtual PathSet queryDerivationOutputs(const Path & path) + virtual StorePathSet queryDerivationOutputs(const StorePath & path) { unsupported("queryDerivationOutputs"); } /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const Path & path) + virtual StringSet queryDerivationOutputNames(const StorePath & path) { unsupported("queryDerivationOutputNames"); } /* Query the full store path given the hash part of a valid store - path, or "" if the path doesn't exist. */ - virtual Path queryPathFromHashPart(const string & hashPart) = 0; + path, or empty if the path doesn't exist. */ + virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0; /* Query which of the given paths have substitutes. */ - virtual PathSet querySubstitutablePaths(const PathSet & paths) { return {}; }; + virtual StorePathSet querySubstitutablePaths(const StorePathSet & paths) { return {}; }; /* Query substitute info (i.e. references, derivers and download sizes) of a set of paths. If a path does not have substitute info, it's omitted from the resulting ‘infos’ map. */ - virtual void querySubstitutablePathInfos(const PathSet & paths, + virtual void querySubstitutablePathInfos(const StorePathSet & paths, SubstitutablePathInfos & infos) { return; }; virtual bool wantMassQuery() { return false; } @@ -419,12 +454,12 @@ public: validity the resulting path. The resulting path is returned. The function object `filter' can be used to exclude files (see libutil/archive.hh). */ - virtual Path addToStore(const string & name, const Path & srcPath, + virtual StorePath addToStore(const string & name, const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; // FIXME: remove? - virtual Path addToStoreFromDump(const string & dump, const string & name, + virtual StorePath addToStoreFromDump(const string & dump, const string & name, bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); @@ -432,11 +467,11 @@ public: /* Like addToStore, but the contents written to the output path is a regular file containing the given string. */ - virtual Path addTextToStore(const string & name, const string & s, - const PathSet & references, RepairFlag repair = NoRepair) = 0; + virtual StorePath addTextToStore(const string & name, const string & s, + const StorePathSet & references, RepairFlag repair = NoRepair) = 0; /* Write a NAR dump of a store path. */ - virtual void narFromPath(const Path & path, Sink & sink) = 0; + virtual void narFromPath(const StorePath & path, Sink & sink) = 0; /* For each path, if it's a derivation, build it. Building a derivation means ensuring that the output paths are valid. If @@ -446,22 +481,24 @@ public: output paths can be created by running the builder, after recursively building any sub-derivations. For inputs that are not derivations, substitute them. */ - virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal); + virtual void buildPaths( + const std::vector<StorePathWithOutputs> & paths, + BuildMode buildMode = bmNormal); /* Build a single non-materialized derivation (i.e. not from an on-disk .drv file). Note that ‘drvPath’ is only used for informational purposes. */ - virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal) = 0; /* Ensure that a path is valid. If it is not currently valid, it may be made valid by running a substitute (if defined for the path). */ - virtual void ensurePath(const Path & path) = 0; + virtual void ensurePath(const StorePath & path) = 0; /* Add a store path as a temporary root of the garbage collector. The root disappears as soon as we exit. */ - virtual void addTempRoot(const Path & path) + virtual void addTempRoot(const StorePath & path) { unsupported("addTempRoot"); } /* Add an indirect root, which is merely a symlink to `path' from @@ -507,7 +544,7 @@ public: /* Return a string representing information about the path that can be loaded into the database using `nix-store --load-db' or `nix-store --register-validity'. */ - string makeValidityRegistration(const PathSet & paths, + string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); /* Write a JSON representation of store path metadata, such as the @@ -515,14 +552,14 @@ public: variable elements such as the registration time are included. If ‘showClosureSize’ is true, the closure size of each path is included. */ - void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, + void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid = DisallowInvalid); /* Return the size of the closure of the specified path, that is, the sum of the size of the NAR serialisation of each path in the closure. */ - std::pair<uint64_t, uint64_t> getClosureSize(const Path & storePath); + std::pair<uint64_t, uint64_t> getClosureSize(const StorePath & storePath); /* Optimise the disk space usage of the Nix store by hard-linking files with the same contents. */ @@ -538,14 +575,14 @@ public: /* Add signatures to the specified store path. The signatures are not verified. */ - virtual void addSignatures(const Path & storePath, const StringSet & sigs) + virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) { unsupported("addSignatures"); } /* Utility functions. */ /* Read a derivation, after ensuring its existence through ensurePath(). */ - Derivation derivationFromPath(const Path & drvPath); + Derivation derivationFromPath(const StorePath & drvPath); /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly @@ -554,36 +591,36 @@ public: `storePath' is returned; that is, the closures under the `referrers' relation instead of the `references' relation is returned. */ - virtual void computeFSClosure(const PathSet & paths, - PathSet & out, bool flipDirection = false, + virtual void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); - void computeFSClosure(const Path & path, - PathSet & out, bool flipDirection = false, + void computeFSClosure(const StorePath & path, + StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false); /* Given a set of paths that are to be built, return the set of derivations that will be built, and the set of output paths that will be substituted. */ - virtual void queryMissing(const PathSet & targets, - PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets, + StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, unsigned long long & downloadSize, unsigned long long & narSize); /* Sort a set of paths topologically under the references relation. If p refers to q, then p precedes q in this list. */ - Paths topoSortPaths(const PathSet & paths); + StorePaths topoSortPaths(const StorePathSet & paths); /* Export multiple paths in the format expected by ‘nix-store --import’. */ - void exportPaths(const Paths & paths, Sink & sink); + void exportPaths(const StorePathSet & paths, Sink & sink); - void exportPath(const Path & path, Sink & sink); + void exportPath(const StorePath & path, Sink & sink); /* Import a sequence of NAR dumps created by exportPaths() into the Nix store. Optionally, the contents of the NARs are preloaded into the specified FS accessor to speed up subsequent access. */ - Paths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, + StorePaths importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs = CheckSigs); struct Stats @@ -607,7 +644,7 @@ public: /* Return the build log of the specified store path, if available, or null otherwise. */ - virtual std::shared_ptr<std::string> getBuildLog(const Path & path) + virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path) { return nullptr; } /* Hack to allow long-running processes like hydra-queue-runner to @@ -673,11 +710,11 @@ public: LocalFSStore(const Params & params); - void narFromPath(const Path & path, Sink & sink) override; + void narFromPath(const StorePath & path, Sink & sink) override; ref<FSAccessor> getFSAccessor() override; /* Register a permanent GC root. */ - Path addPermRoot(const Path & storePath, + Path addPermRoot(const StorePath & storePath, const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); virtual Path getRealStoreDir() { return storeDir; } @@ -688,25 +725,17 @@ public: return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); } - std::shared_ptr<std::string> getBuildLog(const Path & path) override; + std::shared_ptr<std::string> getBuildLog(const StorePath & path) override; }; -/* Extract the name part of the given store path. */ -string storePathToName(const Path & path); - /* Extract the hash part of the given store path. */ string storePathToHash(const Path & path); -/* Check whether ‘name’ is a valid store path name part, i.e. contains - only the characters [a-zA-Z0-9\+\-\.\_\?\=] and doesn't start with - a dot. */ -void checkStoreName(const string & name); - /* Copy a path from one store to another. */ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, - const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + const StorePath & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); /* Copy store paths from one store to another. The paths may be copied @@ -714,7 +743,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, (i.e. if A is a reference of B, then A is copied before B), but the set of store paths is not automatically closed; use copyClosure() for that. */ -void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePaths, +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -722,7 +751,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const PathSet & storePa /* Copy the closure of the specified paths from one store to another. */ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, - const PathSet & storePaths, + const StorePathSet & storePaths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -805,7 +834,9 @@ struct RegisterStoreImplementation string showPaths(const PathSet & paths); -ValidPathInfo decodeValidPathInfo(std::istream & str, +std::optional<ValidPathInfo> decodeValidPathInfo( + const Store & store, + std::istream & str, bool hashGiven = false); diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 6762b609d..857d54d99 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -65,8 +65,9 @@ typedef enum { class Store; struct Source; -Path readStorePath(Store & store, Source & from); -template<class T> T readStorePaths(Store & store, Source & from); +template<class T> T readStorePaths(const Store & store, Source & from); + +void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); } diff --git a/src/libutil/rust-ffi.cc b/src/libutil/rust-ffi.cc index 931d29542..accc5e22b 100644 --- a/src/libutil/rust-ffi.cc +++ b/src/libutil/rust-ffi.cc @@ -1,12 +1,18 @@ #include "logging.hh" #include "rust-ffi.hh" -namespace nix { - extern "C" std::exception_ptr * make_error(rust::StringSlice s) { // FIXME: leak - return new std::exception_ptr(std::make_exception_ptr(Error(std::string(s.ptr, s.size)))); + return new std::exception_ptr(std::make_exception_ptr(nix::Error(std::string(s.ptr, s.size)))); +} + +namespace rust { + +std::ostream & operator << (std::ostream & str, const String & s) +{ + str << (std::string_view) s; + return str; } } diff --git a/src/libutil/rust-ffi.hh b/src/libutil/rust-ffi.hh index 663758bfc..a77f83ac5 100644 --- a/src/libutil/rust-ffi.hh +++ b/src/libutil/rust-ffi.hh @@ -1,16 +1,91 @@ +#pragma once + #include "serialise.hh" +#include <string_view> +#include <cstring> +#include <array> + namespace rust { -// Depending on the internal representation of Rust slices is slightly -// evil... +typedef void (*DropFun)(void *); + +/* A Rust value of N bytes. It can be moved but not copied. When it + goes out of scope, the C++ destructor will run the drop + function. */ +template<std::size_t N, DropFun drop> +struct Value +{ +protected: + + std::array<char, N> raw; + + ~Value() + { + if (!isEvacuated()) { + drop(this); + evacuate(); + } + } + + // Must not be called directly. + Value() + { } + + Value(Value && other) + : raw(other.raw) + { + other.evacuate(); + } + + void operator =(Value && other) + { + if (!isEvacuated()) + drop(this); + raw = other.raw; + other.evacuate(); + } + +private: + + /* FIXME: optimize these (ideally in such a way that the compiler + can elide most calls to evacuate() / isEvacuated(). */ + inline void evacuate() + { + for (auto & i : raw) i = 0; + } + + inline bool isEvacuated() + { + for (auto & i : raw) + if (i != 0) return false; + return true; + } +}; + +/* A Rust vector. */ +template<typename T, DropFun drop> +struct Vec : Value<3 * sizeof(void *), drop> +{ + inline size_t size() const + { + return ((const size_t *) &this->raw)[2]; + } + + const T * data() const + { + return ((const T * *) &this->raw)[0]; + } +}; + +/* A Rust slice. */ template<typename T> struct Slice { - T * ptr; + const T * ptr; size_t size; - Slice(T * ptr, size_t size) : ptr(ptr), size(size) + Slice(const T * ptr, size_t size) : ptr(ptr), size(size) { assert(ptr); } @@ -18,9 +93,44 @@ struct Slice struct StringSlice : Slice<char> { - StringSlice(const std::string & s): Slice((char *) s.data(), s.size()) {} + StringSlice(const std::string & s): Slice(s.data(), s.size()) {} + explicit StringSlice(std::string_view s): Slice(s.data(), s.size()) {} + StringSlice(const char * s): Slice(s, strlen(s)) {} + + operator std::string_view() const + { + return std::string_view(ptr, size); + } +}; + +/* A Rust string. */ +struct String; + +extern "C" { + void ffi_String_new(StringSlice s, String * out); + void ffi_String_drop(void * s); +} + +struct String : Vec<char, ffi_String_drop> +{ + String(std::string_view s) + { + ffi_String_new(StringSlice(s), this); + } + + String(const char * s) + : String({s, std::strlen(s)}) + { + } + + operator std::string_view() const + { + return std::string_view(data(), size()); + } }; +std::ostream & operator << (std::ostream & str, const String & s); + struct Source { size_t (*fun)(void * source_this, rust::Slice<uint8_t> data); @@ -33,7 +143,7 @@ struct Source // FIXME: how to propagate exceptions? static size_t sourceWrapper(void * _this, rust::Slice<uint8_t> data) { - auto n = ((nix::Source *) _this)->read(data.ptr, data.size); + auto n = ((nix::Source *) _this)->read((unsigned char *) data.ptr, data.size); return n; } }; @@ -49,11 +159,20 @@ struct Result std::exception_ptr * exc; }; + ~Result() + { + if (tag == 0) + data.~T(); + else if (tag == 1) + // FIXME: don't leak exc + ; + } + /* Rethrow the wrapped exception or return the wrapped value. */ T unwrap() { if (tag == 0) - return data; + return std::move(data); else if (tag == 1) std::rethrow_exception(*exc); else diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 2cc7793fd..262bc655f 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -17,7 +17,7 @@ void unpackTarfile(Source & source, const Path & destDir) void unpackTarfile(const Path & tarFile, const Path & destDir, std::optional<std::string> baseName) { - if (!baseName) baseName = baseNameOf(tarFile); + if (!baseName) baseName = std::string(baseNameOf(tarFile)); auto source = sinkToSource([&](Sink & sink) { // FIXME: look at first few bytes to determine compression type. diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1e5e4851e..085c1e695 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -189,22 +189,22 @@ Path dirOf(const Path & path) } -string baseNameOf(const Path & path) +std::string_view baseNameOf(std::string_view path) { if (path.empty()) return ""; - Path::size_type last = path.length() - 1; + auto last = path.size() - 1; if (path[last] == '/' && last > 0) last -= 1; - Path::size_type pos = path.rfind('/', last); + auto pos = path.rfind('/', last); if (pos == string::npos) pos = 0; else pos += 1; - return string(path, pos, last - pos + 1); + return path.substr(pos, last - pos + 1); } @@ -1307,9 +1307,10 @@ bool hasPrefix(const string & s, const string & prefix) } -bool hasSuffix(const string & s, const string & suffix) +bool hasSuffix(std::string_view s, std::string_view suffix) { - return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; + return s.size() >= suffix.size() + && s.substr(s.size() - suffix.size()) == suffix; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index c18d1b5a4..cc045016b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -62,7 +62,7 @@ Path dirOf(const Path & path); /* Return the base name of the given canonical path, i.e., everything following the final `/'. */ -string baseNameOf(const Path & path); +std::string_view baseNameOf(std::string_view path); /* Check whether 'path' is a descendant of 'dir'. */ bool isInDir(const Path & path, const Path & dir); @@ -431,7 +431,7 @@ bool hasPrefix(const string & s, const string & prefix); /* Return true iff `s' ends in `suffix'. */ -bool hasSuffix(const string & s, const string & suffix); +bool hasSuffix(std::string_view s, std::string_view suffix); /* Convert a string to lower case. */ @@ -474,10 +474,10 @@ string base64Decode(const string & s); /* Get a value for the specified key from an associate container, or a default value if the key doesn't exist. */ template <class T> -string get(const T & map, const string & key, const string & def = "") +std::optional<std::string> get(const T & map, const std::string & key) { auto i = map.find(key); - return i == map.end() ? def : i->second; + return i == map.end() ? std::optional<std::string>() : i->second; } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 91783edbf..99546876a 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -106,7 +106,7 @@ static void _main(int argc, char * * argv) // Heuristic to see if we're invoked as a shebang script, namely, // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". - if (runEnv && argc > 1 && !std::regex_search(baseNameOf(argv[1]), std::regex("nix-shell"))) { + if (runEnv && argc > 1 && !std::regex_search(std::string(baseNameOf(argv[1])), std::regex("nix-shell"))) { script = argv[1]; try { auto lines = tokenizeString<Strings>(readFile(script), "\n"); @@ -317,11 +317,11 @@ static void _main(int argc, char * * argv) state->printStats(); - auto buildPaths = [&](const PathSet & paths) { + auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths) { /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; + StorePathSet willBuild, willSubstitute, unknown; store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); @@ -337,9 +337,9 @@ static void _main(int argc, char * * argv) throw UsageError("nix-shell requires a single derivation"); auto & drvInfo = drvs.front(); - auto drv = store->derivationFromPath(drvInfo.queryDrvPath()); + auto drv = store->derivationFromPath(store->parseStorePath(drvInfo.queryDrvPath())); - PathSet pathsToBuild; + std::vector<StorePathWithOutputs> pathsToBuild; /* Figure out what bash shell to use. If $NIX_BUILD_SHELL is not set, then build bashInteractive from @@ -358,7 +358,7 @@ static void _main(int argc, char * * argv) if (!drv) throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation"); - pathsToBuild.insert(drv->queryDrvPath()); + pathsToBuild.emplace_back(store->parseStorePath(drv->queryDrvPath())); shell = drv->queryOutPath() + "/bin/bash"; @@ -370,10 +370,11 @@ static void _main(int argc, char * * argv) // Build or fetch all dependencies of the derivation. for (const auto & input : drv.inputDrvs) - if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(input.first, std::regex(exclude)); })) - pathsToBuild.insert(makeDrvPathWithOutputs(input.first, input.second)); + if (std::all_of(envExclude.cbegin(), envExclude.cend(), + [&](const string & exclude) { return !std::regex_search(store->printStorePath(input.first), std::regex(exclude)); })) + pathsToBuild.emplace_back(input.first, input.second); for (const auto & src : drv.inputSrcs) - pathsToBuild.insert(src); + pathsToBuild.emplace_back(src); buildPaths(pathsToBuild); @@ -399,7 +400,7 @@ static void _main(int argc, char * * argv) env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); - auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile", "")); + auto passAsFile = tokenizeString<StringSet>(get(drv.env, "passAsFile").value_or("")); bool keepTmp = false; int fileNr = 0; @@ -468,7 +469,7 @@ static void _main(int argc, char * * argv) else { - PathSet pathsToBuild; + std::vector<StorePathWithOutputs> pathsToBuild; std::map<Path, Path> drvPrefixes; std::map<Path, Path> resultSymlinks; @@ -482,7 +483,7 @@ static void _main(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); - pathsToBuild.insert(drvPath + "!" + outputName); + pathsToBuild.emplace_back(store->parseStorePath(drvPath), StringSet{outputName}); std::string drvPrefix; auto i = drvPrefixes.find(drvPath); @@ -508,7 +509,7 @@ static void _main(int argc, char * * argv) for (auto & symlink : resultSymlinks) if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) - store2->addPermRoot(symlink.second, absPath(symlink.first), true); + store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true); for (auto & path : outPaths) std::cout << path << '\n'; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 70aa5c966..1b337a712 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -27,7 +27,7 @@ static void readChannels() continue; auto split = tokenizeString<std::vector<string>>(line, " "); auto url = std::regex_replace(split[0], std::regex("/*$"), ""); - auto name = split.size() > 1 ? split[1] : baseNameOf(url); + auto name = split.size() > 1 ? split[1] : std::string(baseNameOf(url)); channels[name] = url; } } @@ -98,10 +98,9 @@ static void update(const StringSet & channelNames) // shows something useful). auto cname = name; std::smatch match; - auto urlBase = baseNameOf(url); - if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) { + auto urlBase = std::string(baseNameOf(url)); + if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) cname = cname + (string) match[1]; - } std::string extraAttrs; diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index fdcde8b07..f87035760 100755 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -52,11 +52,11 @@ static int _main(int argc, char ** argv) auto to = toMode ? openStore(remoteUri) : openStore(); auto from = toMode ? openStore() : openStore(remoteUri); - PathSet storePaths2; + StorePathSet storePaths2; for (auto & path : storePaths) storePaths2.insert(from->followLinksToStorePath(path)); - PathSet closure; + StorePathSet closure; from->computeFSClosure(storePaths2, closure, false, includeOutputs); copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix-daemon/nix-daemon.cc index 73d1d1127..134898561 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix-daemon/nix-daemon.cc @@ -286,7 +286,7 @@ static int _main(int argc, char * * argv) if (chdir(socketDir.c_str()) == -1) throw SysError(format("changing to socket directory '%1%'") % socketDir); - auto socketName = baseNameOf(socketPath); + auto socketName = std::string(baseNameOf(socketPath)); auto addr = sockaddr_un{}; addr.sun_family = AF_UNIX; if (socketName.size() + 1 >= sizeof(addr.sun_path)) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 885e14aa1..03503eab1 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -208,10 +208,11 @@ static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) // at a time. static bool isPrebuilt(EvalState & state, DrvInfo & elem) { - Path path = elem.queryOutPath(); + auto path = state.store->parseStorePath(elem.queryOutPath()); if (state.store->isValidPath(path)) return true; - PathSet ps = state.store->querySubstitutablePaths({path}); - return ps.find(path) != ps.end(); + StorePathSet paths; + paths.insert(path.clone()); // FIXME: why doesn't StorePathSet{path.clone()} work? + return state.store->querySubstitutablePaths(paths).count(path); } @@ -371,24 +372,21 @@ static void queryInstSources(EvalState & state, case srcStorePaths: { for (auto & i : args) { - Path path = state.store->followLinksToStorePath(i); + auto path = state.store->followLinksToStorePath(i); - string name = baseNameOf(path); - string::size_type dash = name.find('-'); - if (dash != string::npos) - name = string(name, dash + 1); + std::string name(path.name()); DrvInfo elem(state, "", nullptr); elem.setName(name); - if (isDerivation(path)) { - elem.setDrvPath(path); - elem.setOutPath(state.store->derivationFromPath(path).findOutput("out")); + if (path.isDerivation()) { + elem.setDrvPath(state.store->printStorePath(path)); + elem.setOutPath(state.store->printStorePath(state.store->derivationFromPath(path).findOutput("out"))); if (name.size() >= drvExtension.size() && string(name, name.size() - drvExtension.size()) == drvExtension) name = string(name, 0, name.size() - drvExtension.size()); } - else elem.setOutPath(path); + else elem.setOutPath(state.store->printStorePath(path)); elems.push_back(elem); } @@ -421,13 +419,13 @@ static void queryInstSources(EvalState & state, static void printMissing(EvalState & state, DrvInfos & elems) { - PathSet targets; + std::vector<StorePathWithOutputs> targets; for (auto & i : elems) { Path drvPath = i.queryDrvPath(); if (drvPath != "") - targets.insert(drvPath); + targets.emplace_back(state.store->parseStorePath(drvPath)); else - targets.insert(i.queryOutPath()); + targets.emplace_back(state.store->parseStorePath(i.queryOutPath())); } printMissing(state.store, targets); @@ -697,15 +695,15 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) drv.setName(globals.forceName); if (drv.queryDrvPath() != "") { - PathSet paths = {drv.queryDrvPath()}; + std::vector<StorePathWithOutputs> paths{globals.state->store->parseStorePath(drv.queryDrvPath())}; printMissing(globals.state->store, paths); if (globals.dryRun) return; globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); - } - else { - printMissing(globals.state->store, {drv.queryOutPath()}); + } else { + printMissing(globals.state->store, + {globals.state->store->parseStorePath(drv.queryOutPath())}); if (globals.dryRun) return; - globals.state->store->ensurePath(drv.queryOutPath()); + globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath())); } debug(format("switching to new user environment")); @@ -729,7 +727,7 @@ static void uninstallDerivations(Globals & globals, Strings & selectors, for (auto & j : selectors) /* !!! the repeated calls to followLinksToStorePath() are expensive, should pre-compute them. */ - if ((isPath(j) && i.queryOutPath() == globals.state->store->followLinksToStorePath(j)) + if ((isPath(j) && globals.state->store->parseStorePath(i.queryOutPath()) == globals.state->store->followLinksToStorePath(j)) || DrvName(j).matches(drvName)) { printInfo("uninstalling '%s'", i.queryName()); @@ -953,12 +951,13 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Query which paths have substitutes. */ - PathSet validPaths, substitutablePaths; + StorePathSet validPaths; + StorePathSet substitutablePaths; if (printStatus || globals.prebuiltOnly) { - PathSet paths; + StorePathSet paths; for (auto & i : elems) try { - paths.insert(i.queryOutPath()); + paths.insert(globals.state->store->parseStorePath(i.queryOutPath())); } catch (AssertionError & e) { printMsg(lvlTalkative, "skipping derivation named '%s' which gives an assertion failure", i.queryName()); i.setFailed(); @@ -989,8 +988,8 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) //Activity act(*logger, lvlDebug, format("outputting query result '%1%'") % i.attrPath); if (globals.prebuiltOnly && - validPaths.find(i.queryOutPath()) == validPaths.end() && - substitutablePaths.find(i.queryOutPath()) == substitutablePaths.end()) + !validPaths.count(globals.state->store->parseStorePath(i.queryOutPath())) && + !substitutablePaths.count(globals.state->store->parseStorePath(i.queryOutPath()))) continue; /* For table output. */ @@ -1001,9 +1000,9 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if (printStatus) { Path outPath = i.queryOutPath(); - bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); + bool hasSubs = substitutablePaths.count(globals.state->store->parseStorePath(outPath)); bool isInstalled = installed.find(outPath) != installed.end(); - bool isValid = validPaths.find(outPath) != validPaths.end(); + bool isValid = validPaths.count(globals.state->store->parseStorePath(outPath)); if (xmlOutput) { attrs["installed"] = isInstalled ? "1" : "0"; attrs["valid"] = isValid ? "1" : "0"; @@ -1347,7 +1346,7 @@ static int _main(int argc, char * * argv) using LegacyArgs::LegacyArgs; }; - MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { Operation oldOp = op; if (*arg == "--help") diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 7b9a88281..8e4ecda4e 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -32,16 +32,16 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, { /* Build the components in the user environment, if they don't exist already. */ - PathSet drvsToBuild; + std::vector<StorePathWithOutputs> drvsToBuild; for (auto & i : elems) if (i.queryDrvPath() != "") - drvsToBuild.insert(i.queryDrvPath()); + drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())}); debug(format("building user environment dependencies")); state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); /* Construct the whole top level derivation. */ - PathSet references; + StorePathSet references; Value manifest; state.mkList(manifest, elems.size()); unsigned int n = 0; @@ -77,10 +77,10 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* This is only necessary when installing store paths, e.g., `nix-env -i /nix/store/abcd...-foo'. */ - state.store->addTempRoot(j.second); - state.store->ensurePath(j.second); + state.store->addTempRoot(state.store->parseStorePath(j.second)); + state.store->ensurePath(state.store->parseStorePath(j.second)); - references.insert(j.second); + references.insert(state.store->parseStorePath(j.second)); } // Copy the meta attributes. @@ -95,14 +95,14 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, vMeta.attrs->sort(); v.attrs->sort(); - if (drvPath != "") references.insert(drvPath); + if (drvPath != "") references.insert(state.store->parseStorePath(drvPath)); } /* Also write a copy of the list of user environment elements to the store; we need it for future modifications of the environment. */ - Path manifestFile = state.store->addTextToStore("env-manifest.nix", - (format("%1%") % manifest).str(), references); + auto manifestFile = state.store->addTextToStore("env-manifest.nix", + fmt("%s", manifest), references); /* Get the environment builder expression. */ Value envBuilder; @@ -113,7 +113,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, Value args, topLevel; state.mkAttrs(args, 3); mkString(*state.allocAttr(args, state.symbols.create("manifest")), - manifestFile, {manifestFile}); + state.store->printStorePath(manifestFile), {state.store->printStorePath(manifestFile)}); args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest)); args.attrs->sort(); mkApp(topLevel, envBuilder, args); @@ -123,13 +123,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, state.forceValue(topLevel); PathSet context; Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); - Path topLevelDrv = state.coerceToPath(aDrvPath.pos ? *(aDrvPath.pos) : noPos, *(aDrvPath.value), context); + auto topLevelDrv = state.store->parseStorePath(state.coerceToPath(aDrvPath.pos ? *(aDrvPath.pos) : noPos, *(aDrvPath.value), context)); Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); Path topLevelOut = state.coerceToPath(aOutPath.pos ? *(aOutPath.pos) : noPos, *(aOutPath.value), context); /* Realise the resulting store expression. */ debug("building user environment"); - state.store->buildPaths({topLevelDrv}, state.repair ? bmRepair : bmNormal); + std::vector<StorePathWithOutputs> topLevelDrvs; + topLevelDrvs.push_back(StorePathWithOutputs{topLevelDrv.clone()}); + state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index a736caa8f..5a886d69d 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -75,9 +75,9 @@ void processExpr(EvalState & state, const Strings & attrPaths, if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); if (store2) - drvPath = store2->addPermRoot(drvPath, rootName, indirectRoot); + drvPath = store2->addPermRoot(store2->parseStorePath(drvPath), rootName, indirectRoot); } - std::cout << format("%1%%2%\n") % drvPath % (outputName != "out" ? "!" + outputName : ""); + std::cout << fmt("%s%s\n", drvPath, (outputName != "out" ? "!" + outputName : "")); } } } @@ -105,7 +105,7 @@ static int _main(int argc, char * * argv) using LegacyArgs::LegacyArgs; }; - MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") showManPage("nix-instantiate"); else if (*arg == "--version") diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 78c883833..5040b6fb8 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -64,7 +64,7 @@ static int _main(int argc, char * * argv) using LegacyArgs::LegacyArgs; }; - MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--help") showManPage("nix-prefetch-url"); else if (*arg == "--version") @@ -156,17 +156,17 @@ static int _main(int argc, char * * argv) /* If an expected hash is given, the file may already exist in the store. */ Hash hash, expectedHash(ht); - Path storePath; + std::optional<StorePath> storePath; if (args.size() == 2) { expectedHash = Hash(args[1], ht); storePath = store->makeFixedOutputPath(unpack, expectedHash, name); - if (store->isValidPath(storePath)) + if (store->isValidPath(*storePath)) hash = expectedHash; else - storePath.clear(); + storePath.reset(); } - if (storePath.empty()) { + if (!storePath) { auto actualUri = resolveMirrorUri(*state, uri); @@ -193,7 +193,7 @@ static int _main(int argc, char * * argv) if (hasSuffix(baseNameOf(uri), ".zip")) runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); else - unpackTarfile(tmpFile, unpacked, baseNameOf(uri)); + unpackTarfile(tmpFile, unpacked, std::string(baseNameOf(uri))); /* If the archive unpacks to a single file/directory, then use that as the top-level. */ @@ -217,17 +217,17 @@ static int _main(int argc, char * * argv) into the Nix store. */ storePath = store->addToStore(name, tmpFile, unpack, ht); - assert(storePath == store->makeFixedOutputPath(unpack, hash, name)); + assert(*storePath == store->makeFixedOutputPath(unpack, hash, name)); } stopProgressBar(); if (!printPath) - printInfo(format("path is '%1%'") % storePath); + printInfo("path is '%s'", store->printStorePath(*storePath)); std::cout << printHash16or32(hash) << std::endl; if (printPath) - std::cout << storePath << std::endl; + std::cout << store->printStorePath(*storePath) << std::endl; return 0; } diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index d448654fe..a9d0dbbed 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -10,9 +10,9 @@ using std::cout; namespace nix { -static string dotQuote(const string & s) +static string dotQuote(std::string_view s) { - return "\"" + s + "\""; + return "\"" + std::string(s) + "\""; } @@ -34,7 +34,7 @@ static string makeEdge(const string & src, const string & dst) } -static string makeNode(const string & id, const string & label, +static string makeNode(const string & id, std::string_view label, const string & colour) { format f = format("%1% [label = %2%, shape = box, " @@ -44,13 +44,6 @@ static string makeNode(const string & id, const string & label, } -static string symbolicName(const string & path) -{ - string p = baseNameOf(path); - return string(p, p.find('-') + 1); -} - - #if 0 string pathLabel(const Path & nePath, const string & elemPath) { @@ -91,25 +84,24 @@ void printClosure(const Path & nePath, const StoreExpr & fs) #endif -void printDotGraph(ref<Store> store, const PathSet & roots) +void printDotGraph(ref<Store> store, StorePathSet && roots) { - PathSet workList(roots); - PathSet doneSet; + StorePathSet workList(std::move(roots)); + StorePathSet doneSet; cout << "digraph G {\n"; while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); + auto path = std::move(workList.extract(workList.begin()).value()); - if (!doneSet.insert(path).second) continue; + if (!doneSet.insert(path.clone()).second) continue; - cout << makeNode(path, symbolicName(path), "#ff0000"); + cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); for (auto & p : store->queryPathInfo(path)->references) { if (p != path) { - workList.insert(p); - cout << makeEdge(p, path); + workList.insert(p.clone()); + cout << makeEdge(std::string(p.to_string()), std::string(p.to_string())); } } diff --git a/src/nix-store/dotgraph.hh b/src/nix-store/dotgraph.hh index e2b5fc72f..73b8d06b9 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/nix-store/dotgraph.hh @@ -1,11 +1,9 @@ #pragma once -#include "types.hh" +#include "store-api.hh" namespace nix { -class Store; - -void printDotGraph(ref<Store> store, const PathSet & roots); +void printDotGraph(ref<Store> store, StorePathSet && roots); } diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 670fbe227..347708851 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -11,7 +11,7 @@ using std::cout; namespace nix { -static inline const string & xmlQuote(const string & s) +static inline std::string_view xmlQuote(std::string_view s) { // Luckily, store paths shouldn't contain any character that needs to be // quoted. @@ -19,14 +19,13 @@ static inline const string & xmlQuote(const string & s) } -static string symbolicName(const string & path) +static string symbolicName(const std::string & p) { - string p = baseNameOf(path); return string(p, p.find('-') + 1); } -static string makeEdge(const string & src, const string & dst) +static string makeEdge(std::string_view src, std::string_view dst) { return fmt(" <edge source=\"%1%\" target=\"%2%\"/>\n", xmlQuote(src), xmlQuote(dst)); @@ -41,18 +40,18 @@ static string makeNode(const ValidPathInfo & info) " <data key=\"name\">%3%</data>\n" " <data key=\"type\">%4%</data>\n" " </node>\n", - info.path, + info.path.to_string(), info.narSize, - symbolicName(info.path), - (isDerivation(info.path) ? "derivation" : "output-path")); + symbolicName(std::string(info.path.name())), + (info.path.isDerivation() ? "derivation" : "output-path")); } -void printGraphML(ref<Store> store, const PathSet & roots) +void printGraphML(ref<Store> store, StorePathSet && roots) { - PathSet workList(roots); - PathSet doneSet; - std::pair<PathSet::iterator,bool> ret; + StorePathSet workList(std::move(roots)); + StorePathSet doneSet; + std::pair<StorePathSet::iterator, bool> ret; cout << "<?xml version='1.0' encoding='utf-8'?>\n" << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n" @@ -64,19 +63,18 @@ void printGraphML(ref<Store> store, const PathSet & roots) << "<graph id='G' edgedefault='directed'>\n"; while (!workList.empty()) { - Path path = *(workList.begin()); - workList.erase(path); + auto path = std::move(workList.extract(workList.begin()).value()); - ret = doneSet.insert(path); + ret = doneSet.insert(path.clone()); if (ret.second == false) continue; - ValidPathInfo info = *(store->queryPathInfo(path)); - cout << makeNode(info); + auto info = store->queryPathInfo(path); + cout << makeNode(*info); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : info->references) { if (p != path) { - workList.insert(p); - cout << makeEdge(path, p); + workList.insert(p.clone()); + cout << makeEdge(path.to_string(), p.to_string()); } } diff --git a/src/nix-store/graphml.hh b/src/nix-store/graphml.hh index b78df1e49..78be8a367 100644 --- a/src/nix-store/graphml.hh +++ b/src/nix-store/graphml.hh @@ -1,11 +1,9 @@ #pragma once -#include "types.hh" +#include "store-api.hh" namespace nix { -class Store; - -void printGraphML(ref<Store> store, const PathSet & roots); +void printGraphML(ref<Store> store, StorePathSet && roots); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index dfa473db5..e2e2939f4 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -47,38 +47,37 @@ ref<LocalStore> ensureLocalStore() } -static Path useDeriver(Path path) +static StorePath useDeriver(const StorePath & path) { - if (isDerivation(path)) return path; - Path drvPath = store->queryPathInfo(path)->deriver; - if (drvPath == "") - throw Error(format("deriver of path '%1%' is not known") % path); - return drvPath; + if (path.isDerivation()) return path.clone(); + auto info = store->queryPathInfo(path); + if (!info->deriver) + throw Error("deriver of path '%s' is not known", store->printStorePath(path)); + return info->deriver->clone(); } /* Realise the given path. For a derivation that means build it; for other paths it means ensure their validity. */ -static PathSet realisePath(Path path, bool build = true) +static PathSet realisePath(StorePathWithOutputs path, bool build = true) { - DrvPathWithOutputs p = parseDrvPathWithOutputs(path); - auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store); - if (isDerivation(p.first)) { + if (path.path.isDerivation()) { if (build) store->buildPaths({path}); - Derivation drv = store->derivationFromPath(p.first); + Derivation drv = store->derivationFromPath(path.path); rootNr++; - if (p.second.empty()) - for (auto & i : drv.outputs) p.second.insert(i.first); + if (path.outputs.empty()) + for (auto & i : drv.outputs) path.outputs.insert(i.first); PathSet outputs; - for (auto & j : p.second) { + for (auto & j : path.outputs) { DerivationOutputs::iterator i = drv.outputs.find(j); if (i == drv.outputs.end()) - throw Error(format("derivation '%1%' does not have an output named '%2%'") % p.first % j); - Path outPath = i->second.path; + throw Error("derivation '%s' does not have an output named '%s'", + store2->printStorePath(path.path), j); + auto outPath = store2->printStorePath(i->second.path); if (store2) { if (gcRoot == "") printGCWarning(); @@ -86,7 +85,7 @@ static PathSet realisePath(Path path, bool build = true) Path rootName = gcRoot; if (rootNr > 1) rootName += "-" + std::to_string(rootNr); if (i->first != "out") rootName += "-" + i->first; - outPath = store2->addPermRoot(outPath, rootName, indirectRoot); + outPath = store2->addPermRoot(store->parseStorePath(outPath), rootName, indirectRoot); } } outputs.insert(outPath); @@ -95,8 +94,9 @@ static PathSet realisePath(Path path, bool build = true) } else { - if (build) store->ensurePath(path); - else if (!store->isValidPath(path)) throw Error(format("path '%1%' does not exist and cannot be created") % path); + if (build) store->ensurePath(path.path); + else if (!store->isValidPath(path.path)) + throw Error("path '%s' does not exist and cannot be created", store->printStorePath(path.path)); if (store2) { if (gcRoot == "") printGCWarning(); @@ -104,10 +104,10 @@ static PathSet realisePath(Path path, bool build = true) Path rootName = gcRoot; rootNr++; if (rootNr > 1) rootName += "-" + std::to_string(rootNr); - path = store2->addPermRoot(path, rootName, indirectRoot); + return {store2->addPermRoot(path.path, rootName, indirectRoot)}; } } - return {path}; + return {store->printStorePath(path.path)}; } } @@ -126,23 +126,20 @@ static void opRealise(Strings opFlags, Strings opArgs) else if (i == "--ignore-unknown") ignoreUnknown = true; else throw UsageError(format("unknown flag '%1%'") % i); - Paths paths; - for (auto & i : opArgs) { - DrvPathWithOutputs p = parseDrvPathWithOutputs(i); - paths.push_back(makeDrvPathWithOutputs(store->followLinksToStorePath(p.first), p.second)); - } + std::vector<StorePathWithOutputs> paths; + for (auto & i : opArgs) + paths.push_back(store->parseDrvPathWithOutputs(i)); unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - store->queryMissing(PathSet(paths.begin(), paths.end()), - willBuild, willSubstitute, unknown, downloadSize, narSize); + StorePathSet willBuild, willSubstitute, unknown; + store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); if (ignoreUnknown) { - Paths paths2; + std::vector<StorePathWithOutputs> paths2; for (auto & i : paths) - if (unknown.find(i) == unknown.end()) paths2.push_back(i); - paths = paths2; - unknown = PathSet(); + if (!unknown.count(i.path)) paths2.push_back(i); + paths = std::move(paths2); + unknown = StorePathSet(); } if (settings.printMissing) @@ -151,14 +148,14 @@ static void opRealise(Strings opFlags, Strings opArgs) if (dryRun) return; /* Build all paths at the same time to exploit parallelism. */ - store->buildPaths(PathSet(paths.begin(), paths.end()), buildMode); + store->buildPaths(paths, buildMode); if (!ignoreUnknown) for (auto & i : paths) { - PathSet paths = realisePath(i, false); + auto paths2 = realisePath(i, false); if (!noOutput) - for (auto & j : paths) - cout << format("%1%\n") % j; + for (auto & j : paths2) + cout << fmt("%1%\n", j); } } @@ -169,7 +166,7 @@ static void opAdd(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); for (auto & i : opArgs) - cout << format("%1%\n") % store->addToStore(baseNameOf(i), i); + cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i))); } @@ -190,7 +187,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) - cout << format("%1%\n") % store->addToStore(baseNameOf(i), i, recursive, hashAlgo); + cout << fmt("%s\n", store->printStorePath(store->addToStore(std::string(baseNameOf(i)), i, recursive, hashAlgo))); } @@ -211,22 +208,21 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << format("%1%\n") % - store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name))); } -static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forceRealise) +static StorePathSet maybeUseOutputs(const StorePath & storePath, bool useOutput, bool forceRealise) { if (forceRealise) realisePath(storePath); - if (useOutput && isDerivation(storePath)) { - Derivation drv = store->derivationFromPath(storePath); - PathSet outputs; + if (useOutput && storePath.isDerivation()) { + auto drv = store->derivationFromPath(storePath); + StorePathSet outputs; for (auto & i : drv.outputs) - outputs.insert(i.second.path); + outputs.insert(i.second.path.clone()); return outputs; } - else return {storePath}; + else return singleton(storePath.clone()); } @@ -239,23 +235,23 @@ const string treeLine = "| "; const string treeNull = " "; -static void printTree(const Path & path, - const string & firstPad, const string & tailPad, PathSet & done) +static void printTree(const StorePath & path, + const string & firstPad, const string & tailPad, StorePathSet & done) { - if (!done.insert(path).second) { - cout << format("%1%%2% [...]\n") % firstPad % path; + if (!done.insert(path.clone()).second) { + cout << fmt("%s%s [...]\n", firstPad, store->printStorePath(path)); return; } - cout << format("%1%%2%\n") % firstPad % path; + cout << fmt("%s%s\n", firstPad, store->printStorePath(path)); - auto references = store->queryPathInfo(path)->references; + auto info = store->queryPathInfo(path); /* Topologically sort under the relation A < B iff A \in closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - Paths sorted = store->topoSortPaths(references); + auto sorted = store->topoSortPaths(info->references); reverse(sorted.begin(), sorted.end()); for (auto i = sorted.begin(); i != sorted.end(); ++i) { @@ -318,11 +314,11 @@ static void opQuery(Strings opFlags, Strings opArgs) case qOutputs: { for (auto & i : opArgs) { - i = store->followLinksToStorePath(i); - if (forceRealise) realisePath(i); - Derivation drv = store->derivationFromPath(i); + auto i2 = store->followLinksToStorePath(i); + if (forceRealise) realisePath(i2); + Derivation drv = store->derivationFromPath(i2); for (auto & j : drv.outputs) - cout << format("%1%\n") % j.second.path; + cout << fmt("%1%\n", store->printStorePath(j.second.path)); } break; } @@ -331,51 +327,54 @@ static void opQuery(Strings opFlags, Strings opArgs) case qReferences: case qReferrers: case qReferrersClosure: { - PathSet paths; + StorePathSet paths; for (auto & i : opArgs) { - PathSet ps = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); + auto ps = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { for (auto & p : store->queryPathInfo(j)->references) - paths.insert(p); + paths.insert(p.clone()); + } + else if (query == qReferrers) { + StorePathSet tmp; + store->queryReferrers(j, tmp); + for (auto & i : tmp) + paths.insert(i.clone()); } - else if (query == qReferrers) store->queryReferrers(j, paths); else if (query == qReferrersClosure) store->computeFSClosure(j, paths, true); } } - Paths sorted = store->topoSortPaths(paths); - for (Paths::reverse_iterator i = sorted.rbegin(); + auto sorted = store->topoSortPaths(paths); + for (StorePaths::reverse_iterator i = sorted.rbegin(); i != sorted.rend(); ++i) - cout << format("%s\n") % *i; + cout << fmt("%s\n", store->printStorePath(*i)); break; } case qDeriver: for (auto & i : opArgs) { - Path deriver = store->queryPathInfo(store->followLinksToStorePath(i))->deriver; - cout << format("%1%\n") % - (deriver == "" ? "unknown-deriver" : deriver); + auto info = store->queryPathInfo(store->followLinksToStorePath(i)); + cout << fmt("%s\n", info->deriver ? store->printStorePath(*info->deriver) : "unknown-deriver"); } break; case qBinding: for (auto & i : opArgs) { - Path path = useDeriver(store->followLinksToStorePath(i)); + auto path = useDeriver(store->followLinksToStorePath(i)); Derivation drv = store->derivationFromPath(path); StringPairs::iterator j = drv.env.find(bindingName); if (j == drv.env.end()) - throw Error(format("derivation '%1%' has no environment binding named '%2%'") - % path % bindingName); - cout << format("%1%\n") % j->second; + throw Error("derivation '%s' has no environment binding named '%s'", + store->printStorePath(path), bindingName); + cout << fmt("%s\n", j->second); } break; case qHash: case qSize: for (auto & i : opArgs) { - PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - for (auto & j : paths) { + for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); @@ -387,50 +386,51 @@ static void opQuery(Strings opFlags, Strings opArgs) break; case qTree: { - PathSet done; + StorePathSet done; for (auto & i : opArgs) printTree(store->followLinksToStorePath(i), "", "", done); break; } case qGraph: { - PathSet roots; - for (auto & i : opArgs) { - PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - roots.insert(paths.begin(), paths.end()); - } - printDotGraph(ref<Store>(store), roots); + StorePathSet roots; + for (auto & i : opArgs) + for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) + roots.insert(j.clone()); + printDotGraph(ref<Store>(store), std::move(roots)); break; } case qGraphML: { - PathSet roots; - for (auto & i : opArgs) { - PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); - roots.insert(paths.begin(), paths.end()); - } - printGraphML(ref<Store>(store), roots); + StorePathSet roots; + for (auto & i : opArgs) + for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) + roots.insert(j.clone()); + printGraphML(ref<Store>(store), std::move(roots)); break; } case qResolve: { for (auto & i : opArgs) - cout << format("%1%\n") % store->followLinksToStorePath(i); + cout << fmt("%s\n", store->printStorePath(store->followLinksToStorePath(i))); break; } case qRoots: { - PathSet referrers; - for (auto & i : opArgs) { - store->computeFSClosure( - maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise), - referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); - } + StorePathSet args; + for (auto & i : opArgs) + for (auto & p : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) + args.insert(p.clone()); + + StorePathSet referrers; + store->computeFSClosure( + args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); + Roots roots = store->findRoots(false); for (auto & [target, links] : roots) if (referrers.find(target) != referrers.end()) for (auto & link : links) - cout << format("%1% -> %2%\n") % link % target; + cout << fmt("%1% -> %2%\n", link, store->printStorePath(target)); break; } @@ -446,7 +446,7 @@ static void opPrintEnv(Strings opFlags, Strings opArgs) if (opArgs.size() != 1) throw UsageError("'--print-env' requires one derivation store path"); Path drvPath = opArgs.front(); - Derivation drv = store->derivationFromPath(drvPath); + Derivation drv = store->derivationFromPath(store->parseStorePath(drvPath)); /* Print each environment variable in the derivation in a format that can be sourced by the shell. */ @@ -476,7 +476,7 @@ static void opReadLog(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); auto log = store->getBuildLog(path); if (!log) - throw Error("build log of derivation '%s' is not available", path); + throw Error("build log of derivation '%s' is not available", store->printStorePath(path)); std::cout << *log; } } @@ -487,13 +487,10 @@ static void opDumpDB(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); if (!opArgs.empty()) { for (auto & i : opArgs) - i = store->followLinksToStorePath(i); - for (auto & i : opArgs) - cout << store->makeValidityRegistration({i}, true, true); + cout << store->makeValidityRegistration(singleton(store->followLinksToStorePath(i)), true, true); } else { - PathSet validPaths = store->queryAllValidPaths(); - for (auto & i : validPaths) - cout << store->makeValidityRegistration({i}, true, true); + for (auto & i : store->queryAllValidPaths()) + cout << store->makeValidityRegistration(singleton(i), true, true); } } @@ -503,18 +500,18 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) ValidPathInfos infos; while (1) { - ValidPathInfo info = decodeValidPathInfo(cin, hashGiven); - if (info.path == "") break; - if (!store->isValidPath(info.path) || reregister) { + auto info = decodeValidPathInfo(*store, cin, hashGiven); + if (!info) break; + if (!store->isValidPath(info->path) || reregister) { /* !!! races */ if (canonicalise) - canonicalisePathMetaData(info.path, -1); + canonicalisePathMetaData(store->printStorePath(info->path), -1); if (!hashGiven) { - HashResult hash = hashPath(htSHA256, info.path); - info.narHash = hash.first; - info.narSize = hash.second; + HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); + info->narHash = hash.first; + info->narSize = hash.second; } - infos.push_back(info); + infos.push_back(std::move(*info)); } } @@ -556,12 +553,12 @@ static void opCheckValidity(Strings opFlags, Strings opArgs) else throw UsageError(format("unknown flag '%1%'") % i); for (auto & i : opArgs) { - Path path = store->followLinksToStorePath(i); + auto path = store->followLinksToStorePath(i); if (!store->isValidPath(path)) { if (printInvalid) - cout << format("%1%\n") % path; + cout << fmt("%s\n", store->printStorePath(path)); else - throw Error(format("path '%1%' is not valid") % path); + throw Error("path '%s' is not valid", store->printStorePath(path)); } } } @@ -591,13 +588,13 @@ static void opGC(Strings opFlags, Strings opArgs) if (printRoots) { Roots roots = store->findRoots(false); - std::set<std::pair<Path, Path>> roots2; + std::set<std::pair<Path, StorePath>> roots2; // Transpose and sort the roots. for (auto & [target, links] : roots) for (auto & link : links) - roots2.emplace(link, target); + roots2.emplace(link, target.clone()); for (auto & [link, target] : roots2) - std::cout << link << " -> " << target << "\n"; + std::cout << link << " -> " << store->printStorePath(target) << "\n"; } else { @@ -661,11 +658,13 @@ static void opExport(Strings opFlags, Strings opArgs) for (auto & i : opFlags) throw UsageError(format("unknown flag '%1%'") % i); + StorePathSet paths; + for (auto & i : opArgs) - i = store->followLinksToStorePath(i); + paths.insert(store->followLinksToStorePath(i)); FdSink sink(STDOUT_FILENO); - store->exportPaths(opArgs, sink); + store->exportPaths(paths, sink); sink.flush(); } @@ -678,10 +677,10 @@ static void opImport(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - Paths paths = store->importPaths(source, nullptr, NoCheckSigs); + auto paths = store->importPaths(source, nullptr, NoCheckSigs); for (auto & i : paths) - cout << format("%1%\n") % i << std::flush; + cout << fmt("%s\n", store->printStorePath(i)) << std::flush; } @@ -726,16 +725,16 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) int status = 0; for (auto & i : opArgs) { - Path path = store->followLinksToStorePath(i); - printMsg(lvlTalkative, format("checking path '%1%'...") % path); + auto path = store->followLinksToStorePath(i); + printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); HashSink sink(info->narHash.type); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { printError( - format("path '%1%' was modified! expected hash '%2%', got '%3%'") - % path % info->narHash.to_string() % current.first.to_string()); + "path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(path), info->narHash.to_string(), current.first.to_string()); status = 1; } } @@ -751,10 +750,8 @@ static void opRepairPath(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("no flags expected"); - for (auto & i : opArgs) { - Path path = store->followLinksToStorePath(i); - ensureLocalStore()->repairPath(path); - } + for (auto & i : opArgs) + ensureLocalStore()->repairPath(store->followLinksToStorePath(i)); } /* Optimise the disk space usage of the Nix store by hard-linking @@ -818,7 +815,7 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - PathSet paths = readStorePaths<PathSet>(*store, in); + auto paths = readStorePaths<StorePathSet>(*store, in); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -828,34 +825,39 @@ static void opServe(Strings opFlags, Strings opArgs) flag. */ if (substitute && writeAllowed) { /* Filter out .drv files (we don't want to build anything). */ - PathSet paths2; + std::vector<StorePathWithOutputs> paths2; for (auto & path : paths) - if (!isDerivation(path)) paths2.insert(path); + if (!path.isDerivation()) + paths2.emplace_back(path.clone()); unsigned long long downloadSize, narSize; - PathSet willBuild, willSubstitute, unknown; - store->queryMissing(PathSet(paths2.begin(), paths2.end()), + StorePathSet willBuild, willSubstitute, unknown; + store->queryMissing(paths2, willBuild, willSubstitute, unknown, downloadSize, narSize); /* FIXME: should use ensurePath(), but it only does one path at a time. */ if (!willSubstitute.empty()) try { - store->buildPaths(willSubstitute); + std::vector<StorePathWithOutputs> subs; + for (auto & p : willSubstitute) subs.emplace_back(p.clone()); + store->buildPaths(subs); } catch (Error & e) { printError(format("warning: %1%") % e.msg()); } } - out << store->queryValidPaths(paths); + writeStorePaths(*store, out, store->queryValidPaths(paths)); break; } case cmdQueryPathInfos: { - PathSet paths = readStorePaths<PathSet>(*store, in); + auto paths = readStorePaths<StorePathSet>(*store, in); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); - out << info->path << info->deriver << info->references; + out << store->printStorePath(info->path) + << (info->deriver ? store->printStorePath(*info->deriver) : ""); + writeStorePaths(*store, out, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -869,7 +871,7 @@ static void opServe(Strings opFlags, Strings opArgs) } case cmdDumpStorePath: - store->narFromPath(readStorePath(*store, in), out); + store->narFromPath(store->parseStorePath(readString(in)), out); break; case cmdImportPaths: { @@ -881,14 +883,17 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdExportPaths: { readInt(in); // obsolete - store->exportPaths(readStorePaths<Paths>(*store, in), out); + store->exportPaths(readStorePaths<StorePathSet>(*store, in), out); break; } case cmdBuildPaths: { if (!writeAllowed) throw Error("building paths is not allowed"); - PathSet paths = readStorePaths<PathSet>(*store, in); + + std::vector<StorePathWithOutputs> paths; + for (auto & s : readStrings<Strings>(in)) + paths.emplace_back(store->parseDrvPathWithOutputs(s)); getBuildSettings(); @@ -907,7 +912,7 @@ static void opServe(Strings opFlags, Strings opArgs) if (!writeAllowed) throw Error("building paths is not allowed"); - Path drvPath = readStorePath(*store, in); // informational only + auto drvPath = store->parseStorePath(readString(in)); // informational only BasicDerivation drv; readDerivation(in, *store, drv); @@ -926,30 +931,29 @@ static void opServe(Strings opFlags, Strings opArgs) case cmdQueryClosure: { bool includeOutputs = readInt(in); - PathSet closure; - store->computeFSClosure(readStorePaths<PathSet>(*store, in), + StorePathSet closure; + store->computeFSClosure(readStorePaths<StorePathSet>(*store, in), closure, false, includeOutputs); - out << closure; + writeStorePaths(*store, out, closure); break; } case cmdAddToStoreNar: { if (!writeAllowed) throw Error("importing paths is not allowed"); - ValidPathInfo info; - info.path = readStorePath(*store, in); - in >> info.deriver; - if (!info.deriver.empty()) - store->assertStorePath(info.deriver); + auto path = readString(in); + ValidPathInfo info(store->parseStorePath(path)); + auto deriver = readString(in); + if (deriver != "") + info.deriver = store->parseStorePath(deriver); info.narHash = Hash(readString(in), htSHA256); - info.references = readStorePaths<PathSet>(*store, in); + info.references = readStorePaths<StorePathSet>(*store, in); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(in); in >> info.ca; - if (info.narSize == 0) { + if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); - } SizedSource sizedSource(in, info.narSize); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 296b2c7e4..139db3657 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -40,16 +40,17 @@ struct CmdAddToStore : MixDryRun, StoreCommand StringSink sink; dumpPath(path, sink); - ValidPathInfo info; - info.narHash = hashString(htSHA256, *sink.s); + auto narHash = hashString(htSHA256, *sink.s); + + ValidPathInfo info(store->makeFixedOutputPath(true, narHash, *namePart)); + info.narHash = narHash; info.narSize = sink.s->size(); - info.path = store->makeFixedOutputPath(true, info.narHash, *namePart); info.ca = makeFixedOutputCA(true, info.narHash); if (!dryRun) store->addToStore(info, sink.s); - std::cout << fmt("%s\n", info.path); + std::cout << fmt("%s\n", store->printStorePath(info.path)); } }; diff --git a/src/nix/command.cc b/src/nix/command.cc index 2da5736e7..99b24d2a2 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -51,28 +51,29 @@ StorePathsCommand::StorePathsCommand(bool recursive) void StorePathsCommand::run(ref<Store> store) { - Paths storePaths; + StorePaths storePaths; if (all) { if (installables.size()) throw UsageError("'--all' does not expect arguments"); for (auto & p : store->queryAllValidPaths()) - storePaths.push_back(p); + storePaths.push_back(p.clone()); } else { for (auto & p : toStorePaths(store, realiseMode, installables)) - storePaths.push_back(p); + storePaths.push_back(p.clone()); if (recursive) { - PathSet closure; - store->computeFSClosure(PathSet(storePaths.begin(), storePaths.end()), - closure, false, false); - storePaths = Paths(closure.begin(), closure.end()); + StorePathSet closure; + store->computeFSClosure(storePathsToSet(storePaths), closure, false, false); + storePaths.clear(); + for (auto & p : closure) + storePaths.push_back(p.clone()); } } - run(store, storePaths); + run(store, std::move(storePaths)); } void StorePathCommand::run(ref<Store> store) @@ -107,7 +108,7 @@ MixProfile::MixProfile() .dest(&profile); } -void MixProfile::updateProfile(const Path & storePath) +void MixProfile::updateProfile(const StorePath & storePath) { if (!profile) return; auto store = getStore().dynamic_pointer_cast<LocalFSStore>(); @@ -116,20 +117,20 @@ void MixProfile::updateProfile(const Path & storePath) switchLink(profile2, createGeneration( ref<LocalFSStore>(store), - profile2, storePath)); + profile2, store->printStorePath(storePath))); } void MixProfile::updateProfile(const Buildables & buildables) { if (!profile) return; - std::optional<Path> result; + std::optional<StorePath> result; for (auto & buildable : buildables) { for (auto & output : buildable.outputs) { if (result) throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); - result = output.second; + result = output.second.clone(); } } diff --git a/src/nix/command.hh b/src/nix/command.hh index 4f3f6eb9e..ba75ba6e6 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -3,6 +3,7 @@ #include "installables.hh" #include "args.hh" #include "common-eval-args.hh" +#include "path.hh" #include <optional> @@ -35,8 +36,6 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs { ref<EvalState> getEvalState(); -private: - std::shared_ptr<EvalState> evalState; }; @@ -128,7 +127,7 @@ public: using StoreCommand::run; - virtual void run(ref<Store> store, Paths storePaths) = 0; + virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0; void run(ref<Store> store) override; @@ -140,7 +139,7 @@ struct StorePathCommand : public InstallablesCommand { using StoreCommand::run; - virtual void run(ref<Store> store, const Path & storePath) = 0; + virtual void run(ref<Store> store, const StorePath & storePath) = 0; void run(ref<Store> store) override; }; @@ -167,13 +166,13 @@ static RegisterCommand registerCommand(const std::string & name) Buildables build(ref<Store> store, RealiseMode mode, std::vector<std::shared_ptr<Installable>> installables); -PathSet toStorePaths(ref<Store> store, RealiseMode mode, +std::set<StorePath> toStorePaths(ref<Store> store, RealiseMode mode, std::vector<std::shared_ptr<Installable>> installables); -Path toStorePath(ref<Store> store, RealiseMode mode, +StorePath toStorePath(ref<Store> store, RealiseMode mode, std::shared_ptr<Installable> installable); -PathSet toDerivations(ref<Store> store, +std::set<StorePath> toDerivations(ref<Store> store, std::vector<std::shared_ptr<Installable>> installables, bool useDeriver = false); @@ -188,7 +187,7 @@ struct MixProfile : virtual Args, virtual StoreCommand MixProfile(); /* If 'profile' is set, make it point at 'storePath'. */ - void updateProfile(const Path & storePath); + void updateProfile(const StorePath & storePath); /* If 'profile' is set, make it point at the store path produced by 'buildables'. */ diff --git a/src/nix/copy.cc b/src/nix/copy.cc index b1aceb15c..85c777d38 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -80,14 +80,14 @@ struct CmdCopy : StorePathsCommand return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); } - void run(ref<Store> srcStore, Paths storePaths) override + void run(ref<Store> srcStore, StorePaths storePaths) override { if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass '--from' and/or '--to'"); ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()), + copyPaths(srcStore, dstStore, storePathsToSet(storePaths), NoRepair, checkSigs, substitute); } }; diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 90f1552d9..bb741b572 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -20,7 +20,7 @@ struct CmdDumpPath : StorePathCommand }; } - void run(ref<Store> store, const Path & storePath) override + void run(ref<Store> store, const StorePath & storePath) override { FdSink sink(STDOUT_FILENO); store->narFromPath(storePath, sink); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 6e7c5e2eb..22e994e58 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -263,22 +263,23 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON if (!drvInfo) throw Error("flake attribute '%s' is not a derivation", attrPath); // FIXME: check meta attributes - return drvInfo->queryDrvPath(); + return store->parseStorePath(drvInfo->queryDrvPath()); } catch (Error & e) { e.addPrefix(fmt("while checking the derivation '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); throw; } }; - PathSet drvPaths; + std::vector<StorePathWithOutputs> drvPaths; auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { auto app = App(*state, v); for (auto & i : app.context) { - auto [drvPath, outputName] = decodeContext(i); - if (!outputName.empty() && nix::isDerivation(drvPath)) - drvPaths.insert(drvPath + "!" + outputName); + auto [drvPathS, outputName] = decodeContext(i); + auto drvPath = store->parseStorePath(drvPathS); + if (!outputName.empty() && drvPath.isDerivation()) + drvPaths.emplace_back(drvPath); } } catch (Error & e) { e.addPrefix(fmt("while checking the app definition '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); @@ -388,7 +389,7 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON fmt("%s.%s.%s", name, attr.name, attr2.name), *attr2.value, *attr2.pos); if ((std::string) attr.name == settings.thisSystem.get()) - drvPaths.insert(drvPath); + drvPaths.emplace_back(drvPath); } } } diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 3e0fe2606..86b9fbdb9 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -118,20 +118,30 @@ App Installable::toApp(EvalState & state) struct InstallableStorePath : Installable { - Path storePath; + ref<Store> store; + StorePath storePath; - InstallableStorePath(const Path & storePath) : storePath(storePath) { } + InstallableStorePath(ref<Store> store, const Path & storePath) + : store(store), storePath(store->parseStorePath(storePath)) { } - std::string what() override { return storePath; } + std::string what() override { return store->printStorePath(storePath); } Buildables toBuildables() override { - return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}}; + std::map<std::string, StorePath> outputs; + outputs.insert_or_assign("out", storePath.clone()); + Buildable b{ + .drvPath = storePath.isDerivation() ? storePath.clone() : std::optional<StorePath>(), + .outputs = std::move(outputs) + }; + Buildables bs; + bs.push_back(std::move(b)); + return bs; } - std::optional<Path> getStorePath() override + std::optional<StorePath> getStorePath() override { - return storePath; + return storePath.clone(); } }; @@ -149,8 +159,8 @@ std::vector<flake::EvalCache::Derivation> InstallableValue::toDerivations() std::vector<flake::EvalCache::Derivation> res; for (auto & drvInfo : drvInfos) { res.push_back({ - drvInfo.queryDrvPath(), - drvInfo.queryOutPath(), + state->store->parseStorePath(drvInfo.queryDrvPath()), + state->store->parseStorePath(drvInfo.queryOutPath()), drvInfo.queryOutputName() }); } @@ -160,19 +170,21 @@ std::vector<flake::EvalCache::Derivation> InstallableValue::toDerivations() Buildables InstallableValue::toBuildables() { + auto state = cmd.getEvalState(); + Buildables res; - PathSet drvPaths; + StorePathSet drvPaths; for (auto & drv : toDerivations()) { - Buildable b{drv.drvPath}; - drvPaths.insert(b.drvPath); + Buildable b{.drvPath = drv.drvPath.clone()}; + drvPaths.insert(drv.drvPath.clone()); auto outputName = drv.outputName; if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath); + throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); - b.outputs.emplace(outputName, drv.outPath); + b.outputs.emplace(outputName, drv.outPath.clone()); res.push_back(std::move(b)); } @@ -180,10 +192,13 @@ Buildables InstallableValue::toBuildables() // Hack to recognize .all: if all drvs have the same drvPath, // merge the buildables. if (drvPaths.size() == 1) { - Buildable b{*drvPaths.begin()}; + Buildable b{.drvPath = drvPaths.begin()->clone()}; for (auto & b2 : res) - b.outputs.insert(b2.outputs.begin(), b2.outputs.end()); - return {b}; + for (auto & output : b2.outputs) + b.outputs.insert_or_assign(output.first, output.second.clone()); + Buildables bs; + bs.push_back(std::move(b)); + return bs; } else return res; } @@ -231,10 +246,10 @@ void makeFlakeClosureGCRoot(Store & store, if (std::get_if<FlakeRef::IsPath>(&origFlakeRef.data)) return; /* Get the store paths of all non-local flakes. */ - PathSet closure; + StorePathSet closure; - assert(store.isValidPath(resFlake.flake.sourceInfo.storePath)); - closure.insert(resFlake.flake.sourceInfo.storePath); + assert(store.isValidPath(store.parseStorePath(resFlake.flake.sourceInfo.storePath))); + closure.insert(store.parseStorePath(resFlake.flake.sourceInfo.storePath)); std::queue<std::reference_wrapper<const flake::LockedInputs>> queue; queue.push(resFlake.lockFile); @@ -246,8 +261,8 @@ void makeFlakeClosureGCRoot(Store & store, yet. */ for (auto & dep : flake.inputs) { auto path = dep.second.computeStorePath(store); - if (store.isValidPath(path)) - closure.insert(path); + if (store.isValidPath(store.parseStorePath(path))) + closure.insert(store.parseStorePath(path)); queue.push(dep.second); } } @@ -255,7 +270,8 @@ void makeFlakeClosureGCRoot(Store & store, if (closure.empty()) return; /* Write the closure to a file in the store. */ - auto closurePath = store.addTextToStore("flake-closure", concatStringsSep(" ", closure), closure); + auto closurePath = store.addTextToStore("flake-closure", + concatStringsSep(" ", store.printStorePathSet(closure)), closure); Path cacheDir = getCacheDir() + "/nix/flake-closures"; createDirs(cacheDir); @@ -267,7 +283,7 @@ void makeFlakeClosureGCRoot(Store & store, s = replaceStrings(s, ":", "%3a"); Path symlink = cacheDir + "/" + s; debug("writing GC root '%s' for flake closure of '%s'", symlink, origFlakeRef); - replaceSymlink(closurePath, symlink); + replaceSymlink(store.printStorePath(closurePath), symlink); store.addIndirectRoot(symlink); } @@ -318,7 +334,7 @@ std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> InstallableFlake auto drv = evalCache.getDerivation(fingerprint, attrPath); if (drv) { if (state->store->isValidPath(drv->drvPath)) - return {attrPath, resFlake.flake.sourceInfo.resolvedRef, *drv}; + return {attrPath, resFlake.flake.sourceInfo.resolvedRef, std::move(*drv)}; } if (!vOutputs) @@ -333,14 +349,14 @@ std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> InstallableFlake throw Error("flake output attribute '%s' is not a derivation", attrPath); auto drv = flake::EvalCache::Derivation{ - drvInfo->queryDrvPath(), - drvInfo->queryOutPath(), + state->store->parseStorePath(drvInfo->queryDrvPath()), + state->store->parseStorePath(drvInfo->queryOutPath()), drvInfo->queryOutputName() }; evalCache.addDerivation(fingerprint, attrPath, drv); - return {attrPath, resFlake.flake.sourceInfo.resolvedRef, drv}; + return {attrPath, resFlake.flake.sourceInfo.resolvedRef, std::move(drv)}; } catch (AttrPathNotFound & e) { } } @@ -351,7 +367,9 @@ std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> InstallableFlake std::vector<flake::EvalCache::Derivation> InstallableFlake::toDerivations() { - return {std::get<2>(toDerivation())}; + std::vector<flake::EvalCache::Derivation> res; + res.push_back(std::get<2>(toDerivation())); + return res; } Value * InstallableFlake::toValue(EvalState & state) @@ -406,7 +424,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( } else { - auto follow = [&](const std::string & s) -> std::optional<Path> { + auto follow = [&](const std::string & s) -> std::optional<StorePath> { try { return store->followLinksToStorePath(s); } catch (NotInStore &) { @@ -417,7 +435,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( for (auto & s : ss) { size_t hash; - std::optional<Path> storePath; + std::optional<StorePath> storePath; if (hasPrefix(s, "nixpkgs.")) { bool static warned; @@ -440,7 +458,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( *this, std::move(flakeRef), getDefaultFlakeAttrPaths())); } catch (...) { if (s.find('/') != std::string::npos && (storePath = follow(s))) - result.push_back(std::make_shared<InstallableStorePath>(*storePath)); + result.push_back(std::make_shared<InstallableStorePath>(store, store->printStorePath(*storePath))); else throw; } @@ -467,19 +485,18 @@ Buildables build(ref<Store> store, RealiseMode mode, Buildables buildables; - PathSet pathsToBuild; + std::vector<StorePathWithOutputs> pathsToBuild; for (auto & i : installables) { for (auto & b : i->toBuildables()) { - if (b.drvPath != "") { + if (b.drvPath) { StringSet outputNames; for (auto & output : b.outputs) outputNames.insert(output.first); - pathsToBuild.insert( - b.drvPath + "!" + concatStringsSep(",", outputNames)); + pathsToBuild.push_back({*b.drvPath, outputNames}); } else for (auto & output : b.outputs) - pathsToBuild.insert(output.second); + pathsToBuild.push_back({output.second.clone()}); buildables.push_back(std::move(b)); } } @@ -492,19 +509,19 @@ Buildables build(ref<Store> store, RealiseMode mode, return buildables; } -PathSet toStorePaths(ref<Store> store, RealiseMode mode, +StorePathSet toStorePaths(ref<Store> store, RealiseMode mode, std::vector<std::shared_ptr<Installable>> installables) { - PathSet outPaths; + StorePathSet outPaths; for (auto & b : build(store, mode, installables)) for (auto & output : b.outputs) - outPaths.insert(output.second); + outPaths.insert(output.second.clone()); return outPaths; } -Path toStorePath(ref<Store> store, RealiseMode mode, +StorePath toStorePath(ref<Store> store, RealiseMode mode, std::shared_ptr<Installable> installable) { auto paths = toStorePaths(store, mode, {installable}); @@ -512,17 +529,17 @@ Path toStorePath(ref<Store> store, RealiseMode mode, if (paths.size() != 1) throw Error("argument '%s' should evaluate to one store path", installable->what()); - return *paths.begin(); + return paths.begin()->clone(); } -PathSet toDerivations(ref<Store> store, +StorePathSet toDerivations(ref<Store> store, std::vector<std::shared_ptr<Installable>> installables, bool useDeriver) { - PathSet drvPaths; + StorePathSet drvPaths; for (auto & i : installables) for (auto & b : i->toBuildables()) { - if (b.drvPath.empty()) { + if (!b.drvPath) { if (!useDeriver) throw Error("argument '%s' did not evaluate to a derivation", i->what()); for (auto & output : b.outputs) { @@ -530,10 +547,10 @@ PathSet toDerivations(ref<Store> store, if (derivers.empty()) throw Error("'%s' does not have a known deriver", i->what()); // FIXME: use all derivers? - drvPaths.insert(*derivers.begin()); + drvPaths.insert(derivers.begin()->clone()); } } else - drvPaths.insert(b.drvPath); + drvPaths.insert(b.drvPath->clone()); } return drvPaths; diff --git a/src/nix/installables.hh b/src/nix/installables.hh index 612c8ac92..22e4b38f9 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -1,6 +1,7 @@ #pragma once #include "util.hh" +#include "path.hh" #include "flake/eval-cache.hh" #include <optional> @@ -14,8 +15,8 @@ struct SourceExprCommand; struct Buildable { - Path drvPath; // may be empty - std::map<std::string, Path> outputs; + std::optional<StorePath> drvPath; + std::map<std::string, StorePath> outputs; }; typedef std::vector<Buildable> Buildables; @@ -51,7 +52,7 @@ struct Installable /* Return a value only if this installable is a store path or a symlink to it. */ - virtual std::optional<Path> getStorePath() + virtual std::optional<StorePath> getStorePath() { return {}; } diff --git a/src/nix/local.mk b/src/nix/local.mk index 44a95f910..fff101a63 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -15,7 +15,7 @@ nix_SOURCES := \ $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_LIBS = libexpr libmain libstore libutil +nix_LIBS = libexpr libmain libstore libutil libnixrust nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system diff --git a/src/nix/log.cc b/src/nix/log.cc index 122a3d690..795991cb7 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -43,7 +43,7 @@ struct CmdLog : InstallableCommand RunPager pager; for (auto & sub : subs) { - auto log = b.drvPath != "" ? sub->getBuildLog(b.drvPath) : nullptr; + auto log = b.drvPath ? sub->getBuildLog(*b.drvPath) : nullptr; for (auto & output : b.outputs) { if (log) break; log = sub->getBuildLog(output.second); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 9408cc9da..3ef1f2750 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -67,7 +67,7 @@ struct MixLs : virtual Args, MixJSON if (st.type == FSAccessor::Type::tMissing) throw Error(format("path '%1%' does not exist") % path); doPath(st, path, - st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path), + st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), showDirectory); } diff --git a/src/nix/main.cc b/src/nix/main.cc index 272c944ba..f0e6cac4c 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -140,7 +140,7 @@ void mainWrapped(int argc, char * * argv) initGC(); programPath = argv[0]; - string programName = baseNameOf(programPath); + auto programName = std::string(baseNameOf(programPath)); { auto legacy = (*RegisterLegacyCommand::commands)[programName]; diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 5b99b5084..524d467c2 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -29,34 +29,36 @@ struct CmdMakeContentAddressable : StorePathsCommand }, }; } - void run(ref<Store> store, Paths storePaths) override + void run(ref<Store> store, StorePaths storePaths) override { - auto paths = store->topoSortPaths(PathSet(storePaths.begin(), storePaths.end())); + auto paths = store->topoSortPaths(storePathsToSet(storePaths)); - paths.reverse(); + std::reverse(paths.begin(), paths.end()); - std::map<Path, Path> remappings; + std::map<StorePath, StorePath> remappings; for (auto & path : paths) { + auto pathS = store->printStorePath(path); auto oldInfo = store->queryPathInfo(path); - auto oldHashPart = storePathToHash(path); - auto name = storePathToName(path); + auto oldHashPart = storePathToHash(pathS); StringSink sink; store->narFromPath(path, sink); StringMap rewrites; - ValidPathInfo info; + StorePathSet references; + bool hasSelfReference = false; for (auto & ref : oldInfo->references) { if (ref == path) - info.references.insert("self"); + hasSelfReference = true; else { - auto replacement = get(remappings, ref, ref); + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second.clone() : ref.clone(); // FIXME: warn about unremapped paths? - info.references.insert(replacement); if (replacement != ref) - rewrites[storePathToHash(ref)] = storePathToHash(replacement); + rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); + references.insert(std::move(replacement)); } } @@ -65,24 +67,26 @@ struct CmdMakeContentAddressable : StorePathsCommand HashModuloSink hashModuloSink(htSHA256, oldHashPart); hashModuloSink((unsigned char *) sink.s->data(), sink.s->size()); - info.narHash = hashModuloSink.finish().first; + auto narHash = hashModuloSink.finish().first; + + ValidPathInfo info(store->makeFixedOutputPath(true, narHash, path.name(), references, hasSelfReference)); + info.references = std::move(references); + if (hasSelfReference) info.references.insert(info.path.clone()); + info.narHash = narHash; info.narSize = sink.s->size(); - replaceInSet(info.references, path, std::string("self")); - info.path = store->makeFixedOutputPath(true, info.narHash, name, info.references); - replaceInSet(info.references, std::string("self"), info.path); info.ca = makeFixedOutputCA(true, info.narHash); - printError("rewrote '%s' to '%s'", path, info.path); + printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, storePathToHash(info.path), nextSink); + RewritingSink rsink2(oldHashPart, storePathToHash(store->printStorePath(info.path)), nextSink); rsink2((unsigned char *) sink.s->data(), sink.s->size()); rsink2.flush(); }); store->addToStore(info, *source); - remappings[path] = info.path; + remappings.insert_or_assign(std::move(path), std::move(info.path)); } } }; diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 2cb718f12..bffa7b356 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -78,36 +78,36 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << fmt("\t%6.1f%c", res, idents.at(power)); } - void run(ref<Store> store, Paths storePaths) override + void run(ref<Store> store, StorePaths storePaths) override { size_t pathLen = 0; for (auto & storePath : storePaths) - pathLen = std::max(pathLen, storePath.size()); + pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { JSONPlaceholder jsonRoot(std::cout); store->pathInfoToJSON(jsonRoot, // FIXME: preserve order? - PathSet(storePaths.begin(), storePaths.end()), + storePathsToSet(storePaths), true, showClosureSize, AllowInvalid); } else { - for (auto storePath : storePaths) { + for (auto & storePath : storePaths) { auto info = store->queryPathInfo(storePath); - storePath = info->path; // FIXME: screws up padding + auto storePathS = store->printStorePath(storePath); - std::cout << storePath; + std::cout << storePathS; if (showSize || showClosureSize || showSigs) - std::cout << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' '); + std::cout << std::string(std::max(0, (int) pathLen - (int) storePathS.size()), ' '); if (showSize) printSize(info->narSize); if (showClosureSize) - printSize(store->getClosureSize(storePath).first); + printSize(store->getClosureSize(info->path).first); if (showSigs) { std::cout << '\t'; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 786ebddef..6ea529f52 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -24,7 +24,7 @@ struct ProfileElementSource struct ProfileElement { - PathSet storePaths; + StorePathSet storePaths; std::optional<ProfileElementSource> source; bool active = true; // FIXME: priority @@ -50,7 +50,7 @@ struct ProfileManifest for (auto & e : json["elements"]) { ProfileElement element; for (auto & p : e["storePaths"]) - element.storePaths.insert((std::string) p); + element.storePaths.insert(state.store->parseStorePath((std::string) p)); element.active = e["active"]; if (e.value("uri", "") != "") { element.source = ProfileElementSource{ @@ -74,19 +74,19 @@ struct ProfileManifest for (auto & drvInfo : drvInfos) { ProfileElement element; - element.storePaths = {drvInfo.queryOutPath()}; + element.storePaths = singleton(state.store->parseStorePath(drvInfo.queryOutPath())); elements.emplace_back(std::move(element)); } } } - std::string toJSON() const + std::string toJSON(Store & store) const { auto array = nlohmann::json::array(); for (auto & element : elements) { auto paths = nlohmann::json::array(); for (auto & path : element.storePaths) - paths.push_back(path); + paths.push_back(store.printStorePath(path)); nlohmann::json obj; obj["storePaths"] = paths; obj["active"] = element.active; @@ -103,37 +103,38 @@ struct ProfileManifest return json.dump(); } - Path build(ref<Store> store) + StorePath build(ref<Store> store) { auto tempDir = createTempDir(); - ValidPathInfo info; + StorePathSet references; Packages pkgs; for (auto & element : elements) { for (auto & path : element.storePaths) { if (element.active) - pkgs.emplace_back(path, true, 5); - info.references.insert(path); + pkgs.emplace_back(store->printStorePath(path), true, 5); + references.insert(path.clone()); } } buildProfile(tempDir, std::move(pkgs)); - writeFile(tempDir + "/manifest.json", toJSON()); + writeFile(tempDir + "/manifest.json", toJSON(*store)); /* Add the symlink tree to the store. */ StringSink sink; dumpPath(tempDir, sink); + ValidPathInfo info(store->makeFixedOutputPath(true, info.narHash, "profile", references)); + info.references = std::move(references); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); - info.path = store->makeFixedOutputPath(true, info.narHash, "profile", info.references); info.ca = makeFixedOutputCA(true, info.narHash); store->addToStore(info, sink.s); - return info.path; + return std::move(info.path); } }; @@ -166,21 +167,21 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - PathSet pathsToBuild; + std::vector<StorePathWithOutputs> pathsToBuild; for (auto & installable : installables) { if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) { auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); ProfileElement element; - element.storePaths = {drv.outPath}; // FIXME + element.storePaths = singleton(drv.outPath.clone()); // FIXME element.source = ProfileElementSource{ installable2->flakeRef, resolvedRef, attrPath, }; - pathsToBuild.insert(makeDrvPathWithOutputs(drv.drvPath, {"out"})); // FIXME + pathsToBuild.emplace_back(drv.drvPath.clone(), StringSet{"out"}); // FIXME manifest.elements.emplace_back(std::move(element)); } else @@ -223,13 +224,13 @@ public: return res; } - bool matches(const ProfileElement & element, size_t pos, std::vector<Matcher> matchers) + bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector<Matcher> & matchers) { for (auto & matcher : matchers) { if (auto n = std::get_if<size_t>(&matcher)) { if (*n == pos) return true; } else if (auto path = std::get_if<Path>(&matcher)) { - if (element.storePaths.count(*path)) return true; + if (element.storePaths.count(store.parseStorePath(*path))) return true; } else if (auto regex = std::get_if<std::regex>(&matcher)) { if (element.source && std::regex_match(element.source->attrPath, *regex)) @@ -280,8 +281,8 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem for (size_t i = 0; i < oldManifest.elements.size(); ++i) { auto & element(oldManifest.elements[i]); - if (!matches(element, i, matchers)) - newManifest.elements.push_back(element); + if (!matches(*store, element, i, matchers)) + newManifest.elements.push_back(std::move(element)); } // FIXME: warn about unused matchers? @@ -322,13 +323,13 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); // FIXME: code duplication - PathSet pathsToBuild; + std::vector<StorePathWithOutputs> pathsToBuild; for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); if (element.source && !element.source->originalRef.isImmutable() - && matches(element, i, matchers)) + && matches(*store, element, i, matchers)) { Activity act(*logger, lvlChatty, actUnknown, fmt("checking '%s' for updates", element.source->attrPath)); @@ -342,14 +343,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf printInfo("upgrading '%s' from flake '%s' to '%s'", element.source->attrPath, element.source->resolvedRef, resolvedRef); - element.storePaths = {drv.outPath}; // FIXME + element.storePaths = singleton(drv.outPath.clone()); // FIXME element.source = ProfileElementSource{ installable.flakeRef, resolvedRef, attrPath, }; - pathsToBuild.insert(makeDrvPathWithOutputs(drv.drvPath, {"out"})); // FIXME + pathsToBuild.emplace_back(drv.drvPath, StringSet{"out"}); // FIXME } } @@ -385,7 +386,7 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro std::cout << fmt("%d %s %s %s\n", i, element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", - concatStringsSep(" ", element.storePaths)); + concatStringsSep(" ", store->printStorePathSet(element.storePaths))); } } }; diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index 661966733..d20c09f26 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -24,6 +24,13 @@ static uint64_t getI(const std::vector<Logger::Field> & fields, size_t n) return fields[n].i; } +static std::string_view storePathToName(std::string_view path) +{ + auto base = baseNameOf(path); + auto i = base.find('-'); + return i == std::string::npos ? base.substr(0, 0) : base.substr(i + 1); +} + class ProgressBar : public Logger { private: @@ -148,7 +155,7 @@ public: if (type == actBuild) { auto name = storePathToName(getS(fields, 0)); if (hasSuffix(name, ".drv")) - name.resize(name.size() - 4); + name = name.substr(name.size() - 4); i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); auto machineName = getS(fields, 1); if (machineName != "") @@ -173,7 +180,7 @@ public: if (type == actPostBuildHook) { auto name = storePathToName(getS(fields, 0)); if (hasSuffix(name, ".drv")) - name.resize(name.size() - 4); + name = name.substr(name.size() - 4); i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); i->name = DrvName(name).name; } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 2b4d1a2c4..9f76e1b4f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -413,7 +413,7 @@ Path NixRepl::getDerivationPath(Value & v) { if (!drvInfo) throw Error("expression does not evaluate to a derivation, so I can't build it"); Path drvPath = drvInfo->queryDrvPath(); - if (drvPath == "" || !state.store->isValidPath(drvPath)) + if (drvPath == "" || !state.store->isValidPath(state.store->parseStorePath(drvPath))) throw Error("expression did not evaluate to a valid derivation"); return drvPath; } @@ -521,10 +521,10 @@ bool NixRepl::processLine(string line) but doing it in a child makes it easier to recover from problems / SIGINT. */ if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { - Derivation drv = readDerivation(drvPath); + auto drv = readDerivation(*state.store, drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputs) - std::cout << format(" %1% -> %2%") % i.first % i.second.path << std::endl; + std::cout << fmt(" %s -> %s\n", i.first, state.store->printStorePath(i.second.path)); } } else if (command == ":i") { runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); diff --git a/src/nix/run.cc b/src/nix/run.cc index 2865f5bbb..7fd9c4c7e 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -109,26 +109,26 @@ struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment auto accessor = store->getFSAccessor(); - std::unordered_set<Path> done; - std::queue<Path> todo; - for (auto & path : outPaths) todo.push(path); + std::unordered_set<StorePath> done; + std::queue<StorePath> todo; + for (auto & path : outPaths) todo.push(path.clone()); setEnviron(); auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":"); while (!todo.empty()) { - Path path = todo.front(); + auto path = todo.front().clone(); todo.pop(); - if (!done.insert(path).second) continue; + if (!done.insert(path.clone()).second) continue; if (true) - unixPath.push_front(path + "/bin"); + unixPath.push_front(store->printStorePath(path) + "/bin"); - auto propPath = path + "/nix-support/propagated-user-env-packages"; + auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; if (accessor->stat(propPath).type == FSAccessor::tRegular) { for (auto & p : tokenizeString<Paths>(readFile(propPath))) - todo.push(p); + todo.push(store->parseStorePath(p)); } } diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 6d5f1603c..6b3d02b6b 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -88,7 +88,7 @@ BuildEnvironment readEnvironment(const Path & path) modified derivation with the same dependencies and nearly the same initial environment variables, that just writes the resulting environment to a file and exits. */ -Path getDerivationEnvironment(ref<Store> store, Derivation drv) +StorePath getDerivationEnvironment(ref<Store> store, Derivation drv) { auto builder = baseNameOf(drv.builder); if (builder != "bash") @@ -120,12 +120,11 @@ Path getDerivationEnvironment(ref<Store> store, Derivation drv) drv.env.erase(output.first); drv.env["out"] = ""; drv.env["outputs"] = "out"; - drv.outputs["out"] = DerivationOutput("", "", ""); - Hash h = hashDerivationModulo(*store, drv); - Path shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs["out"].path = shellOutPath; - drv.env["out"] = shellOutPath; - Path shellDrvPath2 = writeDerivation(store, drv, drvName); + Hash h = hashDerivationModulo(*store, drv, true); + auto shellOutPath = store->makeOutputPath("out", h, drvName); + drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", "")); + drv.env["out"] = store->printStorePath(shellOutPath); + auto shellDrvPath2 = writeDerivation(store, drv, drvName); /* Build the derivation. */ store->buildPaths({shellDrvPath2}); @@ -203,11 +202,11 @@ struct Common : InstallableCommand, MixProfile return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()}; } - Path getShellOutPath(ref<Store> store) + StorePath getShellOutPath(ref<Store> store) { auto path = installable->getStorePath(); - if (path && hasSuffix(*path, "-env")) - return *path; + if (path && hasSuffix(path->to_string(), "-env")) + return path->clone(); else { auto drvs = toDerivations(store, {installable}); @@ -227,7 +226,7 @@ struct Common : InstallableCommand, MixProfile updateProfile(shellOutPath); - return readEnvironment(shellOutPath); + return readEnvironment(store->printStorePath(shellOutPath)); } }; @@ -279,7 +278,7 @@ struct CmdDevShell : Common, MixEnvironment setEnviron(); - auto args = Strings{baseNameOf(shell), "--rcfile", rcFilePath}; + auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; restoreAffinity(); restoreSignals(); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 6065adc4d..0ede7b468 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -46,9 +46,9 @@ struct CmdShowDerivation : InstallablesCommand auto drvPaths = toDerivations(store, installables, true); if (recursive) { - PathSet closure; + StorePathSet closure; store->computeFSClosure(drvPaths, closure); - drvPaths = closure; + drvPaths = std::move(closure); } { @@ -56,17 +56,19 @@ struct CmdShowDerivation : InstallablesCommand JSONObject jsonRoot(std::cout, true); for (auto & drvPath : drvPaths) { - if (!isDerivation(drvPath)) continue; + if (!drvPath.isDerivation()) continue; - auto drvObj(jsonRoot.object(drvPath)); + auto drvPathS = store->printStorePath(drvPath); - auto drv = readDerivation(drvPath); + auto drvObj(jsonRoot.object(drvPathS)); + + auto drv = readDerivation(*store, drvPathS); { auto outputsObj(drvObj.object("outputs")); for (auto & output : drv.outputs) { auto outputObj(outputsObj.object(output.first)); - outputObj.attr("path", output.second.path); + outputObj.attr("path", store->printStorePath(output.second.path)); if (output.second.hash != "") { outputObj.attr("hashAlgo", output.second.hashAlgo); outputObj.attr("hash", output.second.hash); @@ -77,13 +79,13 @@ struct CmdShowDerivation : InstallablesCommand { auto inputsList(drvObj.list("inputSrcs")); for (auto & input : drv.inputSrcs) - inputsList.elem(input); + inputsList.elem(store->printStorePath(input)); } { auto inputDrvsObj(drvObj.object("inputDrvs")); for (auto & input : drv.inputDrvs) { - auto inputList(inputDrvsObj.list(input.first)); + auto inputList(inputDrvsObj.list(store->printStorePath(input.first))); for (auto & outputId : input.second) inputList.elem(outputId); } diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 23bc83ad0..5f07448e0 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -27,7 +27,7 @@ struct CmdCopySigs : StorePathsCommand return "copy path signatures from substituters (like binary caches)"; } - void run(ref<Store> store, Paths storePaths) override + void run(ref<Store> store, StorePaths storePaths) override { if (substituterUris.empty()) throw UsageError("you must specify at least one substituter using '-s'"); @@ -44,18 +44,20 @@ struct CmdCopySigs : StorePathsCommand //logger->setExpected(doneLabel, storePaths.size()); - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const Path & storePathS) { //Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % storePath); checkInterrupt(); + auto storePath = store->parseStorePath(storePathS); + auto info = store->queryPathInfo(storePath); StringSet newSigs; for (auto & store2 : substituters) { try { - auto info2 = store2->queryPathInfo(storePath); + auto info2 = store2->queryPathInfo(info->path); /* Don't import signatures that don't match this binary. */ @@ -80,11 +82,11 @@ struct CmdCopySigs : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, storePath)); + pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); pool.process(); - printInfo(format("imported %d signatures") % added); + printInfo("imported %d signatures", added); } }; @@ -109,7 +111,7 @@ struct CmdSignPaths : StorePathsCommand return "sign the specified paths"; } - void run(ref<Store> store, Paths storePaths) override + void run(ref<Store> store, StorePaths storePaths) override { if (secretKeyFile.empty()) throw UsageError("you must specify a secret key file using '-k'"); @@ -123,7 +125,7 @@ struct CmdSignPaths : StorePathsCommand auto info2(*info); info2.sigs.clear(); - info2.sign(secretKey); + info2.sign(*store, secretKey); assert(!info2.sigs.empty()); if (!info->sigs.count(*info2.sigs.begin())) { @@ -132,7 +134,7 @@ struct CmdSignPaths : StorePathsCommand } } - printInfo(format("added %d signatures") % added); + printInfo("added %d signatures", added); } }; diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index e6c369a7c..87f1f9d1b 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -58,13 +58,9 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand printInfo("upgrading Nix in profile '%s'", profileDir); - Path storePath; - { - Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); - storePath = getLatestNix(store); - } + auto storePath = getLatestNix(store); - auto version = DrvName(storePathToName(storePath)).version; + auto version = DrvName(storePath.name()).version; if (dryRun) { stopProgressBar(); @@ -73,13 +69,13 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand } { - Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath)); + Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", store->printStorePath(storePath))); store->ensurePath(storePath); } { - Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", storePath)); - auto program = storePath + "/bin/nix-env"; + Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", store->printStorePath(storePath))); + auto program = store->printStorePath(storePath) + "/bin/nix-env"; auto s = runProgram(program, false, {"--version"}); if (s.find("Nix") == std::string::npos) throw Error("could not verify that '%s' works", program); @@ -88,9 +84,10 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand stopProgressBar(); { - Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir)); + Activity act(*logger, lvlInfo, actUnknown, + fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir)); runProgram(settings.nixBinDir + "/nix-env", false, - {"--profile", profileDir, "-i", storePath, "--no-sandbox"}); + {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); } printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); @@ -129,15 +126,17 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); - if (!store->isValidPath(userEnv)) + if (!store->isValidPath(store->parseStorePath(userEnv))) throw Error("directory '%s' is not in the Nix store", userEnv); return profileDir; } /* Return the store path of the latest stable Nix. */ - Path getLatestNix(ref<Store> store) + StorePath getLatestNix(ref<Store> store) { + Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); + // FIXME: use nixos.org? auto req = DownloadRequest(storePathsUrl); auto res = getDownloader()->download(req); @@ -148,7 +147,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Bindings & bindings(*state->allocBindings(0)); auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v); - return state->forceString(*v2); + return store->parseStorePath(state->forceString(*v2)); } }; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index fa1414196..9b0658803 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -49,7 +49,7 @@ struct CmdVerify : StorePathsCommand }; } - void run(ref<Store> store, Paths storePaths) override + void run(ref<Store> store, StorePaths storePaths) override { std::vector<ref<Store>> substituters; for (auto & s : substituterUris) @@ -80,7 +80,7 @@ struct CmdVerify : StorePathsCommand MaintainCount<std::atomic<size_t>> mcActive(active); update(); - auto info = store->queryPathInfo(storePath); + auto info = store->queryPathInfo(store->parseStorePath(storePath)); if (!noContents) { @@ -88,7 +88,7 @@ struct CmdVerify : StorePathsCommand if (info->ca == "") hashSink = std::make_unique<HashSink>(info->narHash.type); else - hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(info->path)); + hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(store->printStorePath(info->path))); store->narFromPath(info->path, *hashSink); @@ -96,10 +96,10 @@ struct CmdVerify : StorePathsCommand if (hash.first != info->narHash) { corrupted++; - act2.result(resCorruptedPath, info->path); + act2.result(resCorruptedPath, store->printStorePath(info->path)); printError( - format("path '%s' was modified! expected hash '%s', got '%s'") - % info->path % info->narHash.to_string() % hash.first.to_string()); + "path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(info->path), info->narHash.to_string(), hash.first.to_string()); } } @@ -120,7 +120,7 @@ struct CmdVerify : StorePathsCommand auto doSigs = [&](StringSet sigs) { for (auto sig : sigs) { if (!sigsSeen.insert(sig).second) continue; - if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig)) + if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(*store, publicKeys, sig)) validSigs++; } }; @@ -147,8 +147,8 @@ struct CmdVerify : StorePathsCommand if (!good) { untrusted++; - act2.result(resUntrustedPath, info->path); - printError(format("path '%s' is untrusted") % info->path); + act2.result(resUntrustedPath, store->printStorePath(info->path)); + printError("path '%s' is untrusted", store->printStorePath(info->path)); } } @@ -164,7 +164,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, storePath)); + pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); pool.process(); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 3d13a77e4..c539adb7f 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -73,9 +73,9 @@ struct CmdWhyDepends : SourceExprCommand auto packagePath = toStorePath(store, Build, package); auto dependency = parseInstallable(store, _dependency); auto dependencyPath = toStorePath(store, NoBuild, dependency); - auto dependencyPathHash = storePathToHash(dependencyPath); + auto dependencyPathHash = storePathToHash(store->printStorePath(dependencyPath)); - PathSet closure; + StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); if (!closure.count(dependencyPath)) { @@ -91,28 +91,28 @@ struct CmdWhyDepends : SourceExprCommand struct Node { - Path path; - PathSet refs; - PathSet rrefs; + StorePath path; + StorePathSet refs; + StorePathSet rrefs; size_t dist = inf; Node * prev = nullptr; bool queued = false; bool visited = false; }; - std::map<Path, Node> graph; + std::map<StorePath, Node> graph; for (auto & path : closure) - graph.emplace(path, Node{path, store->queryPathInfo(path)->references}); + graph.emplace(path.clone(), Node{path.clone(), cloneStorePathSet(store->queryPathInfo(path)->references)}); // Transpose the graph. for (auto & node : graph) for (auto & ref : node.second.refs) - graph[ref].rrefs.insert(node.first); + graph.find(ref)->second.rrefs.insert(node.first.clone()); /* Run Dijkstra's shortest path algorithm to get the distance of every path in the closure to 'dependency'. */ - graph[dependencyPath].dist = 0; + graph[dependencyPath.clone()].dist = 0; std::priority_queue<Node *> queue; @@ -151,12 +151,14 @@ struct CmdWhyDepends : SourceExprCommand struct BailOut { }; printNode = [&](Node & node, const string & firstPad, const string & tailPad) { + auto pathS = store->printStorePath(node.path); + assert(node.dist != inf); std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "=> " : "", - node.path); + pathS); if (node.path == dependencyPath && !all && packagePath != dependencyPath) @@ -175,7 +177,7 @@ struct CmdWhyDepends : SourceExprCommand auto & node2 = graph.at(ref); if (node2.dist == inf) continue; refs.emplace(node2.dist, &node2); - hashes.insert(storePathToHash(node2.path)); + hashes.insert(storePathToHash(store->printStorePath(node2.path))); } /* For each reference, find the files and symlinks that @@ -187,7 +189,7 @@ struct CmdWhyDepends : SourceExprCommand visitPath = [&](const Path & p) { auto st = accessor->stat(p); - auto p2 = p == node.path ? "/" : std::string(p, node.path.size() + 1); + auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); auto getColour = [&](const std::string & hash) { return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; @@ -231,11 +233,11 @@ struct CmdWhyDepends : SourceExprCommand // FIXME: should use scanForReferences(). - visitPath(node.path); + visitPath(pathS); RunPager pager; for (auto & ref : refs) { - auto hash = storePathToHash(ref.second->path); + auto hash = storePathToHash(store->printStorePath(ref.second->path)); bool last = all ? ref == *refs.rbegin() : true; diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index f9db16268..bf65f7905 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -6,6 +6,6 @@ resolve-system-dependencies_DIR := $(d) resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix -resolve-system-dependencies_LIBS := libstore libmain libutil +resolve-system-dependencies_LIBS := libstore libmain libutil libnixrust resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc index 45e8b6796..8f0c99c84 100644 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc @@ -179,8 +179,8 @@ int main(int argc, char ** argv) if (std::string(argv[1]) == "--test") impurePaths.insert(argv[2]); else { - auto drv = store->derivationFromPath(Path(argv[1])); - impurePaths = tokenizeString<StringSet>(get(drv.env, "__impureHostDeps")); + auto drv = store->derivationFromPath(store->parseStorePath(argv[1])); + impurePaths = tokenizeString<StringSet>(get(drv.env, "__impureHostDeps").value_or("")); impurePaths.insert("/usr/lib/libSystem.dylib"); } |