diff options
37 files changed, 546 insertions, 258 deletions
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 54ad1680c..9cb078660 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -110,7 +110,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); @@ -294,7 +294,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 73817dbdd..f7b77fccb 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1193,7 +1193,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); - auto outPath = state.store->makeFixedOutputPath(method, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput::CAFixed { @@ -1944,7 +1950,13 @@ static void addPath( std::optional<StorePath> expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { StorePath dstPath = settings.readOnlyMode diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 42c98e312..c12872084 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -232,10 +232,15 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, // early exit if pinned and already in the store if (expectedHash && expectedHash->type == htSHA256) { - auto expectedPath = - unpack - ? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {}) - : state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {}); + auto expectedPath = state.store->makeFixedOutputPath( + name, + FixedOutputInfo { + { + .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .hash = *expectedHash, + }, + {} + }); if (state.store->isValidPath(expectedPath)) { state.allowAndSetStorePathString(expectedPath, v); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 6957d2da4..2a1529a60 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -210,7 +210,13 @@ StorePath Input::computeStorePath(Store & store) const auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); + return store.makeFixedOutputPath(getName(), FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + }, + {}, + }); } std::string Input::getType() const diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index dde0ad761..4be923148 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -70,15 +70,21 @@ DownloadFileResult downloadFile( dumpString(res.data, sink); auto hash = hashString(htSHA256, res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + *store, + { + .name = name, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + {}, + }, + }, hashString(htSHA256, sink.s), }; info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; - auto source = StringSource(sink.s); + auto source = StringSource { sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); } diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9226c4e19..9c022f9e9 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -184,8 +184,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon( reads, but typically they'll already be cached. */ for (auto & ref : info.references) try { - if (ref != info.path) - queryPathInfo(ref); + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); @@ -314,11 +313,23 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name, references), + *this, + { + .name = std::string { name }, + .info = FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + { + .references = references, + .hasSelfReference = false, + }, + }, + }, nar.first, }; info.narSize = nar.second; - info.references = references; return info; })->path; } @@ -411,15 +422,23 @@ StorePath BinaryCacheStore::addToStore( }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name, references), + *this, + { + .name = std::string { name }, + .info = FixedOutputInfo { + { + .method = method, + .hash = h, + }, + { + .references = references, + .hasSelfReference = false, + }, + }, + }, nar.first, }; info.narSize = nar.second; - info.references = references; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; return info; })->path; } @@ -431,7 +450,7 @@ StorePath BinaryCacheStore::addTextToStore( RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); + auto path = makeTextPath(name, TextInfo { textHash, references }); if (!repair && isValidPath(path)) return path; @@ -440,10 +459,18 @@ StorePath BinaryCacheStore::addTextToStore( dumpString(s, sink); StringSource source(sink.s); return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; + ValidPathInfo info { + *this, + { + .name = std::string { name }, + .info = TextInfo { + { .hash = textHash }, + references, + }, + }, + nar.first, + }; info.narSize = nar.second; - info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4c91fa4fb..1467c18af 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2256,27 +2256,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } }; - auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> { + auto rewriteRefs = [&]() -> PathReferences<StorePath> { /* In the CA case, we need the rewritten refs to calculate the final path, therefore we look for a *non-rewritten self-reference, and use a bool rather try to solve the computationally intractable fixed point. */ - std::pair<bool, StorePathSet> res { - false, - {}, + PathReferences<StorePath> res { + .hasSelfReference = false, }; for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; if (r == scratchPath) - res.first = true; + res.hasSelfReference = true; else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); + res.references.insert(r); else { std::string newRef = outputRewrites.at(origHash); newRef += '-'; newRef += name; - res.second.insert(StorePath { newRef }); + res.references.insert(StorePath { newRef }); } } return res; @@ -2305,20 +2304,26 @@ DrvOutputs LocalDerivationGoal::registerOutputs() break; } auto got = caSink.finish().first; - auto refs = rewriteRefs(); - - auto finalPath = worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first); - if (scratchPath != finalPath) { + ValidPathInfo newInfo0 { + worker.store, + { + .name = outputPathName(drv->name, outputName), + .info = FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, + }, + rewriteRefs(), + }, + }, + Hash::dummy, + }; + if (scratchPath != newInfo0.path) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { StringSink sink; dumpPath(actualPath, sink); - RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); + RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); rsink2(sink.s); rsink2.flush(); }); @@ -2329,19 +2334,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() } HashResult narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { - finalPath, - narHashAndSize.first, - }; - + newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); assert(newInfo0.ca); return newInfo0; @@ -2362,10 +2356,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); + static_cast<PathReferences<StorePath> &>(newInfo0) = rewriteRefs(); return newInfo0; }, @@ -2673,12 +2664,12 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; - for (auto & ref : i->second.references) + for (auto & ref : i->second.referencesPossiblyToSelf()) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) pathsLeft.push(ref); } } @@ -2717,7 +2708,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo auto used = recursive ? getClosure(info.path).first - : info.references; + : info.referencesPossiblyToSelf(); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index ca5218627..2aee09f21 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -95,7 +95,10 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA({ + .name = std::string { storePath.name() }, + .info = caWithoutRefs(*ca), + }); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { @@ -125,7 +128,7 @@ void PathSubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { + if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) { auto info2 = std::make_shared<ValidPathInfo>(*info); info2->path = storePath; info = info2; @@ -163,8 +166,7 @@ void PathSubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -186,8 +188,7 @@ void PathSubstitutionGoal::referencesValid() } for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index cf32ccdc4..2e6c435ce 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,16 +9,16 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) + +std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: return ""; case FileIngestionMethod::Recursive: return "r:"; - default: - throw Error("impossible, caught both cases"); } + assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) @@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash & th) { - return "text:" + th.hash.to_string(Base32, true); + return "text:" + + th.hash.to_string(Base32, true); }, [](FixedOutputHash & fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } @@ -144,7 +147,18 @@ Hash getContentAddressHash(const ContentAddress & ca) }, [](const FixedOutputHash & fsh) { return fsh.hash; - } + }, + }, ca); +} + +ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { + return std::visit(overloaded { + [&](const TextHash & h) -> ContentAddressWithReferences { + return TextInfo { h, {}}; + }, + [&](const FixedOutputHash & h) -> ContentAddressWithReferences { + return FixedOutputInfo { h, {}}; + }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..a275800f9 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -2,16 +2,56 @@ #include <variant> #include "hash.hh" +#include "path.hh" +#include "comparator.hh" namespace nix { +/* + * Content addressing method + */ + +/* We only have one way to hash text with references, so this is a single-value + type, mainly useful with std::variant. +*/ +struct TextHashMethod : std::monostate { }; + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; + +/* Compute the prefix to the hash algorithm which indicates how the files were + ingested. */ +std::string makeFileIngestionPrefix(FileIngestionMethod m); + + +/* Just the type of a content address. Combine with the hash itself, and we + have a `ContentAddress` as defined below. Combine that, in turn, with info + on references, and we have `ContentAddressWithReferences`, as defined + further below. */ +typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > ContentAddressMethod; + +ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + +std::string renderContentAddressMethod(ContentAddressMethod caMethod); + +/* + * Mini content address + */ + struct TextHash { Hash hash; + + GENERATE_CMP(TextHash, me->hash); }; /// Pair of a hash, and how the file system was ingested @@ -19,6 +59,8 @@ struct FixedOutputHash { FileIngestionMethod method; Hash hash; std::string printMethodAlgo() const; + + GENERATE_CMP(FixedOutputHash, me->method, me->hash); }; /* @@ -37,14 +79,6 @@ typedef std::variant< FixedOutputHash // for path computed by makeFixedOutputPath > ContentAddress; -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); - -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional<ContentAddress> ca); @@ -55,23 +89,87 @@ std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); + /* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; + * References set + */ + +template<typename Ref> +struct PathReferences +{ + std::set<Ref> references; + bool hasSelfReference = false; + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs); + + GENERATE_CMP(PathReferences<Ref>, me->references, me->hasSelfReference); +}; + +template<typename Ref> +StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet refs { references }; + if (hasSelfReference) + refs.insert(self); + return refs; +} + +template<typename Ref> +void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} + +template<typename Ref> +void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); + + references = refs; +} + +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ + +// This matches the additional info that we need for makeTextPath +struct TextInfo : TextHash { + // References for the paths, self references disallowed + StorePathSet references; + + GENERATE_CMP(TextInfo, *(const TextHash *)me, me->references); +}; + +struct FixedOutputInfo : FixedOutputHash { + // References for the paths + PathReferences<StorePath> references; + + GENERATE_CMP(FixedOutputInfo, *(const FixedOutputHash *)me, me->references); }; typedef std::variant< - TextHashMethod, - FixedOutputHashMethod - > ContentAddressMethod; + TextInfo, + FixedOutputInfo +> ContentAddressWithReferences; -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); +ContentAddressWithReferences caWithoutRefs(const ContentAddress &); -std::string renderContentAddressMethod(ContentAddressMethod caMethod); +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + GENERATE_CMP(StorePathDescriptor, me->name, me->info); +}; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index de69b50ee..a59fbae3c 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -335,7 +335,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->references) + for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -757,7 +757,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references); + worker_proto::write(*store, to, i->second.referencesPossiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -780,7 +780,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references); + worker_proto::write(*store, to, i.second.referencesPossiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; @@ -862,7 +862,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(worker_proto::read(*store, from, Phantom<StorePathSet> {})); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1c695de82..08d5e5c5e 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -35,8 +35,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + outputPathName(drvName, outputName), + { hash, {} }); } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 24a0ae773..5d9107ab2 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -5,6 +5,7 @@ #include "realisation.hh" #include <optional> +#include <variant> #include <nlohmann/json_fwd.hpp> diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 9875da909..4adf51573 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->references); + worker_proto::write(*this, teeSink, info->referencesPossiblyToSelf()); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -80,7 +80,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = references; + info.setReferencesPossiblyToSelf(std::move(references)); info.narSize = saved.s.size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index dd34b19c6..22d860667 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -134,11 +134,10 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor /* Hash will be set below. FIXME construct ValidPathInfo at end. */ auto info = std::make_shared<ValidPathInfo>(path, Hash::dummy); - PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -172,7 +171,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize @@ -201,7 +200,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d77fff963..40133cd0c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -710,7 +710,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat envHasRightPath(doia.path, i.first); }, [&](const DerivationOutput::CAFixed & dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} }); envHasRightPath(path, i.first); }, [&](const DerivationOutput::CAFloating &) { @@ -910,7 +910,8 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + info->insertReferencePossiblyToSelf( + parseStorePath(useQueryReferences.getStr(0))); return info; } @@ -1108,7 +1109,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // Recompute store path so that we can use a different store root. if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + subPath = makeFixedOutputPathFromCA({ + .name = std::string { path.first.name() }, + .info = caWithoutRefs(*path.second), + }); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1125,10 +1129,12 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->deriver, info->references, + info->hasSelfReference, + info->deriver, narInfo ? narInfo->fileSize : 0, - info->narSize}); + info->narSize, + }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { @@ -1173,7 +1179,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) + for (auto & j : i.referencesPossiblyToSelf()) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1383,7 +1389,21 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name, references); + auto desc = StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + { + .references = references, + .hasSelfReference = false, + }, + }, + }; + + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); @@ -1403,7 +1423,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name autoGC(); if (inMemory) { - StringSource dumpSource { dump }; + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource); @@ -1428,10 +1448,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name optimisePath(realPath, repair); - ValidPathInfo info { dstPath, narHash.first }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; - info.references = references; - info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1448,7 +1466,10 @@ StorePath LocalStore::addTextToStore( const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 64d172918..9655a0555 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -27,18 +27,15 @@ std::map<StorePath, StorePath> makeContentAddressed( StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + PathReferences<StorePath> refs; + refs.hasSelfReference = oldInfo->hasSelfReference; for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); - references.insert(std::move(replacement)); + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) { + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + refs.references.insert(std::move(replacement)); } } @@ -49,24 +46,30 @@ std::map<StorePath, StorePath> makeContentAddressed( auto narModuloHash = hashModuloSink.finish().first; - auto dstPath = dstStore.makeFixedOutputPath( - FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); + ValidPathInfo info { + dstStore, + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, + }, + std::move(refs), + }, + }, + Hash::dummy, + }; - printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); + printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path)); StringSink sink2; - RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); + RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2); rsink2(sink.s); rsink2.flush(); - ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); + info.narHash = hashString(htSHA256, sink2.s); info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }; StringSource source(sink2.s); dstStore.addToStore(info, source); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 2bbd7aa70..2d5b8d4b9 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -41,8 +41,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; auto info = fut.get(); for (auto& ref : info->references) - if (ref != path) - res.insert(ref); + res.insert(ref); if (includeOutputs && path.isDerivation()) for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 9dd81ddfb..1bb9201e9 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -233,7 +233,7 @@ public: narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath(r)); + narInfo->insertReferencePossiblyToSelf(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 2d75e7a82..2d53a449c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -63,7 +63,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString<Strings>(value, " "); if (!references.empty()) throw corrupt(); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 39ced76e5..fd37b85db 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo std::string system; NarInfo() = delete; + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) + { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index fda55b2b6..003685604 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -14,7 +14,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = worker_proto::read(store, source, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(worker_proto::read(store, source, Phantom<StorePathSet> {})); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -34,7 +34,7 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, references); + worker_proto::write(store, sink, referencesPossiblyToSelf()); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b4b54e593..89886873a 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -14,24 +14,20 @@ namespace nix { class Store; -struct SubstitutablePathInfo +struct SubstitutablePathInfo : PathReferences<StorePath> { std::optional<StorePath> deriver; - StorePathSet references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; - -struct ValidPathInfo +struct ValidPathInfo : PathReferences<StorePath> { StorePath path; std::optional<StorePath> deriver; - // TODO document this Hash narHash; - StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -65,6 +61,7 @@ struct ValidPathInfo return path == i.path && narHash == i.narHash + && hasSelfReference == i.hasSelfReference && references == i.references; } @@ -78,6 +75,8 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); + std::optional<StorePathDescriptor> fullStorePathDescriptorOpt() const; + /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -104,6 +103,9 @@ struct ValidPathInfo ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(const Store & store, + StorePathDescriptor && ca, Hash narHash); + virtual ~ValidPathInfo() { } static ValidPathInfo read(Source & source, const Store & store, unsigned int format); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 77fd0f8dc..6771f0ba5 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,7 @@ #pragma once -#include "content-address.hh" +#include <string_view> + #include "types.hh" namespace nix { @@ -66,8 +67,6 @@ typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; typedef std::map<std::string, StorePath> OutputPathMap; -typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 9070a6ee2..fa53fc487 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,5 +1,8 @@ #pragma once +#include <variant> + +#include "hash.hh" #include "path.hh" #include <nlohmann/json_fwd.hpp> #include "comparator.hh" diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 347e32094..1666fa31b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -402,7 +402,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -421,11 +421,12 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto path = parseStorePath(readString(conn->from)); + SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -635,7 +636,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + worker_proto::write(*this, sink, info.referencesPossiblyToSelf()); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -646,7 +647,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); - assert(importedPaths.size() <= 1); + assert(importedPaths.empty() == 0); // doesn't include possible self reference } else { @@ -654,7 +655,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 59937be4d..4362177f9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -8,6 +8,7 @@ #include "thread-pool.hh" #include "json.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" #include "remote-store.hh" @@ -160,63 +161,61 @@ StorePath Store::makeOutputPath(std::string_view id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ static std::string makeType( const Store & store, std::string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const PathReferences<StorePath> & references) { - for (auto & i : references) { + for (auto & i : references.references) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.hasSelfReference) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(references.empty()); + assert(info.references.references.size() == 0); + assert(!info.references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", PathReferences<StorePath> { info.references }), + info.hash, + name); +} + + +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { - [&](const TextHash & th) { - return makeTextPath(name, th.hash, references); + [&](const TextInfo & ti) { + return makeTextPath(desc.name, ti); }, - [&](const FixedOutputHash & fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](const FixedOutputInfo & foi) { + return makeFixedOutputPath(desc.name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, desc.info); } @@ -226,7 +225,14 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + FixedOutputInfo caInfo { + { + .method = method, + .hash = h, + }, + {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } @@ -235,7 +241,10 @@ StorePath Store::computeStorePathForText( std::string_view s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } @@ -345,11 +354,20 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }, narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { @@ -661,13 +679,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -679,7 +697,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -897,7 +915,8 @@ void copyStorePath( // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref<ValidPathInfo>(*info); - info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore.makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -1039,7 +1058,8 @@ std::map<StorePath, StorePath> copyPaths( auto info = srcStore.queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore.makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -1069,8 +1089,9 @@ std::map<StorePath, StorePath> copyPaths( auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) + storePathForDst = dstStore.makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); + if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) debug("replaced path '%s' to '%s' for substituter '%s'", @@ -1159,7 +1180,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre if (!n) throw Error("number expected"); while ((*n)--) { getline(str, s); - info.references.insert(store.parseStorePath(s)); + info.insertReferencePossiblyToSelf(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional<ValidPathInfo>(std::move(info)); @@ -1182,6 +1203,20 @@ std::string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return PathReferences<StorePath>::referencesPossiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return PathReferences<StorePath>::insertReferencePossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return PathReferences<StorePath>::setReferencesPossiblyToSelf(path, std::move(refs)); +} std::string ValidPathInfo::fingerprint(const Store & store) const { @@ -1192,7 +1227,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); } @@ -1201,24 +1236,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } +std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const +{ + if (! ca) + return std::nullopt; + + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](const TextHash & th) { + TextInfo info { th }; + assert(!hasSelfReference); + info.references = references; + return ContentAddressWithReferences { info }; + }, + [&](const FixedOutputHash & foh) { + FixedOutputInfo info { foh }; + info.references = static_cast<PathReferences<StorePath>>(*this); + return ContentAddressWithReferences { info }; + }, + }, *ca), + }; +} + bool ValidPathInfo::isContentAddressed(const Store & store) const { - if (! ca) return false; + auto fullCaOpt = fullStorePathDescriptorOpt(); - auto caPath = std::visit(overloaded { - [&](const TextHash & th) { - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](const FixedOutputHash & fsh) { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); - } - }, *ca); + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -1250,12 +1298,32 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : references) + for (auto & r : referencesPossiblyToSelf()) refs.push_back(std::string(r.to_string())); return refs; } +ValidPathInfo::ValidPathInfo( + const Store & store, + StorePathDescriptor && info, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(info)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](const TextInfo & ti) { + this->references = ti.references; + this->ca = TextHash { std::move(ti) }; + }, + [this](const FixedOutputInfo & foi) { + *(static_cast<PathReferences<StorePath> *>(this)) = foi.references; + this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + }, + }, std::move(info.info)); +} + + Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0c8a4db56..5ea606886 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -83,6 +83,8 @@ enum BuildMode { bmNormal, bmRepair, bmCheck }; struct BuildResult; +typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; + struct StoreConfig : public Config { using Config::Config; @@ -204,17 +206,11 @@ public: StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 348018f57..7a5316896 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -200,9 +200,8 @@ public: template<typename... Args> SysError(const Args & ... args) - : Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb..36d774dca 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -56,7 +56,7 @@ void printDotGraph(ref<Store> store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 425d61e53..d2eebca7a 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref<Store> store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->references) { + for (auto & p : info->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 153b84137..a97a82379 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -201,10 +201,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -215,7 +215,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string hash = *i++; std::string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + }, + {}, + }))); } @@ -257,7 +263,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->references); + auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -338,7 +344,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) + for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) paths.insert(p); } else if (query == qReferrers) { @@ -852,7 +858,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->references); + worker_proto::write(*store, out, info->referencesPossiblyToSelf()); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -950,7 +956,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(worker_proto::read(*store, in, Phantom<StorePathSet> {})); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 5168413d2..c2494dc9f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -42,14 +42,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand } ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + *store, + StorePathDescriptor { + .name = *namePart, + .info = FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + {}, + }, + }, narHash, }; info.narSize = sink.s.size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource(sink.s); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index f2dd44ba4..35faa7a88 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -67,7 +67,13 @@ std::tuple<StorePath, Hash> prefetchFile( the store. */ if (expectedHash) { hashType = expectedHash->type; - storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { + { + .method = ingestionMethod, + .hash = *expectedHash, + }, + {}, + }); if (store->isValidPath(*storePath)) hash = expectedHash; else diff --git a/src/nix/profile.cc b/src/nix/profile.cc index b151e48d6..3a32a7b55 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -192,12 +192,21 @@ struct ProfileManifest auto narHash = hashString(htSHA256, sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + StorePathDescriptor { + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + { references }, + }, + }, narHash, }; info.references = std::move(references); info.narSize = sink.s.size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; StringSource source(sink.s); store->addToStore(info, source); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 3d659d6d2..a08314a25 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -63,7 +63,8 @@ struct CmdCopySigs : StorePathsCommand binary. */ if (info->narHash != info2->narHash || info->narSize != info2->narSize || - info->references != info2->references) + info->references != info2->references || + info->hasSelfReference != info2->hasSelfReference) continue; for (auto & sig : info2->sigs) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index e92df1303..d132768ec 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -81,14 +81,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount<std::atomic<size_t>> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -173,7 +173,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); |