diff options
38 files changed, 526 insertions, 252 deletions
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index edbf12f7c..28f62dc57 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -110,7 +110,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); @@ -294,7 +294,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0422102a9..4626c4327 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1128,7 +1128,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + { + .method = ingestionMethod, + .hash = h, + }, + {}, + }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputCAFixed { @@ -1879,7 +1885,13 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional<StorePath> expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..c5f327bd8 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,5 +10,5 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); - + } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e158d914b..7a88b4acc 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -210,7 +210,13 @@ StorePath Input::computeStorePath(Store & store) const auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for mutable input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); + return store.makeFixedOutputPath(getName(), FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + }, + {}, + }); } std::string Input::getType() const diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 031ccc5f7..efd34932c 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -70,14 +70,20 @@ DownloadFileResult downloadFile( dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + *store, + { + .name = name, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + {}, + }, + }, hashString(htSHA256, *sink.s), }; info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 3a6be541f..319b06269 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -184,8 +184,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon( reads, but typically they'll already be cached. */ for (auto & ref : info.references) try { - if (ref != info.path) - queryPathInfo(ref); + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); @@ -314,7 +313,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; @@ -404,14 +413,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; return info; })->path; } @@ -420,7 +435,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s const StorePathSet & references, RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); + auto path = makeTextPath(name, TextInfo { textHash, references }); if (!repair && isValidPath(path)) return path; @@ -429,10 +444,19 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s dumpString(s, sink); auto source = StringSource { *sink.s }; return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; + ValidPathInfo info { + *this, + { + .name = name, + .info = TextInfo { + { .hash = textHash }, + references, + }, + }, + nar.first, + }; info.narSize = nar.second; info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 3aa9f12fe..d104d3148 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2220,27 +2220,26 @@ void LocalDerivationGoal::registerOutputs() } }; - auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> { + auto rewriteRefs = [&]() -> PathReferences<StorePath> { /* In the CA case, we need the rewritten refs to calculate the final path, therefore we look for a *non-rewritten self-reference, and use a bool rather try to solve the computationally intractable fixed point. */ - std::pair<bool, StorePathSet> res { - false, - {}, + PathReferences<StorePath> res { + .hasSelfReference = false, }; for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; if (r == scratchPath) - res.first = true; + res.hasSelfReference = true; else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); + res.references.insert(r); else { std::string newRef = outputRewrites.at(origHash); newRef += '-'; newRef += name; - res.second.insert(StorePath { newRef }); + res.references.insert(StorePath { newRef }); } } return res; @@ -2269,20 +2268,26 @@ void LocalDerivationGoal::registerOutputs() break; } auto got = caSink.finish().first; - auto refs = rewriteRefs(); - - auto finalPath = worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first); - if (scratchPath != finalPath) { + ValidPathInfo newInfo0 { + worker.store, + { + .name = outputPathName(drv->name, outputName), + .info = FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, + }, + rewriteRefs(), + }, + }, + Hash::dummy, + }; + if (scratchPath != newInfo0.path) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { StringSink sink; dumpPath(actualPath, sink); - RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); + RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); rsink2(*sink.s); rsink2.flush(); }); @@ -2293,19 +2298,8 @@ void LocalDerivationGoal::registerOutputs() } HashResult narHashAndSize = hashPath(htSHA256, actualPath); - ValidPathInfo newInfo0 { - finalPath, - narHashAndSize.first, - }; - + newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); assert(newInfo0.ca); return newInfo0; @@ -2325,10 +2319,7 @@ void LocalDerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); + static_cast<PathReferences<StorePath> &>(newInfo0) = rewriteRefs(); return newInfo0; }, [&](DerivationOutputCAFixed dof) { @@ -2618,12 +2609,12 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; - for (auto & ref : i->second.references) + for (auto & ref : i->second.referencesPossiblyToSelf()) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) pathsLeft.push(ref); } } @@ -2662,7 +2653,7 @@ void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & out auto used = recursive ? getClosure(info.path).first - : info.references; + : info.referencesPossiblyToSelf(); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 29a8cfb87..cd00e41f9 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -79,7 +79,10 @@ void PathSubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA({ + .name = std::string { storePath.name() }, + .info = caWithoutRefs(*ca), + }); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { @@ -109,7 +112,7 @@ void PathSubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { + if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) { auto info2 = std::make_shared<ValidPathInfo>(*info); info2->path = storePath; info = info2; @@ -147,8 +150,7 @@ void PathSubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + addWaitee(worker.makePathSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -168,8 +170,7 @@ void PathSubstitutionGoal::referencesValid() } for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 90a3ad1f5..d68c60f4f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,6 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } + std::string makeFileIngestionPrefix(const FileIngestionMethod m) { switch (m) { @@ -16,9 +17,8 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; - default: - throw Error("impossible, caught both cases"); } + assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) @@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { - return "text:" + th.hash.to_string(Base32, true); + return "text:" + + th.hash.to_string(Base32, true); }, [](FixedOutputHash fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } @@ -142,7 +145,18 @@ Hash getContentAddressHash(const ContentAddress & ca) }, [](FixedOutputHash fsh) { return fsh.hash; - } + }, + }, ca); +} + +ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { + return std::visit(overloaded { + [&](TextHash h) -> ContentAddressWithReferences { + return TextInfo { h, {}}; + }, + [&](FixedOutputHash h) -> ContentAddressWithReferences { + return FixedOutputInfo { h, {}}; + }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..126244ab5 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -2,14 +2,48 @@ #include <variant> #include "hash.hh" +#include "path.hh" namespace nix { +/* + * Content addressing method + */ + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; +/* + We only have one way to hash text with references, so this is single-value + type is only useful in std::variant. +*/ +struct TextHashMethod { }; + +struct FixedOutputHashMethod { + FileIngestionMethod fileIngestionMethod; + HashType hashType; +}; + +/* Compute the prefix to the hash algorithm which indicates how the files were + ingested. */ +std::string makeFileIngestionPrefix(const FileIngestionMethod m); + + +typedef std::variant< + TextHashMethod, + FixedOutputHashMethod + > ContentAddressMethod; + +ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); + +std::string renderContentAddressMethod(ContentAddressMethod caMethod); + +/* + * Mini content address + */ + struct TextHash { Hash hash; }; @@ -37,14 +71,6 @@ typedef std::variant< FixedOutputHash // for path computed by makeFixedOutputPath > ContentAddress; -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); - -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional<ContentAddress> ca); @@ -56,22 +82,95 @@ std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt); Hash getContentAddressHash(const ContentAddress & ca); /* - We only have one way to hash text with references, so this is single-value - type is only useful in std::variant. -*/ -struct TextHashMethod { }; -struct FixedOutputHashMethod { - FileIngestionMethod fileIngestionMethod; - HashType hashType; + * References set + */ + +template<typename Ref> +struct PathReferences +{ + std::set<Ref> references; + bool hasSelfReference = false; + + bool operator == (const PathReferences<Ref> & other) const + { + return references == other.references + && hasSelfReference == other.hasSelfReference; + } + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs); }; -typedef std::variant< - TextHashMethod, - FixedOutputHashMethod - > ContentAddressMethod; +template<typename Ref> +StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet refs { references }; + if (hasSelfReference) + refs.insert(self); + return refs; +} -ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); +template<typename Ref> +void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} -std::string renderContentAddressMethod(ContentAddressMethod caMethod); +template<typename Ref> +void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); + + references = refs; +} + +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ + +// This matches the additional info that we need for makeTextPath +struct TextInfo : TextHash { + // References for the paths, self references disallowed + StorePathSet references; +}; + +struct FixedOutputInfo : FixedOutputHash { + // References for the paths + PathReferences<StorePath> references; +}; + +typedef std::variant< + TextInfo, + FixedOutputInfo +> ContentAddressWithReferences; + +ContentAddressWithReferences caWithoutRefs(const ContentAddress &); + +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + bool operator == (const StorePathDescriptor & other) const + { + return name == other.name; + // FIXME second field + } + + bool operator < (const StorePathDescriptor & other) const + { + return name < other.name; + // FIXME second field + } +}; } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 487416a13..5bc42e1f7 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -331,7 +331,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->references) + for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -722,7 +722,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references); + worker_proto::write(*store, to, i->second.referencesPossiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -745,7 +745,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references); + worker_proto::write(*store, to, i.second.referencesPossiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; @@ -827,7 +827,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(worker_proto::read(*store, from, Phantom<StorePathSet> {})); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 899475860..717bd56f6 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -30,8 +30,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + outputPathName(drvName, outputName), + { hash, {} }); } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 9d6ace069..ef7c5326c 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -5,6 +5,7 @@ #include "realisation.hh" #include <optional> +#include <variant> #include <nlohmann/json_fwd.hpp> diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 02c839520..55585e977 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->references); + worker_proto::write(*this, teeSink, info->referencesPossiblyToSelf()); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -80,7 +80,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = references; + info.setReferencesPossiblyToSelf(std::move(references)); info.narSize = saved.s->size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index c660d7f5b..571719043 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -133,11 +133,10 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor /* Hash will be set below. FIXME construct ValidPathInfo at end. */ auto info = std::make_shared<ValidPathInfo>(path, Hash::dummy); - PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -171,7 +170,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize @@ -200,7 +199,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 634e9eb8b..768f72209 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -694,7 +694,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat envHasRightPath(doia.path, i.first); }, [&](DerivationOutputCAFixed dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} }); envHasRightPath(path, i.first); }, [&](DerivationOutputCAFloating _) { @@ -887,7 +887,8 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + info->insertReferencePossiblyToSelf( + parseStorePath(useQueryReferences.getStr(0))); return info; } @@ -1085,7 +1086,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // Recompute store path so that we can use a different store root. if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + subPath = makeFixedOutputPathFromCA({ + .name = std::string { path.first.name() }, + .info = caWithoutRefs(*path.second), + }); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1102,10 +1106,12 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->deriver, info->references, + info->hasSelfReference, + info->deriver, narInfo ? narInfo->fileSize : 0, - info->narSize}); + info->narSize, + }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { @@ -1150,7 +1156,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & [_, i] : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) + for (auto & j : i.referencesPossiblyToSelf()) state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1358,7 +1364,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name); + auto desc = StorePathDescriptor { + name, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }; + + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); @@ -1378,7 +1395,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, autoGC(); if (inMemory) { - StringSource dumpSource { dump }; + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource); @@ -1403,9 +1420,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, optimisePath(realPath); - ValidPathInfo info { dstPath, narHash.first }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; - info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1420,7 +1436,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index b4929b445..e894b63db 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -40,8 +40,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths, StorePathSet res; auto info = fut.get(); for (auto& ref : info->references) - if (ref != path) - res.insert(ref); + res.insert(ref); if (includeOutputs && path.isDerivation()) for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 9dd81ddfb..1bb9201e9 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -233,7 +233,7 @@ public: narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath(r)); + narInfo->insertReferencePossiblyToSelf(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 49079388a..4ea30bf47 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -63,7 +63,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString<Strings>(value, " "); if (!references.empty()) throw corrupt(); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 39ced76e5..fd37b85db 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo std::string system; NarInfo() = delete; + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) + { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index fda55b2b6..003685604 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -14,7 +14,7 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = worker_proto::read(store, source, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(worker_proto::read(store, source, Phantom<StorePathSet> {})); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -34,7 +34,7 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, references); + worker_proto::write(store, sink, referencesPossiblyToSelf()); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b4b54e593..89886873a 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -14,24 +14,20 @@ namespace nix { class Store; -struct SubstitutablePathInfo +struct SubstitutablePathInfo : PathReferences<StorePath> { std::optional<StorePath> deriver; - StorePathSet references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; - -struct ValidPathInfo +struct ValidPathInfo : PathReferences<StorePath> { StorePath path; std::optional<StorePath> deriver; - // TODO document this Hash narHash; - StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -65,6 +61,7 @@ struct ValidPathInfo return path == i.path && narHash == i.narHash + && hasSelfReference == i.hasSelfReference && references == i.references; } @@ -78,6 +75,8 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); + std::optional<StorePathDescriptor> fullStorePathDescriptorOpt() const; + /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -104,6 +103,9 @@ struct ValidPathInfo ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(const Store & store, + StorePathDescriptor && ca, Hash narHash); + virtual ~ValidPathInfo() { } static ValidPathInfo read(Source & source, const Store & store, unsigned int format); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 06ba0663b..a152fe8ee 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,7 @@ #pragma once -#include "content-address.hh" +#include <string_view> + #include "types.hh" namespace nix { @@ -64,8 +65,6 @@ typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; typedef std::map<string, StorePath> OutputPathMap; -typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 9070a6ee2..fa53fc487 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -1,5 +1,8 @@ #pragma once +#include <variant> + +#include "hash.hh" #include "path.hh" #include <nlohmann/json_fwd.hpp> #include "comparator.hh" diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b57a4cb05..0c25bfa59 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -366,7 +366,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -385,11 +385,12 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto path = parseStorePath(readString(conn->from)); + SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -600,7 +601,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + worker_proto::write(*this, sink, info.referencesPossiblyToSelf()); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -611,7 +612,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); - assert(importedPaths.size() <= 1); + assert(importedPaths.empty() == 0); // doesn't include possible self reference } else { @@ -619,7 +620,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 970bafd88..3cb3356f3 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "thread-pool.hh" #include "json.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" #include "remote-store.hh" @@ -159,63 +160,61 @@ StorePath Store::makeOutputPath(std::string_view id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ static std::string makeType( const Store & store, string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const PathReferences<StorePath> & references) { - for (auto & i : references) { + for (auto & i : references.references) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.hasSelfReference) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(references.empty()); + assert(info.references.references.size() == 0); + assert(!info.references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", PathReferences<StorePath> { info.references }), + info.hash, + name); +} + + +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { - [&](TextHash th) { - return makeTextPath(name, th.hash, references); + [&](TextInfo ti) { + return makeTextPath(desc.name, ti); }, - [&](FixedOutputHash fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](FixedOutputInfo foi) { + return makeFixedOutputPath(desc.name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, desc.info); } @@ -225,14 +224,24 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + FixedOutputInfo caInfo { + { + .method = method, + .hash = h, + }, + {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } @@ -336,11 +345,20 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }, narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { @@ -523,7 +541,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { + {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -536,11 +554,9 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref<const ValidPathInfo>(info)); @@ -585,13 +601,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -603,7 +619,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -820,7 +836,8 @@ void copyStorePath( // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref<ValidPathInfo>(*info); - info2->path = dstStore.makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore.makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore.storeDir == srcStore.storeDir) assert(info->path == info2->path); info = info2; @@ -962,7 +979,8 @@ std::map<StorePath, StorePath> copyPaths( auto info = srcStore.queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore.makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore.storeDir == srcStore.storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -992,8 +1010,9 @@ std::map<StorePath, StorePath> copyPaths( auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore.makeFixedOutputPathFromCA(storePath.name(), *info->ca); - if (dstStore.storeDir == srcStore.storeDir) + storePathForDst = dstStore.makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); + if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) debug("replaced path '%s' to '%s' for substituter '%s'", @@ -1067,7 +1086,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre if (!n) throw Error("number expected"); while ((*n)--) { getline(str, s); - info.references.insert(store.parseStorePath(s)); + info.insertReferencePossiblyToSelf(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional<ValidPathInfo>(std::move(info)); @@ -1090,6 +1109,20 @@ string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return PathReferences<StorePath>::referencesPossiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return PathReferences<StorePath>::insertReferencePossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return PathReferences<StorePath>::setReferencesPossiblyToSelf(path, std::move(refs)); +} std::string ValidPathInfo::fingerprint(const Store & store) const { @@ -1100,7 +1133,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); } @@ -1109,24 +1142,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } +std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const +{ + if (! ca) + return std::nullopt; + + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](TextHash th) { + TextInfo info { th }; + assert(!hasSelfReference); + info.references = references; + return ContentAddressWithReferences { info }; + }, + [&](FixedOutputHash foh) { + FixedOutputInfo info { foh }; + info.references = static_cast<PathReferences<StorePath>>(*this); + return ContentAddressWithReferences { info }; + }, + }, *ca), + }; +} + bool ValidPathInfo::isContentAddressed(const Store & store) const { - if (! ca) return false; + auto fullCaOpt = fullStorePathDescriptorOpt(); - auto caPath = std::visit(overloaded { - [&](TextHash th) { - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](FixedOutputHash fsh) { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); - } - }, *ca); + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -1158,12 +1204,32 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : references) + for (auto & r : referencesPossiblyToSelf()) refs.push_back(std::string(r.to_string())); return refs; } +ValidPathInfo::ValidPathInfo( + const Store & store, + StorePathDescriptor && info, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(info)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](TextInfo ti) { + this->references = ti.references; + this->ca = TextHash { std::move(ti) }; + }, + [this](FixedOutputInfo foi) { + *(static_cast<PathReferences<StorePath> *>(this)) = foi.references; + this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + }, + }, std::move(info.info)); +} + + Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 54471bdf2..a6a4d0c73 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -174,6 +174,8 @@ struct BuildResult } }; +typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; + struct StoreConfig : public Config { using Config::Config; @@ -296,17 +298,11 @@ public: StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/libutil/error.hh b/src/libutil/error.hh index ff58d3e00..e55986d6f 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -195,9 +195,8 @@ public: template<typename... Args> SysError(const Args & ... args) - : Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index cef3bd481..2ffdc2e9b 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -359,7 +359,7 @@ namespace nix { // constructing without access violation. ErrPos ep(invalid); - + // assignment without access violation. ep = invalid; diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 8b699f39b..45abe0405 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -58,7 +58,7 @@ void printDotGraph(ref<Store> store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 8ca5c9c8d..1cd974e41 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref<Store> store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->references) { + for (auto & p : info->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b327793e7..7947e65fb 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -197,10 +197,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -211,7 +211,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + }, + {}, + }))); } @@ -253,7 +259,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->references); + auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -334,7 +340,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) + for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) paths.insert(p); } else if (query == qReferrers) { @@ -838,7 +844,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->references); + worker_proto::write(*store, out, info->referencesPossiblyToSelf()); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -936,7 +942,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(worker_proto::read(*store, in, Phantom<StorePathSet> {})); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 2ae042789..ab2b62da2 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -42,14 +42,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand } ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + *store, + StorePathDescriptor { + .name = *namePart, + .info = FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + {}, + }, + }, narHash, }; info.narSize = sink.s->size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 12f303a10..9bbfe4747 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -46,19 +46,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + PathReferences<StorePath> refs; + refs.hasSelfReference = oldInfo->hasSelfReference; for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); + refs.references.insert(std::move(replacement)); } *sink.s = rewriteStrings(*sink.s, rewrites); @@ -69,16 +65,20 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + *store, + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + std::move(refs), + }, + }, narHash, }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; if (!json) notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 768d37595..c13603d42 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -67,7 +67,13 @@ std::tuple<StorePath, Hash> prefetchFile( the store. */ if (expectedHash) { hashType = expectedHash->type; - storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { + { + .method = ingestionMethod, + .hash = *expectedHash, + }, + {}, + }); if (store->isValidPath(*storePath)) hash = expectedHash; else diff --git a/src/nix/profile.cc b/src/nix/profile.cc index a1cb3fc76..bd5042d8f 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -162,12 +162,21 @@ struct ProfileManifest auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + StorePathDescriptor { + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + { references }, + }, + }, narHash, }; info.references = std::move(references); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; auto source = StringSource { *sink.s }; store->addToStore(info, source); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 43e0d9148..e89ad2650 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -63,7 +63,8 @@ struct CmdCopySigs : StorePathsCommand binary. */ if (info->narHash != info2->narHash || info->narSize != info2->narSize || - info->references != info2->references) + info->references != info2->references || + info->hasSelfReference != info2->hasSelfReference) continue; for (auto & sig : info2->sigs) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index e92df1303..d132768ec 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -81,14 +81,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount<std::atomic<size_t>> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -173,7 +173,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); |