diff options
37 files changed, 503 insertions, 235 deletions
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 599921151..ea8bbaf34 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -111,7 +111,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * arr = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) arr))); } catch (Error & e) { @@ -287,7 +287,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2b304aab0..c74b67658 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1045,7 +1045,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + { + .method = ingestionMethod, + .hash = h, + }, + {}, + }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputCAFixed { @@ -1764,7 +1770,13 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional<StorePath> expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..c5f327bd8 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,5 +10,5 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); - + } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 49851f7bc..67bb77d3e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -198,7 +198,13 @@ StorePath Input::computeStorePath(Store & store) const auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for mutable input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); + return store.makeFixedOutputPath("source", FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + }, + {}, + }); } std::string Input::getType() const diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 8c0f20475..08c6ae98c 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -71,14 +71,20 @@ DownloadFileResult downloadFile( dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + *store, + { + .name = name, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + {}, + }, + }, hashString(htSHA256, *sink.s), }; info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f6224d6a0..2d92e1c50 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -193,8 +193,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon( reads, but typically they'll already be cached. */ for (auto & ref : info.references) try { - if (ref != info.path) - queryPathInfo(ref); + queryPathInfo(ref); } catch (InvalidPath &) { throw Error("cannot add '%s' to the binary cache because the reference '%s' is not valid", printStorePath(info.path), printStorePath(ref)); @@ -323,7 +322,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; @@ -413,14 +422,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; return info; })->path; } @@ -429,17 +444,26 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s const StorePathSet & references, RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); + auto path = makeTextPath(name, TextInfo { textHash, references }); if (!repair && isValidPath(path)) return path; auto source = StringSource { s }; return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; + ValidPathInfo info { + *this, + { + .name = name, + .info = TextInfo { + { .hash = textHash }, + references, + }, + }, + nar.first, + }; info.narSize = nar.second; info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 05a9ef088..48e123f36 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4043,27 +4043,26 @@ void DerivationGoal::registerOutputs() } }; - auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> { + auto rewriteRefs = [&]() -> PathReferences<StorePath> { /* In the CA case, we need the rewritten refs to calculate the final path, therefore we look for a *non-rewritten self-reference, and use a bool rather try to solve the computationally intractable fixed point. */ - std::pair<bool, StorePathSet> res { - false, - {}, + PathReferences<StorePath> res { + .hasSelfReference = false, }; for (auto & r : references) { auto name = r.name(); auto origHash = std::string { r.hashPart() }; if (r == scratchPath) - res.first = true; + res.hasSelfReference = true; else if (outputRewrites.count(origHash) == 0) - res.second.insert(r); + res.references.insert(r); else { std::string newRef = outputRewrites.at(origHash); newRef += '-'; newRef += name; - res.second.insert(StorePath { newRef }); + res.references.insert(StorePath { newRef }); } } return res; @@ -4092,27 +4091,24 @@ void DerivationGoal::registerOutputs() break; } auto got = caSink.finish().first; - auto refs = rewriteRefs(); HashModuloSink narSink { htSHA256, oldHashPart }; dumpPath(actualPath, narSink); auto narHashAndSize = narSink.finish(); ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.second, - refs.first), + worker.store, + { + .name = outputPathName(drv->name, outputName), + .info = FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, + }, + rewriteRefs(), + }, + }, narHashAndSize.first, }; newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); assert(newInfo0.ca); return newInfo0; @@ -4132,10 +4128,7 @@ void DerivationGoal::registerOutputs() auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; - auto refs = rewriteRefs(); - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); + static_cast<PathReferences<StorePath> &>(newInfo0) = rewriteRefs(); return newInfo0; }, [&](DerivationOutputCAFixed dof) { @@ -4401,12 +4394,12 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) auto i = outputsByPath.find(worker.store.printStorePath(path)); if (i != outputsByPath.end()) { closureSize += i->second.narSize; - for (auto & ref : i->second.references) + for (auto & ref : i->second.referencesPossiblyToSelf()) pathsLeft.push(ref); } else { auto info = worker.store.queryPathInfo(path); closureSize += info->narSize; - for (auto & ref : info->references) + for (auto & ref : info->referencesPossiblyToSelf()) pathsLeft.push(ref); } } @@ -4445,7 +4438,7 @@ void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs) auto used = recursive ? getClosure(info.path).first - : info.references; + : info.referencesPossiblyToSelf(); if (recursive && checks.ignoreSelfRefs) used.erase(info.path); @@ -4902,7 +4895,10 @@ void SubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA({ + .name = std::string { storePath.name() }, + .info = caWithoutRefs(*ca), + }); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { @@ -4932,7 +4928,7 @@ void SubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { + if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) { auto info2 = std::make_shared<ValidPathInfo>(*info); info2->path = storePath; info = info2; @@ -4975,8 +4971,7 @@ void SubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makeSubstitutionGoal(i)); + addWaitee(worker.makeSubstitutionGoal(i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ referencesValid(); @@ -4996,8 +4991,7 @@ void SubstitutionGoal::referencesValid() } for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); + assert(worker.store.isValidPath(i)); state = &SubstitutionGoal::tryToRun; worker.wakeUp(shared_from_this()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 90a3ad1f5..d68c60f4f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,6 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } + std::string makeFileIngestionPrefix(const FileIngestionMethod m) { switch (m) { @@ -16,9 +17,8 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; - default: - throw Error("impossible, caught both cases"); } + assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) @@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { - return "text:" + th.hash.to_string(Base32, true); + return "text:" + + th.hash.to_string(Base32, true); }, [](FixedOutputHash fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } @@ -142,7 +145,18 @@ Hash getContentAddressHash(const ContentAddress & ca) }, [](FixedOutputHash fsh) { return fsh.hash; - } + }, + }, ca); +} + +ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { + return std::visit(overloaded { + [&](TextHash h) -> ContentAddressWithReferences { + return TextInfo { h, {}}; + }, + [&](FixedOutputHash h) -> ContentAddressWithReferences { + return FixedOutputInfo { h, {}}; + }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..e15d76bd7 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -2,14 +2,20 @@ #include <variant> #include "hash.hh" +#include "path.hh" namespace nix { +/* + * Mini content address + */ + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; + struct TextHash { Hash hash; }; @@ -41,10 +47,6 @@ typedef std::variant< ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional<ContentAddress> ca); @@ -74,4 +76,96 @@ ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); std::string renderContentAddressMethod(ContentAddressMethod caMethod); +/* + * References set + */ + +template<typename Ref> +struct PathReferences +{ + std::set<Ref> references; + bool hasSelfReference = false; + + bool operator == (const PathReferences<Ref> & other) const + { + return references == other.references + && hasSelfReference == other.hasSelfReference; + } + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs); +}; + +template<typename Ref> +StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet refs { references }; + if (hasSelfReference) + refs.insert(self); + return refs; +} + +template<typename Ref> +void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} + +template<typename Ref> +void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); + + references = refs; +} + +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ + +// This matches the additional info that we need for makeTextPath +struct TextInfo : TextHash { + // References for the paths, self references disallowed + StorePathSet references; +}; + +struct FixedOutputInfo : FixedOutputHash { + // References for the paths + PathReferences<StorePath> references; +}; + +typedef std::variant< + TextInfo, + FixedOutputInfo +> ContentAddressWithReferences; + +ContentAddressWithReferences caWithoutRefs(const ContentAddress &); + +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + bool operator == (const StorePathDescriptor & other) const + { + return name == other.name; + // FIXME second field + } + + bool operator < (const StorePathDescriptor & other) const + { + return name < other.name; + // FIXME second field + } +}; + } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 99d8add92..7ae88b49a 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -250,7 +250,7 @@ static void writeValidPathInfo( { to << (info->deriver ? store->printStorePath(*info->deriver) : "") << info->narHash.to_string(Base16, false); - worker_proto::write(*store, to, info->references); + worker_proto::write(*store, to, info->referencesPossiblyToSelf()); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate @@ -320,7 +320,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); StorePathSet paths; if (op == wopQueryReferences) - for (auto & i : store->queryPathInfo(path)->references) + for (auto & i : store->queryPathInfo(path)->referencesPossiblyToSelf()) paths.insert(i); else if (op == wopQueryReferrers) store->queryReferrers(path, paths); @@ -694,7 +694,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references); + worker_proto::write(*store, to, i->second.referencesPossiblyToSelf(path)); to << i->second.downloadSize << i->second.narSize; } @@ -717,7 +717,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references); + worker_proto::write(*store, to, i.second.referencesPossiblyToSelf(i.first)); to << i.second.downloadSize << i.second.narSize; } break; @@ -799,7 +799,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(worker_proto::read(*store, from, Phantom<StorePathSet> {})); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); info.ca = parseContentAddressOpt(readString(from)); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 07b4e772b..925a78083 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -27,8 +27,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + outputPathName(drvName, outputName), + { hash, {} }); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 6d292b2e5..e2e5578a8 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -128,8 +128,8 @@ struct Derivation : BasicDerivation /* Return the underlying basic derivation but with these changes: - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. + 1. Input drvs are emptied, but the outputs of them that were used are + added directly to input sources. 2. Input placeholders are replaced with realized input store paths. */ std::optional<BasicDerivation> tryResolve(Store & store); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 02c839520..55585e977 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -45,7 +45,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->references); + worker_proto::write(*this, teeSink, info->referencesPossiblyToSelf()); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -80,7 +80,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = references; + info.setReferencesPossiblyToSelf(std::move(references)); info.narSize = saved.s->size(); // Ignore optional legacy signature. diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 467169ce8..e440536df 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -118,11 +118,10 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig /* Hash will be set below. FIXME construct ValidPathInfo at end. */ auto info = std::make_shared<ValidPathInfo>(path, Hash::dummy); - PathSet references; auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -156,7 +155,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize @@ -185,7 +184,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d29236a9c..e6b02cce6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -567,7 +567,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat envHasRightPath(doia.path, i.first); }, [&](DerivationOutputCAFixed dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} }); envHasRightPath(path, i.first); }, [&](DerivationOutputCAFloating _) { @@ -693,8 +693,10 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + while (useQueryReferences.next()) { + info->insertReferencePossiblyToSelf( + parseStorePath(useQueryReferences.getStr(0))); + } return info; })); @@ -921,7 +923,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // recompute store path so that we can use a different store root if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + subPath = makeFixedOutputPathFromCA({ + .name = std::string { path.first.name() }, + .info = caWithoutRefs(*path.second), + }); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -938,10 +943,12 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst auto narInfo = std::dynamic_pointer_cast<const NarInfo>( std::shared_ptr<const ValidPathInfo>(info)); infos.insert_or_assign(path.first, SubstitutablePathInfo{ - info->deriver, info->references, + info->hasSelfReference, + info->deriver, narInfo ? narInfo->fileSize : 0, - info->narSize}); + info->narSize, + }); } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { @@ -988,7 +995,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) { auto referrer = queryValidPathId(*state, i.path); - for (auto & j : i.references) + for (auto & j : i.referencesPossiblyToSelf()) state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); } @@ -1064,14 +1071,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); // text hashing has long been allowed to have non-self-references because it is used for drv files. - bool refersToSelf = info.references.count(info.path) > 0; - if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf)) + if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && info.hasSelfReference)) settings.requireExperimentalFeature("ca-references"); /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr<AbstractHashSink> hashSink; - if (!info.ca.has_value() || !info.references.count(info.path)) + if (!info.ca.has_value() || !info.hasSelfReference) hashSink = std::make_unique<HashSink>(htSHA256); else hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart())); @@ -1161,7 +1167,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name); + auto desc = StorePathDescriptor { + name, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }; + + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); @@ -1181,7 +1198,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, autoGC(); if (inMemory) { - StringSource dumpSource { dump }; + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource); @@ -1206,9 +1223,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, optimisePath(realPath); - ValidPathInfo info { dstPath, narHash.first }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; - info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1223,7 +1239,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); @@ -1363,7 +1382,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); std::unique_ptr<AbstractHashSink> hashSink; - if (!info->ca || !info->references.count(info->path)) + if (!info->ca || !info->hasSelfReference) hashSink = std::make_unique<HashSink>(info->narHash.type); else hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart())); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index ad4dccef9..052936a8b 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -62,8 +62,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths, } else { for (auto & ref : info->references) - if (ref != path) - enqueue(printStorePath(ref)); + enqueue(printStorePath(ref)); if (includeOutputs && path.isDerivation()) for (auto & i : queryDerivationOutputs(path)) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 8541cc51f..759342d84 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -199,7 +199,7 @@ public: narInfo->fileSize = queryNAR.getInt(5); narInfo->narSize = queryNAR.getInt(7); for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " ")) - narInfo->references.insert(StorePath(r)); + narInfo->insertReferencePossiblyToSelf(StorePath(r)); if (!queryNAR.isNull(9)) narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 3454f34bb..7c5c8fdd1 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -59,7 +59,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & auto refs = tokenizeString<Strings>(value, " "); if (!references.empty()) throw corrupt(); for (auto & r : refs) - references.insert(StorePath(r)); + insertReferencePossiblyToSelf(StorePath(r)); } else if (name == "Deriver") { if (value != "unknown-deriver") diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 39ced76e5..fd37b85db 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo std::string system; NarInfo() = delete; + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) + { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 8ff5c466e..8c4791ac0 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -14,24 +14,20 @@ namespace nix { class Store; -struct SubstitutablePathInfo +struct SubstitutablePathInfo : PathReferences<StorePath> { std::optional<StorePath> deriver; - StorePathSet references; uint64_t downloadSize; /* 0 = unknown or inapplicable */ uint64_t narSize; /* 0 = unknown */ }; typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; - -struct ValidPathInfo +struct ValidPathInfo : PathReferences<StorePath> { StorePath path; std::optional<StorePath> deriver; - // TODO document this Hash narHash; - StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown uint64_t id; // internal use only @@ -65,6 +61,7 @@ struct ValidPathInfo return path == i.path && narHash == i.narHash + && hasSelfReference == i.hasSelfReference && references == i.references; } @@ -78,6 +75,8 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); + std::optional<StorePathDescriptor> fullStorePathDescriptorOpt() const; + /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -104,6 +103,9 @@ struct ValidPathInfo ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(const Store & store, + StorePathDescriptor && ca, Hash narHash); + virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index b03a0f69d..5f239ceb6 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,7 @@ #pragma once -#include "content-address.hh" +#include <string_view> + #include "types.hh" namespace nix { @@ -64,8 +65,6 @@ typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; typedef std::map<string, StorePath> OutputPathMap; -typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 488270f48..5ff787ed2 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -309,7 +309,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(i.first, worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -328,11 +328,12 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto path = parseStorePath(readString(conn->from)); + SubstitutablePathInfo & info { infos[path] }; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(path, worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -347,7 +348,7 @@ ref<const ValidPathInfo> RemoteStore::readValidPathInfo(ConnectionHandle & conn, auto narHash = Hash::parseAny(readString(conn->from), htSHA256); auto info = make_ref<ValidPathInfo>(path, narHash); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info->setReferencesPossiblyToSelf(worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -554,7 +555,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + worker_proto::write(*this, sink, info.referencesPossiblyToSelf()); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -565,7 +566,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); - assert(importedPaths.size() <= 1); + assert(importedPaths.empty() == 0); // doesn't include possible self reference } else { @@ -573,7 +574,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.referencesPossiblyToSelf()); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9f21f0434..c2696f7a5 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "thread-pool.hh" #include "json.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" @@ -163,63 +164,61 @@ StorePath Store::makeOutputPath(std::string_view id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ static std::string makeType( const Store & store, string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const PathReferences<StorePath> & references) { - for (auto & i : references) { + for (auto & i : references.references) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.hasSelfReference) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(references.empty()); + assert(info.references.references.size() == 0); + assert(!info.references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", PathReferences<StorePath> { info.references }), + info.hash, + name); +} + + +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { - [&](TextHash th) { - return makeTextPath(name, th.hash, references); + [&](TextInfo ti) { + return makeTextPath(desc.name, ti); }, - [&](FixedOutputHash fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](FixedOutputInfo foi) { + return makeFixedOutputPath(desc.name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, desc.info); } @@ -229,14 +228,24 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + FixedOutputInfo caInfo { + { + .method = method, + .hash = h, + }, + {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } @@ -326,11 +335,20 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }, narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { @@ -496,7 +514,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { + {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -509,11 +527,9 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref<const ValidPathInfo>(info)); @@ -536,13 +552,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -554,7 +570,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -737,7 +753,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref<ValidPathInfo>(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -799,7 +816,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -826,7 +844,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -891,7 +910,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre if (!string2Int(s, n)) throw Error("number expected"); while (n--) { getline(str, s); - info.references.insert(store.parseStorePath(s)); + info.insertReferencePossiblyToSelf(store.parseStorePath(s)); } if (!str || str.eof()) throw Error("missing input"); return std::optional<ValidPathInfo>(std::move(info)); @@ -914,6 +933,20 @@ string showPaths(const PathSet & paths) return concatStringsSep(", ", quoteStrings(paths)); } +StorePathSet ValidPathInfo::referencesPossiblyToSelf() const +{ + return PathReferences<StorePath>::referencesPossiblyToSelf(path); +} + +void ValidPathInfo::insertReferencePossiblyToSelf(StorePath && ref) +{ + return PathReferences<StorePath>::insertReferencePossiblyToSelf(path, std::move(ref)); +} + +void ValidPathInfo::setReferencesPossiblyToSelf(StorePathSet && refs) +{ + return PathReferences<StorePath>::setReferencesPossiblyToSelf(path, std::move(refs)); +} std::string ValidPathInfo::fingerprint(const Store & store) const { @@ -924,7 +957,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const "1;" + store.printStorePath(path) + ";" + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" - + concatStringsSep(",", store.printStorePathSet(references)); + + concatStringsSep(",", store.printStorePathSet(referencesPossiblyToSelf())); } @@ -933,24 +966,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } +std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const +{ + if (! ca) + return std::nullopt; + + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](TextHash th) { + TextInfo info { th }; + assert(!hasSelfReference); + info.references = references; + return ContentAddressWithReferences { info }; + }, + [&](FixedOutputHash foh) { + FixedOutputInfo info { foh }; + info.references = static_cast<PathReferences<StorePath>>(*this); + return ContentAddressWithReferences { info }; + }, + }, *ca), + }; +} + bool ValidPathInfo::isContentAddressed(const Store & store) const { - if (! ca) return false; + auto fullCaOpt = fullStorePathDescriptorOpt(); - auto caPath = std::visit(overloaded { - [&](TextHash th) { - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](FixedOutputHash fsh) { - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); - } - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); - } - }, *ca); + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -982,12 +1028,32 @@ bool ValidPathInfo::checkSignature(const Store & store, const PublicKeys & publi Strings ValidPathInfo::shortRefs() const { Strings refs; - for (auto & r : references) + for (auto & r : referencesPossiblyToSelf()) refs.push_back(std::string(r.to_string())); return refs; } +ValidPathInfo::ValidPathInfo( + const Store & store, + StorePathDescriptor && info, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(info)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](TextInfo ti) { + this->references = ti.references; + this->ca = TextHash { std::move(ti) }; + }, + [this](FixedOutputInfo foi) { + *(static_cast<PathReferences<StorePath> *>(this)) = foi.references; + this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + }, + }, std::move(info.info)); +} + + Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f77bc21d1..dba6198eb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -170,6 +170,8 @@ struct BuildResult } }; +typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; + struct StoreConfig : public Config { using Config::Config; @@ -315,17 +317,11 @@ public: StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d1b6d82bb..97e11c98e 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -201,9 +201,8 @@ public: template<typename... Args> SysError(const Args & ... args) - : Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 7e53f17c6..d33bd7c1f 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -370,7 +370,7 @@ namespace nix { // constructing without access violation. ErrPos ep(invalid); - + // assignment without access violation. ep = invalid; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 55d02bcf9..6c4c5ab74 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include <list> #include <set> +#include <string> #include <map> #include <vector> diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 3bdee55a7..80493b17b 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -161,8 +161,14 @@ static int main_nix_prefetch_url(int argc, char * * argv) std::optional<StorePath> storePath; if (args.size() == 2) { expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); if (store->isValidPath(*storePath)) hash = *expectedHash; else diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 8b699f39b..45abe0405 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -58,7 +58,7 @@ void printDotGraph(ref<Store> store, StorePathSet && roots) cout << makeNode(std::string(path.to_string()), path.name(), "#ff0000"); - for (auto & p : store->queryPathInfo(path)->references) { + for (auto & p : store->queryPathInfo(path)->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(std::string(p.to_string()), std::string(path.to_string())); diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 8ca5c9c8d..1cd974e41 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -71,7 +71,7 @@ void printGraphML(ref<Store> store, StorePathSet && roots) auto info = store->queryPathInfo(path); cout << makeNode(*info); - for (auto & p : info->references) { + for (auto & p : info->referencesPossiblyToSelf()) { if (p != path) { workList.insert(p); cout << makeEdge(path.to_string(), p.to_string()); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 14baabc36..6bbb7fcab 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -198,10 +198,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -212,7 +212,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + }, + {}, + }))); } @@ -254,7 +260,7 @@ static void printTree(const StorePath & path, closure(B). That is, if derivation A is an (possibly indirect) input of B, then A is printed first. This has the effect of flattening the tree, preventing deeply nested structures. */ - auto sorted = store->topoSortPaths(info->references); + auto sorted = store->topoSortPaths(info->referencesPossiblyToSelf()); reverse(sorted.begin(), sorted.end()); for (const auto &[n, i] : enumerate(sorted)) { @@ -335,7 +341,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : ps) { if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); else if (query == qReferences) { - for (auto & p : store->queryPathInfo(j)->references) + for (auto & p : store->queryPathInfo(j)->referencesPossiblyToSelf()) paths.insert(p); } else if (query == qReferrers) { @@ -867,7 +873,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->references); + worker_proto::write(*store, out, info->referencesPossiblyToSelf()); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -961,7 +967,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {}); + info.setReferencesPossiblyToSelf(worker_proto::read(*store, in, Phantom<StorePathSet> {})); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(in); info.ca = parseContentAddressOpt(readString(in)); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 7fe87d757..c9d07bdcc 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,14 +69,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand } ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + *store, + StorePathDescriptor { + .name = *namePart, + .info = FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + {}, + }, + }, narHash, }; info.narSize = sink.s->size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 2d0a0b6ea..4a936c385 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -92,7 +92,7 @@ struct CmdBundle : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); arg->attrs->sort(); - + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index df3ec5194..53ad39e02 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -55,19 +55,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + PathReferences<StorePath> refs; + refs.hasSelfReference = oldInfo->hasSelfReference; for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); + refs.references.insert(std::move(replacement)); } *sink.s = rewriteStrings(*sink.s, rewrites); @@ -78,16 +74,20 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + *store, + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + std::move(refs), + }, + }, narHash, }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 01aef2f9b..b9ad31cf4 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -130,12 +130,21 @@ struct ProfileManifest auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + StorePathDescriptor { + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + { references }, + }, + }, narHash, }; info.references = std::move(references); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; auto source = StringSource { *sink.s }; store->addToStore(info, source); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 44916c77f..26bd277f5 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -65,7 +65,8 @@ struct CmdCopySigs : StorePathsCommand binary. */ if (info->narHash != info2->narHash || info->narSize != info2->narSize || - info->references != info2->references) + info->references != info2->references || + info->hasSelfReference != info2->hasSelfReference) continue; for (auto & sig : info2->sigs) diff --git a/src/nix/verify.cc b/src/nix/verify.cc index ec7333d03..6fedc8509 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -73,14 +73,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount<std::atomic<size_t>> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -178,7 +178,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); |