diff options
29 files changed, 431 insertions, 205 deletions
diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 599921151..ea8bbaf34 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -111,7 +111,7 @@ SV * queryPathInfo(char * path, int base32) mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * arr = newAV(); - for (auto & i : info->references) + for (auto & i : info->referencesPossiblyToSelf()) av_push(arr, newSVpv(store()->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) arr))); } catch (Error & e) { @@ -287,7 +287,13 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) try { auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(method, h, name); + auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2b304aab0..c74b67658 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1045,7 +1045,13 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); Hash h = newHashAllowEmpty(*outputHash, ht); - auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { + { + .method = ingestionMethod, + .hash = h, + }, + {}, + }); drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputCAFixed { @@ -1764,7 +1770,13 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional<StorePath> expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..c5f327bd8 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,5 +10,5 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, Value & v, std::ostream & out, PathSet & context); - + } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 49851f7bc..67bb77d3e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -198,7 +198,13 @@ StorePath Input::computeStorePath(Store & store) const auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for mutable input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); + return store.makeFixedOutputPath("source", FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = *narHash, + }, + {}, + }); } std::string Input::getType() const diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ca49482a9..b3ee84810 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -71,14 +71,20 @@ DownloadFileResult downloadFile( dumpString(*res.data, sink); auto hash = hashString(htSHA256, *res.data); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name), + *store, + { + .name = name, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Flat, + .hash = hash, + }, + {}, + }, + }, hashString(htSHA256, *sink.s), }; info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = hash, - }; auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index d592f16dd..2d92e1c50 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -322,7 +322,17 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & nam unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = nar.first, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; @@ -412,14 +422,20 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name), + *this, + { + .name = name, + .info = FixedOutputInfo { + { + .method = method, + .hash = h, + }, + {}, + }, + }, nar.first, }; info.narSize = nar.second; - info.ca = FixedOutputHash { - .method = method, - .hash = h, - }; return info; })->path; } @@ -428,17 +444,26 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s const StorePathSet & references, RepairFlag repair) { auto textHash = hashString(htSHA256, s); - auto path = makeTextPath(name, textHash, references); + auto path = makeTextPath(name, TextInfo { textHash, references }); if (!repair && isValidPath(path)) return path; auto source = StringSource { s }; return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info { path, nar.first }; + ValidPathInfo info { + *this, + { + .name = name, + .info = TextInfo { + { .hash = textHash }, + references, + }, + }, + nar.first, + }; info.narSize = nar.second; info.ca = TextHash { textHash }; - info.references = references; return info; })->path; } diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 97a832c6b..12ce6f2ec 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -4056,25 +4056,24 @@ void DerivationGoal::registerOutputs() break; } auto got = caSink.finish().first; - auto refs = rewriteRefs(); HashModuloSink narSink { htSHA256, oldHashPart }; dumpPath(actualPath, narSink); auto narHashAndSize = narSink.finish(); ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( - outputHash.method, - got, - outputPathName(drv->name, outputName), - refs.references, - refs.hasSelfReference), + worker.store, + { + .name = outputPathName(drv->name, outputName), + .info = FixedOutputInfo { + { + .method = outputHash.method, + .hash = got, + }, + rewriteRefs(), + }, + }, narHashAndSize.first, }; newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - static_cast<PathReferences<StorePath> &>(newInfo0) = refs; assert(newInfo0.ca); return newInfo0; @@ -4861,7 +4860,10 @@ void SubstitutionGoal::tryNext() subs.pop_front(); if (ca) { - subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); + subPath = sub->makeFixedOutputPathFromCA({ + .name = std::string { storePath.name() }, + .info = caWithoutRefs(*ca), + }); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { @@ -4891,7 +4893,7 @@ void SubstitutionGoal::tryNext() } if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { + if (info->isContentAddressed(*sub) && info->references.empty() && !info->hasSelfReference) { auto info2 = std::make_shared<ValidPathInfo>(*info); info2->path = storePath; info = info2; diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 90a3ad1f5..d68c60f4f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -9,6 +9,7 @@ std::string FixedOutputHash::printMethodAlgo() const return makeFileIngestionPrefix(method) + printHashType(hash.type); } + std::string makeFileIngestionPrefix(const FileIngestionMethod m) { switch (m) { @@ -16,9 +17,8 @@ std::string makeFileIngestionPrefix(const FileIngestionMethod m) return ""; case FileIngestionMethod::Recursive: return "r:"; - default: - throw Error("impossible, caught both cases"); } + assert(false); } std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) @@ -32,10 +32,13 @@ std::string renderContentAddress(ContentAddress ca) { return std::visit(overloaded { [](TextHash th) { - return "text:" + th.hash.to_string(Base32, true); + return "text:" + + th.hash.to_string(Base32, true); }, [](FixedOutputHash fsh) { - return makeFixedOutputCA(fsh.method, fsh.hash); + return "fixed:" + + makeFileIngestionPrefix(fsh.method) + + fsh.hash.to_string(Base32, true); } }, ca); } @@ -142,7 +145,18 @@ Hash getContentAddressHash(const ContentAddress & ca) }, [](FixedOutputHash fsh) { return fsh.hash; - } + }, + }, ca); +} + +ContentAddressWithReferences caWithoutRefs(const ContentAddress & ca) { + return std::visit(overloaded { + [&](TextHash h) -> ContentAddressWithReferences { + return TextInfo { h, {}}; + }, + [&](FixedOutputHash h) -> ContentAddressWithReferences { + return FixedOutputInfo { h, {}}; + }, }, ca); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f6a6f5140..e15d76bd7 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -2,14 +2,20 @@ #include <variant> #include "hash.hh" +#include "path.hh" namespace nix { +/* + * Mini content address + */ + enum struct FileIngestionMethod : uint8_t { Flat = false, Recursive = true }; + struct TextHash { Hash hash; }; @@ -41,10 +47,6 @@ typedef std::variant< ingested. */ std::string makeFileIngestionPrefix(const FileIngestionMethod m); -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - std::string renderContentAddress(ContentAddress ca); std::string renderContentAddress(std::optional<ContentAddress> ca); @@ -74,4 +76,96 @@ ContentAddressMethod parseContentAddressMethod(std::string_view rawCaMethod); std::string renderContentAddressMethod(ContentAddressMethod caMethod); +/* + * References set + */ + +template<typename Ref> +struct PathReferences +{ + std::set<Ref> references; + bool hasSelfReference = false; + + bool operator == (const PathReferences<Ref> & other) const + { + return references == other.references + && hasSelfReference == other.hasSelfReference; + } + + /* Functions to view references + hasSelfReference as one set, mainly for + compatibility's sake. */ + StorePathSet referencesPossiblyToSelf(const Ref & self) const; + void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); + void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs); +}; + +template<typename Ref> +StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const +{ + StorePathSet refs { references }; + if (hasSelfReference) + refs.insert(self); + return refs; +} + +template<typename Ref> +void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) +{ + if (ref == self) + hasSelfReference = true; + else + references.insert(std::move(ref)); +} + +template<typename Ref> +void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs) +{ + if (refs.count(self)) + hasSelfReference = true; + refs.erase(self); + + references = refs; +} + +/* + * Full content address + * + * See the schema for store paths in store-api.cc + */ + +// This matches the additional info that we need for makeTextPath +struct TextInfo : TextHash { + // References for the paths, self references disallowed + StorePathSet references; +}; + +struct FixedOutputInfo : FixedOutputHash { + // References for the paths + PathReferences<StorePath> references; +}; + +typedef std::variant< + TextInfo, + FixedOutputInfo +> ContentAddressWithReferences; + +ContentAddressWithReferences caWithoutRefs(const ContentAddress &); + +struct StorePathDescriptor { + std::string name; + ContentAddressWithReferences info; + + bool operator == (const StorePathDescriptor & other) const + { + return name == other.name; + // FIXME second field + } + + bool operator < (const StorePathDescriptor & other) const + { + return name < other.name; + // FIXME second field + } +}; + } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 07b4e772b..925a78083 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -27,8 +27,8 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return store.makeFixedOutputPath( - hash.method, hash.hash, - outputPathName(drvName, outputName)); + outputPathName(drvName, outputName), + { hash, {} }); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index d48266774..be19aa300 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -130,8 +130,8 @@ struct Derivation : BasicDerivation /* Return the underlying basic derivation but with these changes: - 1. Input drvs are emptied, but the outputs of them that were used are - added directly to input sources. + 1. Input drvs are emptied, but the outputs of them that were used are + added directly to input sources. 2. Input placeholders are replaced with realized input store paths. */ std::optional<BasicDerivation> tryResolve(Store & store); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 18545f659..e6b02cce6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -567,7 +567,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat envHasRightPath(doia.path, i.first); }, [&](DerivationOutputCAFixed dof) { - StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); + StorePath path = makeFixedOutputPath(drvName, { dof.hash, {} }); envHasRightPath(path, i.first); }, [&](DerivationOutputCAFloating _) { @@ -923,7 +923,10 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst // recompute store path so that we can use a different store root if (path.second) { - subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); + subPath = makeFixedOutputPathFromCA({ + .name = std::string { path.first.name() }, + .info = caWithoutRefs(*path.second), + }); if (sub->storeDir == storeDir) assert(subPath == path.first); if (subPath != path.first) @@ -1164,7 +1167,18 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name); + auto desc = StorePathDescriptor { + name, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }; + + auto dstPath = makeFixedOutputPathFromCA(desc); addTempRoot(dstPath); @@ -1184,7 +1198,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, autoGC(); if (inMemory) { - StringSource dumpSource { dump }; + StringSource dumpSource { dump }; /* Restore from the NAR in memory. */ if (method == FileIngestionMethod::Recursive) restorePath(realPath, dumpSource); @@ -1209,9 +1223,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, optimisePath(realPath); - ValidPathInfo info { dstPath, narHash.first }; + ValidPathInfo info { *this, std::move(desc), narHash.first }; info.narSize = narHash.second; - info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1226,7 +1239,10 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { auto hash = hashString(htSHA256, s); - auto dstPath = makeTextPath(name, hash, references); + auto dstPath = makeTextPath(name, TextInfo { + { .hash = hash }, + references, + }); addTempRoot(dstPath); diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 39ced76e5..fd37b85db 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,6 +17,9 @@ struct NarInfo : ValidPathInfo std::string system; NarInfo() = delete; + NarInfo(const Store & store, StorePathDescriptor && ca, Hash narHash) + : ValidPathInfo(store, std::move(ca), narHash) + { } NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index 509f100d7..8c4791ac0 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -13,47 +13,6 @@ namespace nix { class Store; -template<typename Ref> -struct PathReferences -{ - std::set<Ref> references; - bool hasSelfReference = false; - - /* Functions to view references + hasSelfReference as one set, mainly for - compatibility's sake. */ - StorePathSet referencesPossiblyToSelf(const Ref & self) const; - void insertReferencePossiblyToSelf(const Ref & self, Ref && ref); - void setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs); -}; - -template<typename Ref> -StorePathSet PathReferences<Ref>::referencesPossiblyToSelf(const Ref & self) const -{ - StorePathSet refs { references }; - if (hasSelfReference) - refs.insert(self); - return refs; -} - -template<typename Ref> -void PathReferences<Ref>::insertReferencePossiblyToSelf(const Ref & self, Ref && ref) -{ - if (ref == self) - hasSelfReference = true; - else - references.insert(std::move(ref)); -} - -template<typename Ref> -void PathReferences<Ref>::setReferencesPossiblyToSelf(const Ref & self, std::set<Ref> && refs) -{ - if (refs.count(self)) - hasSelfReference = true; - refs.erase(self); - - references = refs; -} - struct SubstitutablePathInfo : PathReferences<StorePath> { @@ -68,7 +27,6 @@ struct ValidPathInfo : PathReferences<StorePath> { StorePath path; std::optional<StorePath> deriver; - // TODO document this Hash narHash; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown @@ -117,6 +75,8 @@ struct ValidPathInfo : PathReferences<StorePath> void sign(const Store & store, const SecretKey & secretKey); + std::optional<StorePathDescriptor> fullStorePathDescriptorOpt() const; + /* Return true iff the path is verifiably content-addressed. */ bool isContentAddressed(const Store & store) const; @@ -143,6 +103,9 @@ struct ValidPathInfo : PathReferences<StorePath> ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(const Store & store, + StorePathDescriptor && ca, Hash narHash); + virtual ~ValidPathInfo() { } }; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index b03a0f69d..5f239ceb6 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,7 @@ #pragma once -#include "content-address.hh" +#include <string_view> + #include "types.hh" namespace nix { @@ -64,8 +65,6 @@ typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; typedef std::map<string, StorePath> OutputPathMap; -typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; - /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7041edbe5..5d63b8e3c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -7,6 +7,7 @@ #include "thread-pool.hh" #include "json.hh" #include "url.hh" +#include "references.hh" #include "archive.hh" #include "callback.hh" @@ -163,63 +164,61 @@ StorePath Store::makeOutputPath(std::string_view id, } +/* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ static std::string makeType( const Store & store, string && type, - const StorePathSet & references, - bool hasSelfReference = false) + const PathReferences<StorePath> & references) { - for (auto & i : references) { + for (auto & i : references.references) { type += ":"; type += store.printStorePath(i); } - if (hasSelfReference) type += ":self"; + if (references.hasSelfReference) type += ":self"; return std::move(type); } -StorePath Store::makeFixedOutputPath( - FileIngestionMethod method, - const Hash & hash, - std::string_view name, - const StorePathSet & references, - bool hasSelfReference) const +StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (hash.type == htSHA256 && method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { - assert(references.empty()); + assert(info.references.references.size() == 0); + assert(!info.references.hasSelfReference); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references, bool hasSelfReference) const + +StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +{ + assert(info.hash.type == htSHA256); + return makeStorePath( + makeType(*this, "text", PathReferences<StorePath> { info.references }), + info.hash, + name); +} + + +StorePath Store::makeFixedOutputPathFromCA(const StorePathDescriptor & desc) const { // New template return std::visit(overloaded { - [&](TextHash th) { - return makeTextPath(name, th.hash, references); + [&](TextInfo ti) { + return makeTextPath(desc.name, ti); }, - [&](FixedOutputHash fsh) { - return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + [&](FixedOutputInfo foi) { + return makeFixedOutputPath(desc.name, foi); } - }, ca); -} - -StorePath Store::makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references) const -{ - assert(hash.type == htSHA256); - /* Stuff the references (if any) into the type. This is a bit - hacky, but we can't put them in `s' since that would be - ambiguous. */ - return makeStorePath(makeType(*this, "text", references), hash, name); + }, desc.info); } @@ -229,14 +228,24 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, Hash h = method == FileIngestionMethod::Recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); - return std::make_pair(makeFixedOutputPath(method, h, name), h); + FixedOutputInfo caInfo { + { + .method = method, + .hash = h, + }, + {}, + }; + return std::make_pair(makeFixedOutputPath(name, caInfo), h); } StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, TextInfo { + { .hash = hashString(htSHA256, s) }, + references, + }); } @@ -326,11 +335,20 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, throw Error("hash mismatch for '%s'", srcPath); ValidPathInfo info { - makeFixedOutputPath(method, hash, name), + *this, + StorePathDescriptor { + std::string { name }, + FixedOutputInfo { + { + .method = method, + .hash = hash, + }, + {}, + }, + }, narHash, }; info.narSize = narSize; - info.ca = FixedOutputHash { .method = method, .hash = hash }; if (!isValidPath(info.path)) { auto source = sinkToSource([&](Sink & scratchpadSink) { @@ -496,7 +514,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { + {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -509,11 +527,9 @@ void Store::queryPathInfo(const StorePath & storePath, state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref<const ValidPathInfo>(info)); @@ -536,13 +552,13 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m std::condition_variable wakeup; ThreadPool pool; - auto doQuery = [&](const Path & path) { + auto doQuery = [&](const StorePath & path) { checkInterrupt(); - queryPathInfo(parseStorePath(path), {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { + queryPathInfo(path, {[path, this, &state_, &wakeup](std::future<ref<const ValidPathInfo>> fut) { auto state(state_.lock()); try { auto info = fut.get(); - state->valid.insert(parseStorePath(path)); + state->valid.insert(path); } catch (InvalidPath &) { } catch (...) { state->exc = std::current_exception(); @@ -554,7 +570,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m }; for (auto & path : paths) - pool.enqueue(std::bind(doQuery, printStorePath(path))); // FIXME + pool.enqueue(std::bind(doQuery, path)); pool.process(); @@ -737,7 +753,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref<ValidPathInfo>(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); + info2->path = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -799,7 +816,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -826,7 +844,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor auto storePathForDst = storePath; if (info->ca && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); + storePathForDst = dstStore->makeFixedOutputPathFromCA( + info->fullStorePathDescriptorOpt().value()); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -947,19 +966,37 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } +std::optional<StorePathDescriptor> ValidPathInfo::fullStorePathDescriptorOpt() const +{ + if (! ca) + return std::nullopt; + + return StorePathDescriptor { + .name = std::string { path.name() }, + .info = std::visit(overloaded { + [&](TextHash th) { + TextInfo info { th }; + assert(!hasSelfReference); + info.references = references; + return ContentAddressWithReferences { info }; + }, + [&](FixedOutputHash foh) { + FixedOutputInfo info { foh }; + info.references = static_cast<PathReferences<StorePath>>(*this); + return ContentAddressWithReferences { info }; + }, + }, *ca), + }; +} + bool ValidPathInfo::isContentAddressed(const Store & store) const { - if (! ca) return false; + auto fullCaOpt = fullStorePathDescriptorOpt(); - auto caPath = std::visit(overloaded { - [&](TextHash th) { - assert(!hasSelfReference); - return store.makeTextPath(path.name(), th.hash, references); - }, - [&](FixedOutputHash fsh) { - return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), references, hasSelfReference); - } - }, *ca); + if (! fullCaOpt) + return false; + + auto caPath = store.makeFixedOutputPathFromCA(*fullCaOpt); bool res = caPath == path; @@ -997,6 +1034,26 @@ Strings ValidPathInfo::shortRefs() const } +ValidPathInfo::ValidPathInfo( + const Store & store, + StorePathDescriptor && info, + Hash narHash) + : path(store.makeFixedOutputPathFromCA(info)) + , narHash(narHash) +{ + std::visit(overloaded { + [this](TextInfo ti) { + this->references = ti.references; + this->ca = TextHash { std::move(ti) }; + }, + [this](FixedOutputInfo foi) { + *(static_cast<PathReferences<StorePath> *>(this)) = foi.references; + this->ca = FixedOutputHash { (FixedOutputHash) std::move(foi) }; + }, + }, std::move(info.info)); +} + + Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 854446987..e6a6053a3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -170,6 +170,8 @@ struct BuildResult } }; +typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; + struct StoreConfig : public Config { using Config::Config; @@ -313,17 +315,11 @@ public: StorePath makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const; - StorePath makeFixedOutputPath(FileIngestionMethod method, - const Hash & hash, std::string_view name, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - StorePath makeTextPath(std::string_view name, const Hash & hash, - const StorePathSet & references = {}) const; + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, - const StorePathSet & references = {}, - bool hasSelfReference = false) const; + StorePath makeFixedOutputPathFromCA(const StorePathDescriptor & info) const; /* This is the preparatory part of addToStore(); it computes the store path to which srcPath is to be copied. Returns the store diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 147602415..453fe60f9 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -244,7 +244,7 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) diff --git a/src/libutil/error.hh b/src/libutil/error.hh index f3babcbde..260ed3cf8 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -201,9 +201,8 @@ public: template<typename... Args> SysError(const Args & ... args) - : Error("") + : Error(""), errNo(errno) { - errNo = errno; auto hf = hintfmt(args...); err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 6e69bdce2..11dbef9db 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -103,7 +103,7 @@ class hintformat public: hintformat(const string &format) :fmt(format) { - fmt.exceptions(boost::io::all_error_bits ^ + fmt.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit ^ boost::io::too_few_args_bit); } diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 7e53f17c6..d33bd7c1f 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -370,7 +370,7 @@ namespace nix { // constructing without access violation. ErrPos ep(invalid); - + // assignment without access violation. ep = invalid; diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 55d02bcf9..6c4c5ab74 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -4,6 +4,7 @@ #include <list> #include <set> +#include <string> #include <map> #include <vector> diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 377ae03a8..99cc0cdec 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -161,8 +161,14 @@ static int _main(int argc, char * * argv) std::optional<StorePath> storePath; if (args.size() == 2) { expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); + const auto method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + storePath = store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); if (store->isValidPath(*storePath)) hash = *expectedHash; else diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 9092dbd80..7981bbbdd 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -195,10 +195,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) /* Hack to support caching in `nix-prefetch-url'. */ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { - auto recursive = FileIngestionMethod::Flat; + auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") recursive = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::Recursive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -209,7 +209,13 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) string hash = *i++; string name = *i++; - cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(recursive, Hash::parseAny(hash, hashAlgo), name))); + cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), + }, + {}, + }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 023ffa4ed..86616d66b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -69,14 +69,20 @@ struct CmdAddToStore : MixDryRun, StoreCommand } ValidPathInfo info { - store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + *store, + StorePathDescriptor { + .name = *namePart, + .info = FixedOutputInfo { + { + .method = std::move(ingestionMethod), + .hash = std::move(hash), + }, + {}, + }, + }, narHash, }; info.narSize = sink.s->size(); - info.ca = std::optional { FixedOutputHash { - .method = ingestionMethod, - .hash = hash, - } }; if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index fc41da9e4..510df7504 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -91,7 +91,7 @@ struct CmdBundle : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); arg->attrs->sort(); - + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 7737f6d91..7695c98f8 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -55,19 +55,15 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON StringMap rewrites; - StorePathSet references; - bool hasSelfReference = false; + PathReferences<StorePath> refs; + refs.hasSelfReference = oldInfo->hasSelfReference; for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); + refs.references.insert(std::move(replacement)); } *sink.s = rewriteStrings(*sink.s, rewrites); @@ -78,16 +74,20 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON auto narHash = hashModuloSink.finish().first; ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + *store, + StorePathDescriptor { + .name = std::string { path.name() }, + .info = FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + std::move(refs), + }, + }, narHash, }; - info.references = std::move(references); - info.hasSelfReference = std::move(hasSelfReference); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 7ce4dfe4c..41a4857fc 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -130,12 +130,21 @@ struct ProfileManifest auto narHash = hashString(htSHA256, *sink.s); ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + *store, + StorePathDescriptor { + "profile", + FixedOutputInfo { + { + .method = FileIngestionMethod::Recursive, + .hash = narHash, + }, + { references }, + }, + }, narHash, }; info.references = std::move(references); info.narSize = sink.s->size(); - info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; auto source = StringSource { *sink.s }; store->addToStore(info, source); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 26f755fd9..d189a2fd3 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -73,14 +73,14 @@ struct CmdVerify : StorePathsCommand ThreadPool pool; - auto doPath = [&](const Path & storePath) { + auto doPath = [&](const StorePath & storePath) { try { checkInterrupt(); MaintainCount<std::atomic<size_t>> mcActive(active); update(); - auto info = store->queryPathInfo(store->parseStorePath(storePath)); + auto info = store->queryPathInfo(storePath); // Note: info->path can be different from storePath // for binary cache stores when using --all (since we @@ -178,7 +178,7 @@ struct CmdVerify : StorePathsCommand }; for (auto & storePath : storePaths) - pool.enqueue(std::bind(doPath, store->printStorePath(storePath))); + pool.enqueue(std::bind(doPath, storePath)); pool.process(); |