diff options
29 files changed, 278 insertions, 199 deletions
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index a4937e722..9055f59a1 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,7 +1,7 @@ #include "get-drvs.hh" #include "util.hh" #include "eval-inline.hh" -#include "derivations.hh" +#include "store-api.hh" #include <cstring> #include <regex> diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f5fbd3fa6..3830d8107 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -776,7 +776,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { .path = std::move(outPath), - .hash = DerivationOutputHash { + .hash = FixedOutputHash { .method = ingestionMethod, .hash = std::move(h), }, @@ -795,7 +795,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drv.outputs.insert_or_assign(i, DerivationOutput { .path = StorePath::dummy, - .hash = std::optional<DerivationOutputHash> {}, + .hash = std::optional<FixedOutputHash> {}, }); } @@ -807,7 +807,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drv.outputs.insert_or_assign(i, DerivationOutput { .path = std::move(outPath), - .hash = std::optional<DerivationOutputHash>(), + .hash = std::optional<FixedOutputHash>(), }); } } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index efa2e9576..301e8c5dd 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -1,6 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" -#include "derivations.hh" +#include "store-api.hh" namespace nix { diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index ac83d52b9..f5356f0af 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -70,7 +70,10 @@ DownloadFileResult downloadFile( ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name)); info.narHash = hashString(htSHA256, *sink.s); info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash); + info.ca = FixedOutputHash { + .method = FileIngestionMethod::Flat, + .hash = hash, + }; auto source = StringSource { *sink.s }; store->addToStore(info, source, NoRepair, NoCheckSigs); storePath = std::move(info.path); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index b80b5c251..697c3fcff 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -297,7 +297,7 @@ public: GoalPtr makeDerivationGoal(const StorePath & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr<DerivationGoal> makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode = bmNormal); - GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<std::string> ca = std::nullopt); + GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); /* Remove a dead goal. */ void removeGoal(GoalPtr goal); @@ -3714,7 +3714,7 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::string ca; + std::optional<ContentAddress> ca; if (fixedOutput) { @@ -3764,7 +3764,10 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = makeFixedOutputCA(i.second.hash->method, h2); + ca = FixedOutputHash { + .method = i.second.hash->method, + .hash = h2, + }; } /* Get rid of all weird permissions. This also checks that @@ -3837,7 +3840,10 @@ void DerivationGoal::registerOutputs() info.ca = ca; worker.store.signPathInfo(info); - if (!info.references.empty()) info.ca.clear(); + if (!info.references.empty()) { + // FIXME don't we have an experimental feature for fixed output with references? + info.ca = {}; + } infos.emplace(i.first, std::move(info)); } @@ -4295,10 +4301,10 @@ private: GoalState state; /* Content address for recomputing store path */ - std::optional<std::string> ca; + std::optional<ContentAddress> ca; public: - SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<std::string> ca = std::nullopt); + SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); ~SubstitutionGoal(); void timedOut(Error && ex) override { abort(); }; @@ -4328,7 +4334,7 @@ public: }; -SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<std::string> ca) +SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca) : Goal(worker) , storePath(storePath) , repair(repair) @@ -4408,7 +4414,7 @@ void SubstitutionGoal::tryNext() subs.pop_front(); auto subPath = storePath; - if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) { + if (ca) { subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); @@ -4537,7 +4543,7 @@ void SubstitutionGoal::tryToRun() PushActivity pact(act.id); auto subPath = storePath; - if (ca && (hasPrefix(*ca, "fixed:") || hasPrefix(*ca, "text:"))) { + if (ca) { subPath = sub->makeFixedOutputPathFromCA(storePath.name(), *ca); if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); @@ -4677,7 +4683,7 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath } -GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<std::string> ca) +GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca) { GoalPtr goal = substitutionGoals[path].lock(); // FIXME if (!goal) { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc new file mode 100644 index 000000000..3d753836f --- /dev/null +++ b/src/libstore/content-address.cc @@ -0,0 +1,85 @@ +#include "content-address.hh" + +namespace nix { + +std::string FixedOutputHash::printMethodAlgo() const { + return makeFileIngestionPrefix(method) + printHashType(*hash.type); +} + +std::string makeFileIngestionPrefix(const FileIngestionMethod m) { + switch (m) { + case FileIngestionMethod::Flat: + return ""; + case FileIngestionMethod::Recursive: + return "r:"; + default: + throw Error("impossible, caught both cases"); + } +} + +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) +{ + return "fixed:" + + makeFileIngestionPrefix(method) + + hash.to_string(Base32, true); +} + +// FIXME Put this somewhere? +template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; +template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; + +std::string renderContentAddress(ContentAddress ca) { + return std::visit(overloaded { + [](TextHash th) { + return "text:" + th.hash.to_string(Base32, true); + }, + [](FixedOutputHash fsh) { + return makeFixedOutputCA(fsh.method, fsh.hash); + } + }, ca); +} + +ContentAddress parseContentAddress(std::string_view rawCa) { + auto prefixSeparator = rawCa.find(':'); + if (prefixSeparator != string::npos) { + auto prefix = string(rawCa, 0, prefixSeparator); + if (prefix == "text") { + auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos); + Hash hash = Hash(string(hashTypeAndHash)); + if (*hash.type != htSHA256) { + throw Error("parseContentAddress: the text hash should have type SHA256"); + } + return TextHash { hash }; + } else if (prefix == "fixed") { + // This has to be an inverse of makeFixedOutputCA + auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos); + if (methodAndHash.substr(0,2) == "r:") { + std::string_view hashRaw = methodAndHash.substr(2,string::npos); + return FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = Hash(string(hashRaw)), + }; + } else { + std::string_view hashRaw = methodAndHash; + return FixedOutputHash { + .method = FileIngestionMethod::Flat, + .hash = Hash(string(hashRaw)), + }; + } + } else { + throw Error("parseContentAddress: format not recognized; has to be text or fixed"); + } + } else { + throw Error("Not a content address because it lacks an appropriate prefix"); + } +}; + +std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt); +}; + +std::string renderContentAddress(std::optional<ContentAddress> ca) { + return ca ? renderContentAddress(*ca) : ""; +} + +} diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh new file mode 100644 index 000000000..ba4797f5b --- /dev/null +++ b/src/libstore/content-address.hh @@ -0,0 +1,56 @@ +#pragma once + +#include <variant> +#include "hash.hh" + +namespace nix { + +enum struct FileIngestionMethod : uint8_t { + Flat = false, + Recursive = true +}; + +struct TextHash { + Hash hash; +}; + +/// Pair of a hash, and how the file system was ingested +struct FixedOutputHash { + FileIngestionMethod method; + Hash hash; + std::string printMethodAlgo() const; +}; + +/* + We've accumulated several types of content-addressed paths over the years; + fixed-output derivations support multiple hash algorithms and serialisation + methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms: + + * ‘text:sha256:<sha256 hash of file contents>’: For paths + computed by makeTextPath() / addTextToStore(). + + * ‘fixed:<r?>:<ht>:<h>’: For paths computed by + makeFixedOutputPath() / addToStore(). +*/ +typedef std::variant< + TextHash, // for paths computed by makeTextPath() / addTextToStore + FixedOutputHash // for path computed by makeFixedOutputPath +> ContentAddress; + +/* Compute the prefix to the hash algorithm which indicates how the files were + ingested. */ +std::string makeFileIngestionPrefix(const FileIngestionMethod m); + +/* Compute the content-addressability assertion (ValidPathInfo::ca) + for paths created by makeFixedOutputPath() / addToStore(). */ +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); + +std::string renderContentAddress(ContentAddress ca); + +std::string renderContentAddress(std::optional<ContentAddress> ca); + +ContentAddress parseContentAddress(std::string_view rawCa); + +std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt); + +} diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index c613ab47d..c2c8612d1 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -658,7 +658,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << info->ca; + << renderContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -716,7 +716,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, info.references = readStorePaths<StorePathSet>(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); - from >> info.ca >> repair >> dontCheckSigs; + info.ca = parseContentAddressOpt(readString(from)); + from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a79b78db6..6c49075ba 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -8,11 +8,6 @@ namespace nix { -std::string DerivationOutputHash::printMethodAlgo() const { - return makeFileIngestionPrefix(method) + printHashType(*hash.type); -} - - const StorePath & BasicDerivation::findOutput(const string & id) const { auto i = outputs.find(id); @@ -113,7 +108,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - std::optional<DerivationOutputHash> fsh; + std::optional<FixedOutputHash> fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { @@ -121,7 +116,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream hashAlgo = string(hashAlgo, 2); } const HashType hashType = parseHashType(hashAlgo); - fsh = DerivationOutputHash { + fsh = FixedOutputHash { .method = std::move(method), .hash = Hash(hash, hashType), }; @@ -411,7 +406,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) auto hashAlgo = readString(in); const auto hash = readString(in); - std::optional<DerivationOutputHash> fsh; + std::optional<FixedOutputHash> fsh; if (hashAlgo != "") { auto method = FileIngestionMethod::Flat; if (string(hashAlgo, 0, 2) == "r:") { @@ -419,7 +414,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) hashAlgo = string(hashAlgo, 2); } const HashType hashType = parseHashType(hashAlgo); - fsh = DerivationOutputHash { + fsh = FixedOutputHash { .method = std::move(method), .hash = Hash(hash, hashType), }; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 7b677ca49..68c53c1ff 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,8 +1,9 @@ #pragma once +#include "path.hh" #include "types.hh" #include "hash.hh" -#include "store-api.hh" +#include "content-address.hh" #include <map> @@ -12,18 +13,10 @@ namespace nix { /* Abstract syntax of derivations. */ -/// Pair of a hash, and how the file system was ingested -struct DerivationOutputHash { - FileIngestionMethod method; - Hash hash; - std::string printMethodAlgo() const; -}; - struct DerivationOutput { StorePath path; - std::optional<DerivationOutputHash> hash; /* hash used for expected hash computation */ - void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; + std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */ }; typedef std::map<string, DerivationOutput> DerivationOutputs; @@ -76,6 +69,7 @@ struct Derivation : BasicDerivation class Store; +enum RepairFlag : bool { NoRepair = false, Repair = true }; /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(ref<Store> store, diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 45c70fad6..5657aa593 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -114,7 +114,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - conn->from >> info->ca; + info->ca = parseContentAddressOpt(readString(conn->from)); info->sigs = readStrings<StringSet>(conn->from); } @@ -146,7 +146,7 @@ struct LegacySSHStore : public Store << info.narSize << info.ultimate << info.sigs - << info.ca; + << renderContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 39065e600..659f80e77 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -580,7 +580,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs) { - if (info.ca != "" && !info.isContentAddressed(*this)) + if (info.ca.has_value() && !info.isContentAddressed(*this)) throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", printStorePath(info.path)); @@ -592,7 +592,7 @@ uint64_t LocalStore::addValidPath(State & state, (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (info.ca, !info.ca.empty()) + (renderContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = sqlite3_last_insert_rowid(state.db); @@ -666,7 +666,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString<StringSet>(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = s; + if (s) info->ca = parseContentAddressOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -689,7 +689,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (info.ca, !info.ca.empty()) + (renderContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } @@ -851,7 +851,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst auto subPath(path.first); // recompute store path so that we can use a different store root - if (path.second && (hasPrefix(*path.second, "fixed:") || hasPrefix(*path.second, "text:"))) { + if (path.second) { subPath = makeFixedOutputPathFromCA(path.first.name(), *path.second); if (sub->storeDir == storeDir) assert(subPath == path.first); @@ -997,15 +997,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); - if (info.ca != "" && - !((hasPrefix(info.ca, "text:") && !info.references.count(info.path)) - || info.references.empty())) + // text hashing has long been allowed to have non-self-references because it is used for drv files. + bool refersToSelf = info.references.count(info.path) > 0; + if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf)) settings.requireExperimentalFeature("ca-references"); /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr<AbstractHashSink> hashSink; - if (info.ca == "" || !info.references.count(info.path)) + if (!info.ca.has_value() || !info.references.count(info.path)) hashSink = std::make_unique<HashSink>(htSHA256); else hashSink = std::make_unique<HashModuloSink>(htSHA256, std::string(info.path.hashPart())); @@ -1091,7 +1091,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam ValidPathInfo info(dstPath); info.narHash = hash.first; info.narSize = hash.second; - info.ca = makeFixedOutputCA(method, h); + info.ca = FixedOutputHash { .method = method, .hash = h }; registerValidPath(info); } @@ -1155,7 +1155,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, info.narHash = narHash; info.narSize = sink.s->size(); info.references = references; - info.ca = "text:" + hash.to_string(Base32, true); + info.ca = TextHash { .hash = hash }; registerValidPath(info); } @@ -1266,7 +1266,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); std::unique_ptr<AbstractHashSink> hashSink; - if (info->ca == "" || !info->references.count(info->path)) + if (!info->ca || !info->references.count(info->path)) hashSink = std::make_unique<HashSink>(*info->narHash.type); else hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart())); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index a01e81b78..3256a48e0 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -108,28 +108,11 @@ void Store::computeFSClosure(const StorePath & startPath, } -std::optional<std::string> getDerivationCA(const BasicDerivation & drv) +std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv) { - auto outputHashMode = drv.env.find("outputHashMode"); - auto outputHash = drv.env.find("outputHash"); - - std::optional<HashType> ht = std::nullopt; - auto outputHashAlgo = drv.env.find("outputHashAlgo"); - if (outputHashAlgo != drv.env.end()) - ht = parseHashTypeOpt(outputHashAlgo->second); - - if (outputHashMode != drv.env.end() && outputHash != drv.env.end()) { - auto h = Hash(outputHash->second, ht); - FileIngestionMethod ingestionMethod; - if (outputHashMode->second == "recursive") - ingestionMethod = FileIngestionMethod::Recursive; - else if (outputHashMode->second == "flat") - ingestionMethod = FileIngestionMethod::Flat; - else - throw Error("unknown outputHashMode: '%s'", outputHashMode->second); - return makeFixedOutputCA(ingestionMethod, h); - } - + auto out = drv.outputs.find("out"); + if (out != drv.outputs.end()) + return out->second.hash; return std::nullopt; } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 552970248..012dea6ea 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ public: narInfo->deriver = StorePath(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = queryNAR.getStr(11); + narInfo->ca = parseContentAddressOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ public: (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (info->ca) + (renderContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index bb4448c90..04550ed97 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -67,8 +67,9 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (!ca.empty()) corrupt(); - ca = value; + if (ca) corrupt(); + // FIXME: allow blank ca or require skipping field? + ca = parseContentAddressOpt(value); } pos = eol + 1; @@ -104,8 +105,8 @@ std::string NarInfo::to_string(const Store & store) const for (auto sig : sigs) res += "Sig: " + sig + "\n"; - if (!ca.empty()) - res += "CA: " + ca + "\n"; + if (ca) + res += "CA: " + renderContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index d24d1eb4f..0b8e8d031 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,4 +1,4 @@ -#include "derivations.hh" +#include "store-api.hh" #include <nlohmann/json_fwd.hpp> diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 91962d114..a07186323 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,5 +1,6 @@ #pragma once +#include "content-address.hh" #include "types.hh" namespace nix { @@ -62,16 +63,11 @@ public: typedef std::set<StorePath> StorePathSet; typedef std::vector<StorePath> StorePaths; -typedef std::map<StorePath, std::optional<std::string>> StorePathCAMap; +typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; -enum struct FileIngestionMethod : uint8_t { - Flat = false, - Recursive = true -}; - struct StorePathWithOutputs { StorePath path; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f6158685c..32c736d08 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,15 +38,12 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::map<StorePath, std::optional<std::string>> readStorePathCAMap(const Store & store, Source & from) +StorePathCAMap readStorePathCAMap(const Store & store, Source & from) { StorePathCAMap paths; auto count = readNum<size_t>(from); - while (count--) { - auto path = readString(from); - auto ca = readString(from); - paths.insert_or_assign(store.parseStorePath(path), !ca.empty() ? std::optional(ca) : std::nullopt); - } + while (count--) + paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); return paths; } @@ -55,7 +52,7 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & out << paths.size(); for (auto & i : paths) { out << store.printStorePath(i.first); - out << (i.second ? *i.second : ""); + out << renderContentAddress(i.second); } } @@ -407,7 +404,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings<StringSet>(conn->from); - conn->from >> info->ca; + info->ca = parseContentAddressOpt(readString(conn->from)); } } callback(std::move(info)); @@ -491,7 +488,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << info.ca + << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 0458fd9a6..c4fc1c861 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -192,18 +192,22 @@ StorePath Store::makeFixedOutputPath( } } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, std::string ca, +// FIXME Put this somewhere? +template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; +template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; + +StorePath Store::makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, const StorePathSet & references, bool hasSelfReference) const { - if (hasPrefix(ca, "fixed:")) { - FileIngestionMethod ingestionMethod { ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(ca, ingestionMethod == FileIngestionMethod::Recursive ? 8 : 6)); - return makeFixedOutputPath(ingestionMethod, hash, name, references, hasSelfReference); - } else if (hasPrefix(ca, "text:")) { - Hash hash(std::string(ca, 5)); - return makeTextPath(name, hash, references); - } else - throw Error("'%s' is not a valid ca", ca); + // New template + return std::visit(overloaded { + [&](TextHash th) { + return makeTextPath(name, th.hash, references); + }, + [&](FixedOutputHash fsh) { + return makeFixedOutputPath(fsh.method, fsh.hash, name, references, hasSelfReference); + } + }, ca); } StorePath Store::makeTextPath(std::string_view name, const Hash & hash, @@ -484,8 +488,8 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store jsonRefs.elem(printStorePath(ref)); } - if (info->ca != "") - jsonPath.attr("ca", info->ca); + if (info->ca) + jsonPath.attr("ca", renderContentAddress(info->ca)); std::pair<uint64_t, uint64_t> closureSizes; @@ -595,9 +599,9 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, uint64_t total = 0; // recompute store path on the chance dstStore does it differently - if (info->isContentAddressed(*srcStore) && info->references.empty()) { + if (info->ca && info->references.empty()) { auto info2 = make_ref<ValidPathInfo>(*info); - info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), info->ca); + info2->path = dstStore->makeFixedOutputPathFromCA(info->path.name(), *info->ca); if (dstStore->storeDir == srcStore->storeDir) assert(info->path == info2->path); info = info2; @@ -674,8 +678,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; - if (info->isContentAddressed(*srcStore) && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); + if (info->ca && info->references.empty()) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -702,8 +706,8 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor auto info = srcStore->queryPathInfo(storePath); auto storePathForDst = storePath; - if (info->isContentAddressed(*srcStore) && info->references.empty()) { - storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), info->ca); + if (info->ca && info->references.empty()) { + storePathForDst = dstStore->makeFixedOutputPathFromCA(storePath.name(), *info->ca); if (dstStore->storeDir == srcStore->storeDir) assert(storePathForDst == storePath); if (storePathForDst != storePath) @@ -807,41 +811,31 @@ void ValidPathInfo::sign(const Store & store, const SecretKey & secretKey) sigs.insert(secretKey.signDetached(fingerprint(store))); } - bool ValidPathInfo::isContentAddressed(const Store & store) const { - auto warn = [&]() { - logWarning( - ErrorInfo{ - .name = "Path not content-addressed", - .hint = hintfmt("path '%s' claims to be content-addressed but isn't", store.printStorePath(path)) - }); - }; + if (! ca) return false; - if (hasPrefix(ca, "text:")) { - Hash hash(ca.substr(5)); - if (store.makeTextPath(path.name(), hash, references) == path) - return true; - else - warn(); - } - - else if (hasPrefix(ca, "fixed:")) { - FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; - Hash hash(ca.substr(recursive == FileIngestionMethod::Recursive ? 8 : 6)); - auto refs = references; - bool hasSelfReference = false; - if (refs.count(path)) { - hasSelfReference = true; - refs.erase(path); + auto caPath = std::visit(overloaded { + [&](TextHash th) { + return store.makeTextPath(path.name(), th.hash, references); + }, + [&](FixedOutputHash fsh) { + auto refs = references; + bool hasSelfReference = false; + if (refs.count(path)) { + hasSelfReference = true; + refs.erase(path); + } + return store.makeFixedOutputPath(fsh.method, fsh.hash, path.name(), refs, hasSelfReference); } - if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path) - return true; - else - warn(); - } + }, *ca); + + bool res = caPath == path; + + if (!res) + printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); - return false; + return res; } @@ -872,25 +866,6 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFileIngestionPrefix(const FileIngestionMethod m) { - switch (m) { - case FileIngestionMethod::Flat: - return ""; - case FileIngestionMethod::Recursive: - return "r:"; - default: - throw Error("impossible, caught both cases"); - } -} - -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) -{ - return "fixed:" - + makeFileIngestionPrefix(method) - + hash.to_string(Base32, true); -} - - } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2b12879e3..c4a8d3890 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,12 +2,14 @@ #include "path.hh" #include "hash.hh" +#include "content-address.hh" #include "serialise.hh" #include "crypto.hh" #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" #include "config.hh" +#include "derivations.hh" #include <atomic> #include <limits> @@ -17,6 +19,7 @@ #include <memory> #include <string> #include <chrono> +#include <variant> namespace nix { @@ -31,15 +34,12 @@ MakeError(SubstituterDisabled, Error); MakeError(NotInStore, Error); -struct BasicDerivation; -struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; class JSONPlaceholder; -enum RepairFlag : bool { NoRepair = false, Repair = true }; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; @@ -111,7 +111,6 @@ struct SubstitutablePathInfo typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; - struct ValidPathInfo { StorePath path; @@ -140,21 +139,11 @@ struct ValidPathInfo that a particular output path was produced by a derivation; the path then implies the contents.) - Ideally, the content-addressability assertion would just be a - Boolean, and the store path would be computed from - the name component, ‘narHash’ and ‘references’. However, - 1) we've accumulated several types of content-addressed paths - over the years; and 2) fixed-output derivations support - multiple hash algorithms and serialisation methods (flat file - vs NAR). Thus, ‘ca’ has one of the following forms: - - * ‘text:sha256:<sha256 hash of file contents>’: For paths - computed by makeTextPath() / addTextToStore(). - - * ‘fixed:<r?>:<ht>:<h>’: For paths computed by - makeFixedOutputPath() / addToStore(). + Ideally, the content-addressability assertion would just be a Boolean, + and the store path would be computed from the name component, ‘narHash’ + and ‘references’. However, we support many types of content addresses. */ - std::string ca; + std::optional<ContentAddress> ca; bool operator == (const ValidPathInfo & i) const { @@ -356,7 +345,7 @@ public: StorePath makeTextPath(std::string_view name, const Hash & hash, const StorePathSet & references = {}) const; - StorePath makeFixedOutputPathFromCA(std::string_view name, std::string ca, + StorePath makeFixedOutputPathFromCA(std::string_view name, ContentAddress ca, const StorePathSet & references = {}, bool hasSelfReference = false) const; @@ -846,18 +835,9 @@ std::optional<ValidPathInfo> decodeValidPathInfo( std::istream & str, bool hashGiven = false); -/* Compute the prefix to the hash algorithm which indicates how the files were - ingested. */ -std::string makeFileIngestionPrefix(const FileIngestionMethod m); - -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - - /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); -std::optional<std::string> getDerivationCA(const BasicDerivation & drv); +std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index d2944674f..5a3e33c08 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -69,7 +69,7 @@ template<class T> T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); -std::map<StorePath, std::optional<std::string>> readStorePathCAMap(const Store & store, Source & from); +StorePathCAMap readStorePathCAMap(const Store & store, Source & from); void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e49eb4569..c8fcdfed0 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -223,7 +223,7 @@ Hash newHashAllowEmpty(std::string hashStr, std::optional<HashType> ht) if (!ht) throw BadHash("empty hash requires explicit hash type"); Hash h(*ht); - warn("found empty hash, assuming '%s'", h.to_string(Base::SRI, true)); + warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); return h; } else return Hash(hashStr, ht); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 4e02aa2bf..7d81bf54f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -864,7 +864,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { } } @@ -952,7 +952,7 @@ static void opServe(Strings opFlags, Strings opArgs) info.references = readStorePaths<StorePathSet>(*store, in); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(in); - in >> info.ca; + info.ca = parseContentAddressOpt(readString(in)); if (info.narSize == 0) throw Error("narInfo is too old and missing the narSize field"); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 7abb82556..48a4722ae 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -70,7 +70,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart)); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(ingestionMethod, hash); + info.ca = std::optional { FixedOutputHash { + .method = ingestionMethod, + .hash = hash, + } }; if (!dryRun) { auto source = StringSource { *sink.s }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 8b85caf82..037987313 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -137,7 +137,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) auto shellOutPath = store->makeOutputPath("out", h, drvName); drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath, - .hash = DerivationOutputHash { + .hash = FixedOutputHash { .method = FileIngestionMethod::Flat, .hash = Hash { }, }, diff --git a/src/nix/hash.cc b/src/nix/hash.cc index f435192fc..b97c6d21f 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "hash.hh" +#include "content-address.hh" #include "legacy.hh" #include "shared.hh" #include "references.hh" diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 0ebb8f13b..fb36fc410 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -82,7 +82,10 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON if (hasSelfReference) info.references.insert(info.path); info.narHash = narHash; info.narSize = sink.s->size(); - info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash); + info.ca = FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = info.narHash, + }; if (!json) printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index fb7bacc4c..b89a44f83 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -115,7 +115,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << '\t'; Strings ss; if (info->ultimate) ss.push_back("ultimate"); - if (info->ca != "") ss.push_back("ca:" + info->ca); + if (info->ca) ss.push_back("ca:" + renderContentAddress(*info->ca)); for (auto & sig : info->sigs) ss.push_back(sig); std::cout << concatStringsSep(" ", ss); } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index d1aba08e3..bb5e4529b 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -87,7 +87,7 @@ struct CmdVerify : StorePathsCommand if (!noContents) { std::unique_ptr<AbstractHashSink> hashSink; - if (info->ca == "") + if (!info->ca) hashSink = std::make_unique<HashSink>(*info->narHash.type); else hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart())); |