From 507aa48739f23f9a16c8b7079bfa6fc1806be78e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Jun 2020 18:41:33 +0000 Subject: WIP: Make Hash always store a valid hash type --- src/libstore/binary-cache-store.cc | 6 +++--- src/libstore/build.cc | 12 +++++++----- src/libstore/builtins/fetchurl.cc | 2 +- src/libstore/daemon.cc | 4 ++-- src/libstore/derivations.cc | 2 +- src/libstore/export-import.cc | 4 ++-- src/libstore/legacy-ssh-store.cc | 4 ++-- src/libstore/local-store.cc | 26 +++++++++++++------------- src/libstore/nar-info-disk-cache.cc | 4 ++-- src/libstore/nar-info.cc | 27 +++++++++++++-------------- src/libstore/nar-info.hh | 2 +- src/libstore/references.cc | 8 ++++---- src/libstore/references.hh | 3 +-- src/libstore/remote-store.cc | 2 +- src/libstore/store-api.cc | 8 ++++---- src/libstore/store-api.hh | 3 ++- 16 files changed, 59 insertions(+), 58 deletions(-) (limited to 'src/libstore') diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 9f52ddafa..98a3eaebb 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -181,7 +181,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), duration); - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : @@ -338,7 +338,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - Hash h; + std::optional h; if (method == FileIngestionMethod::Recursive) { dumpPath(srcPath, sink, filter); h = hashString(hashAlgo, *sink.s); @@ -348,7 +348,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath h = hashString(hashAlgo, s); } - ValidPathInfo info(makeFixedOutputPath(method, h, name)); + ValidPathInfo info(makeFixedOutputPath(method, *h, name)); auto source = StringSource { *sink.s }; addToStore(info, source, repair, CheckSigs, nullptr); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 82a2ab831..80351a675 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -3730,8 +3730,8 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive - ? hashPath(*i.second.hash->hash.type, actualPath).first - : hashFile(*i.second.hash->hash.type, actualPath); + ? hashPath(i.second.hash->hash.type, actualPath).first + : hashFile(i.second.hash->hash.type, actualPath); auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); @@ -3777,8 +3777,10 @@ void DerivationGoal::registerOutputs() time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ debug("scanning for references inside '%1%'", path); - HashResult hash; - auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash)); + // HashResult hash; + auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths)); + auto references = worker.store.parseStorePathSet(pathSetAndHash.first); + HashResult hash = pathSetAndHash.second; if (buildMode == bmCheck) { if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue; @@ -4997,7 +4999,7 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(*info->narHash.type, store.printStorePath(path)); + HashResult current = hashPath(info->narHash->type, store.printStorePath(path)); Hash nullHash(htSHA256); res = info->narHash == nullHash || info->narHash == current.first; } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 1cfe4a46a..f3827684b 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashTypeOpt(getAttr("outputHashAlgo")); auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e370e278c..296bfad5d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -314,7 +314,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash->to_string(Base16, false); break; } @@ -646,7 +646,7 @@ static void performOp(TunnelLogger * logger, ref store, if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); + << info->narHash->to_string(Base16, false); writeStorePaths(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a79b78db6..53bd4da2a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,7 +9,7 @@ namespace nix { std::string DerivationOutputHash::printMethodAlgo() const { - return makeFileIngestionPrefix(method) + printHashType(*hash.type); + return makeFileIngestionPrefix(method) + printHashType(hash.type); } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 57b7e9590..9b8cc5c3a 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -55,9 +55,9 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); - if (hash != info->narHash && info->narHash != Hash(*info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash->type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 45c70fad6..d3880459c 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -113,7 +113,7 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); - info->narHash = s.empty() ? Hash() : Hash(s); + info->narHash = s.empty() ? std::optional{} : Hash(s); conn->from >> info->ca; info->sigs = readStrings(conn->from); } @@ -139,7 +139,7 @@ struct LegacySSHStore : public Store << cmdAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c6b55ff7c..aa2ee5b30 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -586,7 +586,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -686,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash->to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -897,7 +897,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash && i.narHash->type == htSHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1010,7 +1010,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1067,12 +1067,12 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam the path in the database. We may just have computed it above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ - HashResult hash; - if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); - hash.second = dump.size(); - } else - hash = hashPath(htSHA256, realPath); + HashResult hash = method == FileIngestionMethod::Recursive + ? HashResult { + hashAlgo == htSHA256 ? h : hashString(htSHA256, dump), + dump.size(), + } + : hashPath(htSHA256, realPath); optimisePath(realPath); // FIXME: combine with hashPath() @@ -1255,9 +1255,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) std::unique_ptr hashSink; if (info->ca == "" || !info->references.count(info->path)) - hashSink = std::make_unique(*info->narHash.type); + hashSink = std::make_unique(info->narHash->type); else - hashSink = std::make_unique(*info->narHash.type, std::string(info->path.hashPart())); + hashSink = std::make_unique(info->narHash->type, std::string(info->path.hashPart())); dumpPath(Store::toRealPath(i), *hashSink); auto current = hashSink->finish(); @@ -1266,7 +1266,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) logError({ .name = "Invalid hash - path modified", .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)) + printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true)) }); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 552970248..6036b905e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -230,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash->to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index bb4448c90..fb84b0410 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -7,15 +7,14 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & : ValidPathInfo(StorePath(StorePath::dummy)) // FIXME: hack { auto corrupt = [&]() { - throw Error("NAR info file '%1%' is corrupt", whence); + return Error("NAR info file '%1%' is corrupt", whence); }; auto parseHashField = [&](const string & s) { try { return Hash(s); } catch (BadHash &) { - corrupt(); - return Hash(); // never reached + throw corrupt(); } }; @@ -25,12 +24,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) corrupt(); + if (colon == std::string::npos) throw corrupt(); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) corrupt(); + if (eol == std::string::npos) throw corrupt(); std::string value(s, colon + 2, eol - colon - 2); @@ -45,16 +44,16 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) corrupt(); + if (!string2Int(value, fileSize)) throw corrupt(); } else if (name == "NarHash") narHash = parseHashField(value); else if (name == "NarSize") { - if (!string2Int(value, narSize)) corrupt(); + if (!string2Int(value, narSize)) throw corrupt(); } else if (name == "References") { auto refs = tokenizeString(value, " "); - if (!references.empty()) corrupt(); + if (!references.empty()) throw corrupt(); for (auto & r : refs) references.insert(StorePath(r)); } @@ -67,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (!ca.empty()) corrupt(); + if (!ca.empty()) throw corrupt(); ca = value; } @@ -76,7 +75,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & if (compression == "") compression = "bzip2"; - if (!havePath || url.empty() || narSize == 0 || !narHash) corrupt(); + if (!havePath || url.empty() || narSize == 0 || !narHash) throw corrupt(); } std::string NarInfo::to_string(const Store & store) const @@ -86,11 +85,11 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; + assert(fileHash && fileHash->type == htSHA256); + res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + assert(narHash && narHash->type == htSHA256); + res += "NarHash: " + narHash->to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 373c33427..eff19f0ef 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -10,7 +10,7 @@ struct NarInfo : ValidPathInfo { std::string url; std::string compression; - Hash fileHash; + std::optional fileHash; uint64_t fileSize = 0; std::string system; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a10d536a3..4733bc388 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -79,8 +79,8 @@ void RefScanSink::operator () (const unsigned char * data, size_t len) } -PathSet scanForReferences(const string & path, - const PathSet & refs, HashResult & hash) +std::pair scanForReferences(const string & path, + const PathSet & refs) { RefScanSink sink; std::map backMap; @@ -112,9 +112,9 @@ PathSet scanForReferences(const string & path, found.insert(j->second); } - hash = sink.hashSink.finish(); + auto hash = sink.hashSink.finish(); - return found; + return std::pair(found, hash); } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index c38bdd720..598a3203a 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -5,8 +5,7 @@ namespace nix { -PathSet scanForReferences(const Path & path, const PathSet & refs, - HashResult & hash); +std::pair scanForReferences(const Path & path, const PathSet & refs); struct RewritingSink : Sink { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f5f2ab7fd..c206a4538 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -462,7 +462,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << wopAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 982fc22b6..c74aeec48 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -430,7 +430,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash->to_string(Base16, false) + "\n"; s += (format("%1%\n") % info->narSize).str(); } @@ -462,7 +462,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string(hashBase, true)) + .attr("narHash", info->narHash->to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -505,7 +505,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); + jsonPath.attr("downloadHash", narInfo->fileHash->to_string(Base32, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -746,7 +746,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash->to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a05048290..de9b6a791 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -116,7 +116,8 @@ struct ValidPathInfo { StorePath path; std::optional deriver; - Hash narHash; + // TODO document this + std::optional narHash; StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown -- cgit v1.2.3 From 1be279af2622b196cc4630c48254adc96071c7e9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 2 Jul 2020 21:44:18 +0000 Subject: Fix Narinfo corruption detection bug The aim of this check was just to ensure each key occurs once. --- src/libstore/nar-info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore') diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index c403d4bec..ca471463c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -66,7 +66,7 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (!value.empty()) throw corrupt(); + if (ca) throw corrupt(); // FIXME: allow blank ca or require skipping field? ca = parseContentAddressOpt(value); } -- cgit v1.2.3