diff options
Diffstat (limited to 'src/libstore')
27 files changed, 446 insertions, 278 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 357962007..583f9d8ac 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -137,7 +137,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource auto narInfo = make_ref<NarInfo>(info); narInfo->narSize = nar->size(); - narInfo->narHash = hashString(htSHA256, *nar); + narInfo->narHash = hashString(HashType::SHA256, *nar); if (info.narHash && info.narHash != narInfo->narHash) throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path)); @@ -172,16 +172,16 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource auto now1 = std::chrono::steady_clock::now(); auto narCompressed = compress(compression, *nar, parallelCompression); auto now2 = std::chrono::steady_clock::now(); - narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileHash = hashString(HashType::SHA256, *narCompressed); narInfo->fileSize = narCompressed->size(); auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count(); - printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", + printMsg(Verbosity::Talkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache", printStorePath(narInfo->path), narInfo->narSize, ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0), duration); - narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base::Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "br" ? ".br" : @@ -209,7 +209,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource // to a GC'ed file, so overwriting might be useful... if (fileExists(key)) return; - printMsg(lvlTalkative, "creating debuginfo link from '%s' to '%s'", key, target); + printMsg(Verbosity::Talkative, "creating debuginfo link from '%s' to '%s'", key, target); upsertFile(key, json.dump(), "application/json"); }; @@ -302,7 +302,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, { auto uri = getUri(); auto storePathS = printStorePath(storePath); - auto act = std::make_shared<Activity>(*logger, lvlTalkative, actQueryPathInfo, + auto act = std::make_shared<Activity>(*logger, Verbosity::Talkative, ActivityType::QueryPathInfo, fmt("querying info about '%s' on '%s'", storePathS, uri), Logger::Fields{storePathS, uri}); PushActivity pact(act->id); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f5c132a83..7f79d6a2a 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -107,7 +107,13 @@ typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; class Goal : public std::enable_shared_from_this<Goal> { public: - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; + enum struct ExitCode { + Busy, + Success, + Failed, + NoSubstituters, + IncompleteClosure, + }; protected: @@ -141,7 +147,7 @@ protected: Goal(Worker & worker) : worker(worker) { nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; + exitCode = ExitCode::Busy; } virtual ~Goal() @@ -361,8 +367,8 @@ public: { actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); + act.setExpected(ActivityType::Download, expectedDownloadSize + doneDownloadSize); + act.setExpected(ActivityType::CopyPath, expectedNarSize + doneNarSize); } }; @@ -395,13 +401,13 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) trace(format("waitee '%1%' done; %2% left") % waitee->name % waitees.size()); - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; + if (result == ExitCode::Failed || result == ExitCode::NoSubstituters || result == ExitCode::IncompleteClosure) ++nrFailed; - if (result == ecNoSubstituters) ++nrNoSubstituters; + if (result == ExitCode::NoSubstituters) ++nrNoSubstituters; - if (result == ecIncompleteClosure) ++nrIncompleteClosure; + if (result == ExitCode::IncompleteClosure) ++nrIncompleteClosure; - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { + if (waitees.empty() || (result == ExitCode::Failed && !settings.keepGoing)) { /* If we failed and keepGoing is not set, we remove all remaining waitees. */ @@ -421,8 +427,8 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) void Goal::amDone(ExitCode result) { trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); + assert(exitCode == ExitCode::Busy); + assert(result == ExitCode::Success || result == ExitCode::Failed || result == ExitCode::NoSubstituters || result == ExitCode::IncompleteClosure); exitCode = result; for (auto & i : waiters) { GoalPtr goal = i.lock(); @@ -672,7 +678,7 @@ HookInstance::HookInstance() Strings args = { std::string(baseNameOf(settings.buildHook.get())), - std::to_string(verbosity), + std::to_string((uint64_t)verbosity), }; execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); @@ -1401,7 +1407,7 @@ void DerivationGoal::started() { "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); fmt("building '%s'", worker.store.printStorePath(drvPath)); if (hook) msg += fmt(" on '%s'", machineName); - act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, + act = std::make_unique<Activity>(*logger, Verbosity::Info, ActivityType::Build, msg, Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); worker.updateProgress(); @@ -1457,6 +1463,20 @@ void DerivationGoal::tryToBuild() supported for local builds. */ bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); + auto started = [&]() { + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds); + fmt("building '%s'", worker.store.printStorePath(drvPath)); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique<Activity>(*logger, Verbosity::Info, ActivityType::Build, msg, + Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds}); + mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); + worker.updateProgress(); + }; + /* Is the build hook willing to accept this job? */ if (!buildLocally) { switch (tryBuildHook()) { @@ -1659,7 +1679,7 @@ void DerivationGoal::buildDone() registerOutputs(); if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, + Activity act(*logger, Verbosity::Info, ActivityType::PostBuildHook, fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{worker.store.printStorePath(drvPath)}); PushActivity pact(act.id); @@ -1694,7 +1714,7 @@ void DerivationGoal::buildDone() if (settings.verboseBuild) { printError("post-build-hook: " + currentLine); } else { - act.result(resPostBuildLogLine, currentLine); + act.result(ResultType::PostBuildLogLine, currentLine); } currentLine.clear(); } @@ -2097,7 +2117,7 @@ void DerivationGoal::startBuilder() /* Clean up the chroot directory automatically. */ autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir); - printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); + printMsg(Verbosity::Chatty, format("setting up chroot environment in '%1%'") % chrootRootDir); if (mkdir(chrootRootDir.c_str(), 0750) == -1) throw SysError(format("cannot create '%1%'") % chrootRootDir); @@ -2206,7 +2226,7 @@ void DerivationGoal::startBuilder() } if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) { - printMsg(lvlChatty, format("executing pre-build hook '%1%'") + printMsg(Verbosity::Chatty, format("executing pre-build hook '%1%'") % settings.preBuildHook); auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) : Strings({ worker.store.printStorePath(drvPath) }); @@ -2248,7 +2268,7 @@ void DerivationGoal::startBuilder() startDaemon(); /* Run the builder. */ - printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); + printMsg(Verbosity::Chatty, format("executing builder '%1%'") % drv->builder); /* Create the log file. */ Path logFile = openLogFile(); @@ -2484,8 +2504,8 @@ void DerivationGoal::initTmpDir() { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - auto hash = hashString(htSHA256, i.first); - string fn = ".attr-" + hash.to_string(Base32, false); + auto hash = hashString(HashType::SHA256, i.first); + string fn = ".attr-" + hash.to_string(Base::Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -2734,7 +2754,7 @@ struct RestrictedStore : public LocalFSStore { throw Error("queryPathFromHashPart"); } StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override { throw Error("addToStore"); } @@ -2747,7 +2767,7 @@ struct RestrictedStore : public LocalFSStore } StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); goal.addDependency(path); @@ -3688,14 +3708,11 @@ void DerivationGoal::registerOutputs() /* Check that fixed-output derivations produced the right outputs (i.e., the content hash should match the specified hash). */ - std::string ca; + std::optional<ContentAddress> ca; if (fixedOutput) { - FileIngestionMethod outputHashMode; Hash h; - i.second.parseHashInfo(outputHashMode, h); - - if (outputHashMode == FileIngestionMethod::Flat) { + if (i.second.hash->method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) throw BuildError( @@ -3705,20 +3722,22 @@ void DerivationGoal::registerOutputs() /* Check the hash. In hash mode, move the path produced by the derivation to its content-addressed location. */ - Hash h2 = outputHashMode == FileIngestionMethod::Recursive - ? hashPath(h.type, actualPath).first - : hashFile(h.type, actualPath); + Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive + ? hashPath(i.second.hash->hash.type, actualPath).first + : hashFile(i.second.hash->hash.type, actualPath); - auto dest = worker.store.makeFixedOutputPath(outputHashMode, h2, i.second.path.name()); + auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name()); - if (h != h2) { + if (i.second.hash->hash != h2) { /* Throw an error after registering the path as valid. */ worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); + worker.store.printStorePath(dest), + i.second.hash->hash.to_string(Base::SRI), + h2.to_string(Base::SRI))); Path actualDest = worker.store.Store::toRealPath(dest); @@ -3738,7 +3757,10 @@ void DerivationGoal::registerOutputs() else assert(worker.store.parseStorePath(path) == dest); - ca = makeFixedOutputCA(outputHashMode, h2); + ca = FileSystemHash { + .method = i.second.hash->method, + .hash = h2, + }; } /* Get rid of all weird permissions. This also checks that @@ -3811,7 +3833,10 @@ void DerivationGoal::registerOutputs() info.ca = ca; worker.store.signPathInfo(info); - if (!info.references.empty()) info.ca.clear(); + if (!info.references.empty()) { + // FIXME don't we have an experimental feature for fixed output with references? + info.ca = {}; + } infos.emplace(i.first, std::move(info)); } @@ -4163,7 +4188,7 @@ void DerivationGoal::flushLine() if (logTail.size() > settings.logLines) logTail.pop_front(); } - act->result(resBuildLogLine, currentLogLine); + act->result(ResultType::BuildLogLine, currentLogLine); } currentLogLine = ""; @@ -4190,7 +4215,7 @@ void DerivationGoal::addHashRewrite(const StorePath & path) auto h1 = std::string(((std::string_view) path.to_string()).substr(0, 32)); auto p = worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); + Hash(HashType::SHA256), path.name()); auto h2 = std::string(((std::string_view) p.to_string()).substr(0, 32)); deletePath(worker.store.printStorePath(p)); inputRewrites[h1] = h2; @@ -4203,7 +4228,7 @@ void DerivationGoal::done(BuildResult::Status status, const string & msg) { result.status = status; result.errorMsg = msg; - amDone(result.success() ? ecSuccess : ecFailed); + amDone(result.success() ? ExitCode::Success : ExitCode::Failed); if (result.status == BuildResult::TimedOut) worker.timedOut = true; if (result.status == BuildResult::PermanentFailure) @@ -4344,7 +4369,7 @@ void SubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - amDone(ecSuccess); + amDone(ExitCode::Success); return; } @@ -4369,7 +4394,7 @@ void SubstitutionGoal::tryNext() /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); + amDone(substituterFailed ? ExitCode::Failed : ExitCode::NoSubstituters); if (substituterFailed) { worker.failedSubstitutions++; @@ -4452,7 +4477,7 @@ void SubstitutionGoal::referencesValid() if (nrFailed > 0) { debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); + amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ExitCode::IncompleteClosure : ExitCode::Failed); return; } @@ -4490,7 +4515,7 @@ void SubstitutionGoal::tryToRun() /* Wake up the worker loop when we're done. */ Finally updateStats([this]() { outPipe.writeSide = -1; }); - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); + Activity act(*logger, ActivityType::Substitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); copyStorePath(ref<Store>(sub), ref<Store>(worker.store.shared_from_this()), @@ -4539,7 +4564,7 @@ void SubstitutionGoal::finished() worker.markContentsGood(storePath.clone()); - printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); + printMsg(Verbosity::Chatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); maintainRunningSubstitutions.reset(); @@ -4557,7 +4582,7 @@ void SubstitutionGoal::finished() worker.updateProgress(); - amDone(ecSuccess); + amDone(ExitCode::Success); } @@ -4576,9 +4601,9 @@ void SubstitutionGoal::handleEOF(int fd) Worker::Worker(LocalStore & store) - : act(*logger, actRealise) - , actDerivations(*logger, actBuilds) - , actSubstitutions(*logger, actCopyPaths) + : act(*logger, ActivityType::Realise) + , actDerivations(*logger, ActivityType::Builds) + , actSubstitutions(*logger, ActivityType::CopyPaths) , store(store) { /* Debugging: prevent recursive workers. */ @@ -4662,7 +4687,7 @@ void Worker::removeGoal(GoalPtr goal) topGoals.erase(goal); /* If a top-level goal failed, then kill all other goals (unless keepGoing was set). */ - if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) + if (goal->getExitCode() == Goal::ExitCode::Failed && !settings.keepGoing) topGoals.clear(); } @@ -4806,7 +4831,7 @@ void Worker::run(const Goals & _topGoals) void Worker::waitForInput() { - printMsg(lvlVomit, "waiting for children"); + printMsg(Verbosity::Vomit, "waiting for children"); /* Process output from the file descriptors attached to the children, namely log output and output path creation commands. @@ -4898,7 +4923,7 @@ void Worker::waitForInput() if (errno != EINTR) throw SysError("%s: read failed", goal->getName()); } else { - printMsg(lvlVomit, format("%1%: read %2% bytes") + printMsg(Verbosity::Vomit, format("%1%: read %2% bytes") % goal->getName() % rd); string data((char *) buffer.data(), rd); j->lastOutput = after; @@ -4907,7 +4932,7 @@ void Worker::waitForInput() } } - if (goal->getExitCode() == Goal::ecBusy && + if (goal->getExitCode() == Goal::ExitCode::Busy && 0 != settings.maxSilentTime && j->respectTimeouts && after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) @@ -4918,7 +4943,7 @@ void Worker::waitForInput() goal->timedOut(); } - else if (goal->getExitCode() == Goal::ecBusy && + else if (goal->getExitCode() == Goal::ExitCode::Busy && 0 != settings.buildTimeout && j->respectTimeouts && after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) @@ -4980,7 +5005,7 @@ bool Worker::pathContentsGood(const StorePath & path) res = false; else { HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); + Hash nullHash(HashType::SHA256); res = info->narHash == nullHash || info->narHash == current.first; } pathContentsGoodCache.insert_or_assign(path.clone(), res); @@ -5029,7 +5054,7 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, StorePathSet failed; for (auto & i : goals) { - if (i->getExitCode() != Goal::ecSuccess) { + if (i->getExitCode() != Goal::ExitCode::Success) { DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get()); if (i2) failed.insert(i2->getDrvPath()); else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath()); @@ -5074,7 +5099,7 @@ void LocalStore::ensurePath(const StorePath & path) worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) + if (goal->getExitCode() != Goal::ExitCode::Success) throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } @@ -5087,7 +5112,7 @@ void LocalStore::repairPath(const StorePath & path) worker.run(goals); - if (goal->getExitCode() != Goal::ecSuccess) { + if (goal->getExitCode() != Goal::ExitCode::Success) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 486babf14..b70e960f8 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; auto ht = parseHashType(getAttr("outputHashAlgo")); auto h = Hash(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index f1afdff69..45bfa6753 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -114,7 +114,13 @@ struct TunnelLogger : public Logger } StringSink buf; - buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; + buf << STDERR_START_ACTIVITY + << act + << (uint64_t) lvl + << (uint64_t) type + << s + << fields + << parent; enqueueMsg(*buf.s); } @@ -130,7 +136,10 @@ struct TunnelLogger : public Logger { if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; StringSink buf; - buf << STDERR_RESULT << act << type << fields; + buf << STDERR_RESULT + << act + << (uint64_t) type + << fields; enqueueMsg(*buf.s); } }; @@ -302,7 +311,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash.to_string(Base::Base16, false); break; } @@ -550,7 +559,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, clientSettings.maxBuildJobs = readInt(from); clientSettings.maxSilentTime = readInt(from); readInt(from); // obsolete useBuildHook - clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from); + clientSettings.verboseBuild = Verbosity::Error == (Verbosity) readInt(from); readInt(from); // obsolete logType readInt(from); // obsolete printBuildTrace clientSettings.buildCores = readInt(from); @@ -635,13 +644,13 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; to << (info->deriver ? store->printStorePath(*info->deriver) : "") - << info->narHash.to_string(Base16, false); + << info->narHash.to_string(Base::Base16, false); writeStorePaths(*store, to, info->references); to << info->registrationTime << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { to << info->ultimate << info->sigs - << info->ca; + << renderContentAddress(info->ca); } } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); @@ -695,11 +704,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto deriver = readString(from); if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.narHash = Hash(readString(from), htSHA256); + info.narHash = Hash(readString(from), HashType::SHA256); info.references = readStorePaths<StorePathSet>(*store, from); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); - from >> info.ca >> repair >> dontCheckSigs; + { + string caOptRaw; + from >> caOptRaw; + info.ca = parseCaOpt(caOptRaw); + } + from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) @@ -778,7 +792,7 @@ void processConnection( Finally finally([&]() { _isInterrupted = false; - prevLogger->log(lvlDebug, fmt("%d operations", opCount)); + prevLogger->log(Verbosity::Debug, fmt("%d operations", opCount)); }); if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c68e7b16b..f1569bf22 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -8,25 +8,6 @@ namespace nix { - -void DerivationOutput::parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const -{ - recursive = FileIngestionMethod::Flat; - string algo = hashAlgo; - - if (string(algo, 0, 2) == "r:") { - recursive = FileIngestionMethod::Recursive; - algo = string(algo, 2); - } - - HashType hashType = parseHashType(algo); - if (hashType == htUnknown) - throw Error("unknown hash algorithm '%s'", algo); - - hash = Hash(this->hash, hashType); -} - - BasicDerivation::BasicDerivation(const BasicDerivation & other) : platform(other.platform) , builder(other.builder) @@ -35,7 +16,7 @@ BasicDerivation::BasicDerivation(const BasicDerivation & other) { for (auto & i : other.outputs) outputs.insert_or_assign(i.first, - DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); + DerivationOutput(i.second.path.clone(), std::optional<FileSystemHash>(i.second.hash))); for (auto & i : other.inputSrcs) inputSrcs.insert(i.clone()); } @@ -142,6 +123,33 @@ static StringSet parseStrings(std::istream & str, bool arePaths) } +static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str) +{ + expect(str, ","); auto path = store.parseStorePath(parsePath(str)); + expect(str, ","); auto hashAlgo = parseString(str); + expect(str, ","); const auto hash = parseString(str); + expect(str, ")"); + + auto method = FileIngestionMethod::Flat; + std::optional<FileSystemHash> fsh; + if (hashAlgo != "") { + if (string(hashAlgo, 0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = string(hashAlgo, 2); + } + const HashType hashType = parseHashType(hashAlgo); + if (hashType == HashType::Unknown) + throw Error("unknown hash hashAlgorithm '%s'", hashAlgo); + fsh = FileSystemHash { + std::move(method), + Hash(hash, hashType), + }; + } + + return DerivationOutput(std::move(path), std::move(fsh)); +} + + static Derivation parseDerivation(const Store & store, const string & s) { Derivation drv; @@ -151,11 +159,7 @@ static Derivation parseDerivation(const Store & store, const string & s) /* Parse the list of outputs. */ while (!endOfList(str)) { expect(str, "("); std::string id = parseString(str); - expect(str, ","); auto path = store.parseStorePath(parsePath(str)); - expect(str, ","); auto hashAlgo = parseString(str); - expect(str, ","); auto hash = parseString(str); - expect(str, ")"); - drv.outputs.emplace(id, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); + drv.outputs.emplace(id, parseDerivationOutput(store, str)); } /* Parse the list of input derivations. */ @@ -275,8 +279,9 @@ string Derivation::unparse(const Store & store, bool maskOutputs, if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path)); - s += ','; printUnquotedString(s, i.second.hashAlgo); - s += ','; printUnquotedString(s, i.second.hash); + s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : ""); + s += ','; printUnquotedString(s, + i.second.hash ? i.second.hash->hash.to_string(Base::Base16, false) : ""); s += ')'; } @@ -332,7 +337,7 @@ bool BasicDerivation::isFixedOutput() const { return outputs.size() == 1 && outputs.begin()->first == "out" && - outputs.begin()->second.hash != ""; + outputs.begin()->second.hash; } @@ -364,9 +369,9 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput /* Return a fixed hash for fixed-output derivations. */ if (drv.isFixedOutput()) { DerivationOutputs::const_iterator i = drv.outputs.begin(); - return hashString(htSHA256, "fixed:out:" - + i->second.hashAlgo + ":" - + i->second.hash + ":" + return hashString(HashType::SHA256, "fixed:out:" + + i->second.hash->printMethodAlgo() + ":" + + i->second.hash->hash.to_string(Base::Base16, false) + ":" + store.printStorePath(i->second.path)); } @@ -380,10 +385,10 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, readDerivation(store, store.toRealPath(i.first)), false)).first; } - inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); + inputs2.insert_or_assign(h->second.to_string(Base::Base16, false), i.second); } - return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + return hashString(HashType::SHA256, drv.unparse(store, maskOutputs, &inputs2)); } @@ -409,6 +414,30 @@ StorePathSet BasicDerivation::outputPaths() const return paths; } +static DerivationOutput readDerivationOutput(Source & in, const Store & store) +{ + auto path = store.parseStorePath(readString(in)); + auto hashAlgo = readString(in); + const auto hash = readString(in); + + auto method = FileIngestionMethod::Flat; + std::optional<FileSystemHash> fsh; + if (hashAlgo != "") { + if (string(hashAlgo, 0, 2) == "r:") { + method = FileIngestionMethod::Recursive; + hashAlgo = string(hashAlgo, 2); + } + HashType hashType = parseHashType(hashAlgo); + if (hashType == HashType::Unknown) + throw Error("unknown hash hashAlgorithm '%s'", hashAlgo); + fsh = FileSystemHash { + std::move(method), + Hash(hash, hashType), + }; + } + + return DerivationOutput(std::move(path), std::move(fsh)); +} Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) { @@ -416,10 +445,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) auto nr = readNum<size_t>(in); for (size_t n = 0; n < nr; n++) { auto name = readString(in); - auto path = store.parseStorePath(readString(in)); - auto hashAlgo = readString(in); - auto hash = readString(in); - drv.outputs.emplace(name, DerivationOutput(std::move(path), std::move(hashAlgo), std::move(hash))); + auto output = readDerivationOutput(in, store); + drv.outputs.emplace(name, output); } drv.inputSrcs = readStorePaths<StorePathSet>(store, in); @@ -441,7 +468,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr { out << drv.outputs.size(); for (auto & i : drv.outputs) - out << i.first << store.printStorePath(i.second.path) << i.second.hashAlgo << i.second.hash; + out << i.first + << store.printStorePath(i.second.path) + << i.second.hash->printMethodAlgo() + << i.second.hash->hash.to_string(Base::Base16, false); writeStorePaths(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); @@ -453,7 +483,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr std::string hashPlaceholder(const std::string & outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); + return "/" + hashString(HashType::SHA256, "nix-output:" + outputName).to_string(Base::Base32, false); } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b1224b93b..e314beac1 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,8 +1,9 @@ #pragma once +#include "path.hh" #include "types.hh" #include "hash.hh" -#include "store-api.hh" +#include "file-hash.hh" #include <map> @@ -15,13 +16,14 @@ namespace nix { struct DerivationOutput { StorePath path; - std::string hashAlgo; /* hash used for expected hash computation */ - std::string hash; /* expected hash, may be null */ - DerivationOutput(StorePath && path, std::string && hashAlgo, std::string && hash) + std::optional<FileSystemHash> hash; /* hash used for expected hash computation */ + DerivationOutput(StorePath && path, std::optional<FileSystemHash> && hash) : path(std::move(path)) - , hashAlgo(std::move(hashAlgo)) , hash(std::move(hash)) { } + DerivationOutput(const DerivationOutput &) = default; + DerivationOutput(DerivationOutput &&) = default; + DerivationOutput & operator = (const DerivationOutput &) = default; void parseHashInfo(FileIngestionMethod & recursive, Hash & hash) const; }; @@ -76,6 +78,7 @@ struct Derivation : BasicDerivation class Store; +enum RepairFlag : bool { NoRepair = false, Repair = true }; /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(ref<Store> store, diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index f0d01a240..c9731a6bd 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -11,7 +11,7 @@ struct HashAndWriteSink : Sink { Sink & writeSink; HashSink hashSink; - HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) + HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(HashType::SHA256) { } virtual void operator () (const unsigned char * data, size_t len) @@ -34,7 +34,7 @@ void Store::exportPaths(const StorePathSet & paths, Sink & sink) //logger->incExpected(doneLabel, sorted.size()); for (auto & path : sorted) { - //Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); + //Activity act(*logger, Verbosity::Info, format("exporting path '%s'") % path); sink << 1; exportPath(path, sink); //logger->incProgress(doneLabel); @@ -86,7 +86,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces ValidPathInfo info(parseStorePath(readString(source))); - //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); + //Activity act(*logger, Verbosity::Info, format("importing path '%s'") % info.path); info.references = readStorePaths<StorePathSet>(*this, source); @@ -94,7 +94,7 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces if (deriver != "") info.deriver = parseStorePath(deriver); - info.narHash = hashString(htSHA256, *tee.source.data); + info.narHash = hashString(HashType::SHA256, *tee.source.data); info.narSize = tee.source.data->size(); // Ignore optional legacy signature. diff --git a/src/libstore/file-hash.cc b/src/libstore/file-hash.cc new file mode 100644 index 000000000..494079c18 --- /dev/null +++ b/src/libstore/file-hash.cc @@ -0,0 +1,53 @@ +#include "file-hash.hh" + +namespace nix { + +std::string FileSystemHash::printMethodAlgo() const { + return makeFileIngestionPrefix(method) + printHashType(hash.type); +} + +std::string makeFileIngestionPrefix(const FileIngestionMethod m) { + switch (m) { + case FileIngestionMethod::Flat: + return ""; + case FileIngestionMethod::Recursive: + return "r:"; + default: + throw Error("impossible, caught both cases"); + } +} + +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash) +{ + return "fixed:" + + makeFileIngestionPrefix(method) + + hash.to_string(); +} + +template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; }; +template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>; + +std::string renderContentAddress(ContentAddress ca) { + return std::visit(overloaded { + [](TextHash th) { + return "text:" + th.hash.to_string(); + }, + [](FileSystemHash fsh) { + return makeFixedOutputCA(fsh.method, fsh.hash); + } + }, ca); +} + +ContentAddress parseCa(std::string_view rawCa) { + throw Error("TODO"); +}; + +std::optional<ContentAddress> parseCaOpt(std::string_view rawCaOpt) { + return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseCa(rawCaOpt); +}; + +std::string renderContentAddress(std::optional<ContentAddress> ca) { + return ca ? renderContentAddress(*ca) : ""; +} + +} diff --git a/src/libstore/file-hash.hh b/src/libstore/file-hash.hh new file mode 100644 index 000000000..9d12f5fe7 --- /dev/null +++ b/src/libstore/file-hash.hh @@ -0,0 +1,66 @@ +#pragma once + +#include <variant> +#include "hash.hh" + +namespace nix { + +enum struct FileIngestionMethod : uint8_t { + Flat = false, + Recursive = true +}; + +struct TextHash { + Hash hash; + TextHash(const TextHash &) = default; + TextHash(TextHash &&) = default; + TextHash & operator = (const TextHash &) = default; +}; + +/// Pair of a hash, and how the file system was ingested +struct FileSystemHash { + FileIngestionMethod method; + Hash hash; + FileSystemHash(FileIngestionMethod method, Hash hash) + : method(std::move(method)) + , hash(std::move(hash)) + { } + FileSystemHash(const FileSystemHash &) = default; + FileSystemHash(FileSystemHash &&) = default; + FileSystemHash & operator = (const FileSystemHash &) = default; + std::string printMethodAlgo() const; +}; + +/* + We've accumulated several types of content-addressed paths over the years; + fixed-output derivations support multiple hash algorithms and serialisation + methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms: + + * ‘text:sha256:<sha256 hash of file contents>’: For paths + computed by makeTextPath() / addTextToStore(). + + * ‘fixed:<r?>:<ht>:<h>’: For paths computed by + makeFixedOutputPath() / addToStore(). +*/ +typedef std::variant< + TextHash, // for paths computed by makeTextPath() / addTextToStore + FileSystemHash // for path computed by makeFixedOutputPath +> ContentAddress; + +/* Compute the prefix to the hash algorithm which indicates how the files were + ingested. */ +std::string makeFileIngestionPrefix(const FileIngestionMethod m); + +/* Compute the content-addressability assertion (ValidPathInfo::ca) + for paths created by makeFixedOutputPath() / addToStore(). */ +std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); + +std::string renderContentAddress(ContentAddress ca); + +std::string renderContentAddress(std::optional<ContentAddress> ca); + +ContentAddress parseCa(std::string_view rawCa); + +std::optional<ContentAddress> parseCaOpt(std::string_view rawCaOpt); + +} diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index e9684b3d4..933c8038d 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -77,7 +77,7 @@ struct curlFileTransfer : public FileTransfer Callback<FileTransferResult> && callback) : fileTransfer(fileTransfer) , request(request) - , act(*logger, lvlTalkative, actFileTransfer, + , act(*logger, Verbosity::Talkative, ActivityType::Download, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) @@ -163,7 +163,7 @@ struct curlFileTransfer : public FileTransfer { size_t realSize = size * nmemb; std::string line((char *) contents, realSize); - printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); + printMsg(Verbosity::Vomit, format("got header for '%s': %s") % request.uri % trim(line)); if (line.compare(0, 5, "HTTP/") == 0) { // new response starts result.etag = ""; auto ss = tokenizeString<vector<string>>(line, " "); @@ -246,7 +246,7 @@ struct curlFileTransfer : public FileTransfer curl_easy_reset(req); - if (verbosity >= lvlVomit) { + if (verbosity >= Verbosity::Vomit) { curl_easy_setopt(req, CURLOPT_VERBOSE, 1); curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 95a4bc934..f2646f6b3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -78,7 +78,7 @@ void LocalStore::syncWithGC() void LocalStore::addIndirectRoot(const Path & path) { - string hash = hashString(htSHA1, path).to_string(Base32, false); + string hash = hashString(HashType::SHA1, path).to_string(Base::Base32, false); Path realRoot = canonPath((format("%1%/%2%/auto/%3%") % stateDir % gcRootsDir % hash).str()); makeSymlink(realRoot, path); @@ -637,7 +637,7 @@ void LocalStore::tryToDelete(GCState & state, const Path & path) auto realPath = realStoreDir + "/" + std::string(baseNameOf(path)); if (realPath == linksDir || realPath == trashDir) return; - //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); + //Activity act(*logger, Verbosity::Debug, format("considering whether to delete '%1%'") % path); auto storePath = maybeParseStorePath(path); @@ -702,7 +702,7 @@ void LocalStore::removeUnusedLinks(const GCState & state) continue; } - printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + printMsg(Verbosity::Talkative, format("deleting unused link '%1%'") % path); if (unlink(path.c_str()) == -1) throw SysError(format("deleting '%1%'") % path); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index af20d389b..8471ed073 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -114,7 +114,11 @@ struct LegacySSHStore : public Store if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { auto s = readString(conn->from); info->narHash = s.empty() ? Hash() : Hash(s); - conn->from >> info->ca; + { + std::string rawCaOpt; + conn->from >> rawCaOpt; + info->ca = parseCaOpt(rawCaOpt); + } info->sigs = readStrings<StringSet>(conn->from); } @@ -139,14 +143,14 @@ struct LegacySSHStore : public Store << cmdAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(Base::Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs - << info.ca; + << renderContentAddress(info.ca); try { copyNAR(source, conn->to); } catch (...) { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b591..fe346f96a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -557,10 +557,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat if (out == drv.outputs.end()) throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath)); - FileIngestionMethod method; Hash h; - out->second.parseHashInfo(method, h); - - check(makeFixedOutputPath(method, h, drvName), out->second.path, "out"); + check( + makeFixedOutputPath( + out->second.hash->method, + out->second.hash->hash, + drvName), + out->second.path, "out"); } else { @@ -574,19 +576,19 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs) { - if (info.ca != "" && !info.isContentAddressed(*this)) + if (info.ca && !info.isContentAddressed(*this)) throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", printStorePath(info.path)); state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base::Base16)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (info.ca, !info.ca.empty()) + (renderContentAddress(info.ca), (bool) info.ca) .exec(); uint64_t id = sqlite3_last_insert_rowid(state.db); @@ -660,7 +662,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, if (s) info->sigs = tokenizeString<StringSet>(s, " "); s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); - if (s) info->ca = s; + if (s) info->ca = parseCaOpt(s); /* Get the references. */ auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); @@ -680,10 +682,10 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base::Base16)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) - (info.ca, !info.ca.empty()) + (renderContentAddress(info.ca), (bool) info.ca) (printStorePath(info.path)) .exec(); } @@ -908,7 +910,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & i : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash.type == HashType::SHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -997,18 +999,16 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, deletePath(realPath); - if (info.ca != "" && - !((hasPrefix(info.ca, "text:") && !info.references.count(info.path)) - || info.references.empty())) + if (info.ca && !info.references.empty() && !std::holds_alternative<TextHash>(*info.ca)) settings.requireExperimentalFeature("ca-references"); /* While restoring the path from the NAR, compute the hash of the NAR. */ std::unique_ptr<AbstractHashSink> hashSink; - if (info.ca == "" || !info.references.count(info.path)) - hashSink = std::make_unique<HashSink>(htSHA256); + if (info.ca || !info.references.count(info.path)) + hashSink = std::make_unique<HashSink>(HashType::SHA256); else - hashSink = std::make_unique<HashModuloSink>(htSHA256, storePathToHash(printStorePath(info.path))); + hashSink = std::make_unique<HashModuloSink>(HashType::SHA256, storePathToHash(printStorePath(info.path))); LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { size_t n = source.read(data, len); @@ -1081,17 +1081,20 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam sha256); otherwise, compute it here. */ HashResult hash; if (method == FileIngestionMethod::Recursive) { - hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); + hash.first = hashAlgo == HashType::SHA256 ? h : hashString(HashType::SHA256, dump); hash.second = dump.size(); } else - hash = hashPath(htSHA256, realPath); + hash = hashPath(HashType::SHA256, realPath); optimisePath(realPath); // FIXME: combine with hashPath() ValidPathInfo info(dstPath.clone()); info.narHash = hash.first; info.narSize = hash.second; - info.ca = makeFixedOutputCA(method, h); + info.ca = FileSystemHash { + .method = method, + .hash = h, + }; registerValidPath(info); } @@ -1123,7 +1126,7 @@ StorePath LocalStore::addToStore(const string & name, const Path & _srcPath, StorePath LocalStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - auto hash = hashString(htSHA256, s); + auto hash = hashString(HashType::SHA256, s); auto dstPath = makeTextPath(name, hash, references); addTempRoot(dstPath); @@ -1147,7 +1150,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, StringSink sink; dumpString(s, sink); - auto narHash = hashString(htSHA256, *sink.s); + auto narHash = hashString(HashType::SHA256, *sink.s); optimisePath(realPath); @@ -1155,7 +1158,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, info.narHash = narHash; info.narSize = sink.s->size(); info.references = cloneStorePathSet(references); - info.ca = "text:" + hash.to_string(); + info.ca = TextHash { .hash = hash }; registerValidPath(info); } @@ -1233,9 +1236,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking link hashes..."); for (auto & link : readDirectory(linksDir)) { - printMsg(lvlTalkative, "checking contents of '%s'", link.name); + printMsg(Verbosity::Talkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); + string hash = hashPath(HashType::SHA256, linkPath).first.to_string(Base::Base32, false); if (hash != link.name) { printError( "link '%s' was modified! expected hash '%s', got '%s'", @@ -1253,17 +1256,17 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking store hashes..."); - Hash nullHash(htSHA256); + Hash nullHash(HashType::SHA256); for (auto & i : validPaths) { try { auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); + printMsg(Verbosity::Talkative, "checking contents of '%s'", printStorePath(i)); std::unique_ptr<AbstractHashSink> hashSink; - if (info->ca == "" || !info->references.count(info->path)) + if (info->ca || !info->references.count(info->path)) hashSink = std::make_unique<HashSink>(info->narHash.type); else hashSink = std::make_unique<HashModuloSink>(info->narHash.type, storePathToHash(printStorePath(info->path))); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c1e75390c..bc9a415b3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -157,7 +157,7 @@ public: true) or simply the contents of a regular file (if recursive == false). */ StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9c47fe524..fb538a1c5 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -112,7 +112,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets, StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_, unsigned long long & downloadSize_, unsigned long long & narSize_) { - Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); + Activity act(*logger, Verbosity::Debug, ActivityType::Unknown, "querying info about missing paths"); downloadSize_ = narSize_ = 0; diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 442541330..b4346d4d1 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -203,7 +203,7 @@ public: narInfo->deriver = StorePath::fromBaseName(queryNAR.getStr(9)); for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " ")) narInfo->sigs.insert(sig); - narInfo->ca = queryNAR.getStr(11); + narInfo->ca = parseCaOpt(queryNAR.getStr(11)); return {oValid, narInfo}; }); @@ -237,7 +237,7 @@ public: (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (concatStringsSep(" ", info->sigs)) - (info->ca) + (renderContentAddress(info->ca)) (time(0)).exec(); } else { diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1375094b5..4f7c732ce 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,7 +4,7 @@ namespace nix { NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) - : ValidPathInfo(StorePath::dummy.clone()) // FIXME: hack + : ValidPathInfo(StorePath::dummy) // FIXME: hack { auto corrupt = [&]() { throw Error(format("NAR info file '%1%' is corrupt") % whence); @@ -67,8 +67,9 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "Sig") sigs.insert(value); else if (name == "CA") { - if (!ca.empty()) corrupt(); - ca = value; + if (ca) corrupt(); + // FIXME: allow blank ca or require skipping field? + ca = parseCaOpt(value); } pos = eol + 1; @@ -86,11 +87,11 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32) + "\n"; + assert(fileHash.type == HashType::SHA256); + res += "FileHash: " + fileHash.to_string(Base::Base32) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32) + "\n"; + assert(narHash.type == HashType::SHA256); + res += "NarHash: " + narHash.to_string(Base::Base32) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; @@ -104,8 +105,8 @@ std::string NarInfo::to_string(const Store & store) const for (auto sig : sigs) res += "Sig: " + sig + "\n"; - if (!ca.empty()) - res += "CA: " + ca + "\n"; + if (ca) + res += "CA: " + renderContentAddress(*ca) + "\n"; return res; } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8ac382e9d..5c01e1b3b 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -57,7 +57,7 @@ LocalStore::InodeHash LocalStore::loadInodeHash() } if (errno) throw SysError(format("reading directory '%1%'") % linksDir); - printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); + printMsg(Verbosity::Talkative, format("loaded %1% hash inodes") % inodeHash.size()); return inodeHash; } @@ -149,11 +149,11 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path).first; + Hash hash = hashPath(HashType::SHA256, path).first; debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(Base::Base32, false); retry: if (!pathExists(linkPath)) { @@ -199,7 +199,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, goto retry; } - printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); + printMsg(Verbosity::Talkative, format("linking '%1%' to '%2%'") % path % linkPath); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its @@ -246,13 +246,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, stats.blocksFreed += st.st_blocks; if (act) - act->result(resFileLinked, st.st_size, st.st_blocks); + act->result(ResultType::FileLinked, st.st_size, st.st_blocks); } void LocalStore::optimiseStore(OptimiseStats & stats) { - Activity act(*logger, actOptimiseStore); + Activity act(*logger, ActivityType::OptimiseStore); auto paths = queryAllValidPaths(); InodeHash inodeHash = loadInodeHash(); @@ -265,7 +265,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) addTempRoot(i); if (!isValidPath(i)) continue; /* path was GC'ed, probably */ { - Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); + Activity act(*logger, Verbosity::Talkative, ActivityType::Unknown, fmt("optimising path '%s'", printStorePath(i))); optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash); } done++; diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index f4df5dd54..d0cc70f9c 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,4 +1,4 @@ -#include "derivations.hh" +#include "store-api.hh" #include <nlohmann/json_fwd.hpp> diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 5122e7422..dfc0a9531 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -1,6 +1,7 @@ #pragma once #include "rust-ffi.hh" +#include "file-hash.hh" namespace nix { @@ -13,6 +14,7 @@ extern "C" { void ffi_StorePath_drop(void *); bool ffi_StorePath_less_than(const StorePath & a, const StorePath & b); bool ffi_StorePath_eq(const StorePath & a, const StorePath & b); + void ffi_StorePath_clone_to(const StorePath & _other, StorePath & _this); unsigned char * ffi_StorePath_hash_data(const StorePath & p); } @@ -43,6 +45,19 @@ struct StorePath : rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop> return !(*this == other); } + StorePath(StorePath && that) = default; + + StorePath(const StorePath & that) + { + ffi_StorePath_clone_to(that, *this); + } + + void operator = (const StorePath & that) + { + (rust::Value<3 * sizeof(void *) + 24, ffi_StorePath_drop>::operator = (that)); + ffi_StorePath_clone_to(that, *this); + } + StorePath clone() const; /* Check whether a file name ends with the extension for @@ -73,11 +88,6 @@ const size_t storePathHashLen = 32; // i.e. 160 bits /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; -enum struct FileIngestionMethod : uint8_t { - Flat = false, - Recursive = true -}; - struct StorePathWithOutputs { StorePath path; diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 102e15921..6652e1e26 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -54,7 +54,7 @@ struct RefScanSink : Sink string tail; - RefScanSink() : hashSink(htSHA256) { } + RefScanSink() : hashSink(HashType::SHA256) { } void operator () (const unsigned char * data, size_t len); }; @@ -96,7 +96,7 @@ PathSet scanForReferences(const string & path, string s = string(baseName, 0, pos); assert(s.size() == refLength); assert(backMap.find(s) == backMap.end()); - // parseHash(htSHA256, s); + // parseHash(HashType::SHA256, s); sink.hashes.insert(s); backMap[s] = i; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e6..2744245f3 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -177,11 +177,11 @@ void RemoteStore::setOptions(Connection & conn) << settings.keepFailed << settings.keepGoing << settings.tryFallback - << verbosity + << (uint64_t) verbosity << settings.maxBuildJobs << settings.maxSilentTime << true - << (settings.verboseBuild ? lvlError : lvlVomit) + << (uint64_t) (settings.verboseBuild ? Verbosity::Error : Verbosity::Vomit) << 0 // obsolete log type << 0 /* obsolete print build trace */ << settings.buildCores @@ -375,13 +375,15 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, info = std::make_shared<ValidPathInfo>(path.clone()); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->narHash = Hash(readString(conn->from), htSHA256); + info->narHash = Hash(readString(conn->from), HashType::SHA256); info->references = readStorePaths<StorePathSet>(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; info->sigs = readStrings<StringSet>(conn->from); - conn->from >> info->ca; + string caOptRaw; + conn->from >> caOptRaw; + info->ca = parseCaOpt(caOptRaw); } } callback(std::move(info)); @@ -471,10 +473,10 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << wopAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(Base::Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize - << info.ultimate << info.sigs << info.ca + << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; if (!tunnel) copyNAR(source, conn->to); @@ -495,7 +497,7 @@ StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, conn->to << wopAddToStore << name - << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << ((hashAlgo == HashType::SHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << (method == FileIngestionMethod::Recursive ? 1 : 0) << printHashType(hashAlgo); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3c86b4524..bd541779c 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -65,7 +65,7 @@ public: std::shared_ptr<FSAccessor> accessor) override; StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; StorePath addTextToStore(const string & name, const string & s, diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index b24e7b7d6..f43581bf6 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -68,9 +68,9 @@ static void initAWS() shared.cc), so don't let aws-sdk-cpp override it. */ options.cryptoOptions.initAndCleanupOpenSSL = false; - if (verbosity >= lvlDebug) { + if (verbosity >= Verbosity::Debug) { options.loggingOptions.logLevel = - verbosity == lvlDebug + verbosity == Verbosity::Debug ? Aws::Utils::Logging::LogLevel::Debug : Aws::Utils::Logging::LogLevel::Trace; options.loggingOptions.logger_create_fn = [options]() { diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 84548a6e4..f61c094a2 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -58,7 +58,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string addCommonSSHOpts(args); if (socketPath != "") args.insert(args.end(), {"-S", socketPath}); - if (verbosity >= lvlChatty) + if (verbosity >= Verbosity::Chatty) args.push_back("-v"); } @@ -110,7 +110,7 @@ Path SSHMaster::startMaster() , "-o", "LocalCommand=echo started" , "-o", "PermitLocalCommand=yes" }; - if (verbosity >= lvlChatty) + if (verbosity >= Verbosity::Chatty) args.push_back("-v"); addCommonSSHOpts(args); execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 095363d0c..ea30ed105 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -142,8 +142,8 @@ StorePath Store::makeStorePath(const string & type, const Hash & hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name); - auto h = compressHash(hashString(htSHA256, s), 20); + string s = type + ":" + hash.to_string(Base::Base16) + ":" + storeDir + ":" + std::string(name); + auto h = compressHash(hashString(HashType::SHA256, s), 20); return StorePath::make(h.hash, name); } @@ -172,22 +172,19 @@ static std::string makeType( StorePath Store::makeFixedOutputPath( - FileIngestionMethod recursive, + FileIngestionMethod method, const Hash & hash, std::string_view name, const StorePathSet & references, bool hasSelfReference) const { - if (hash.type == htSHA256 && recursive == FileIngestionMethod::Recursive) { + if (hash.type == HashType::SHA256 && method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", references, hasSelfReference), hash, name); } else { assert(references.empty()); - return makeStorePath("output:out", - hashString(htSHA256, - "fixed:out:" - + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") - + hash.to_string(Base16) + ":"), - name); + return makeStorePath("output:out", hashString(HashType::SHA256, + "fixed:out:" + makeFileIngestionPrefix(method) + + hash.to_string(Base::Base16) + ":"), name); } } @@ -195,7 +192,7 @@ StorePath Store::makeFixedOutputPath( StorePath Store::makeTextPath(std::string_view name, const Hash & hash, const StorePathSet & references) const { - assert(hash.type == htSHA256); + assert(hash.type == HashType::SHA256); /* Stuff the references (if any) into the type. This is a bit hacky, but we can't put them in `s' since that would be ambiguous. */ @@ -216,7 +213,7 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, StorePath Store::computeStorePathForText(const string & name, const string & s, const StorePathSet & references) const { - return makeTextPath(name, hashString(htSHA256, s), references); + return makeTextPath(name, hashString(HashType::SHA256, s), references); } @@ -429,7 +426,7 @@ string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash.to_string(Base::Base16, false) + "\n"; s += (format("%1%\n") % info->narSize).str(); } @@ -470,8 +467,8 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store jsonRefs.elem(printStorePath(ref)); } - if (info->ca != "") - jsonPath.attr("ca", info->ca); + if (info->ca) + jsonPath.attr("ca", renderContentAddress(info->ca)); std::pair<uint64_t, uint64_t> closureSizes; @@ -567,7 +564,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, auto srcUri = srcStore->getUri(); auto dstUri = dstStore->getUri(); - Activity act(*logger, lvlInfo, actCopyPath, + Activity act(*logger, Verbosity::Info, ActivityType::CopyPath, srcUri == "local" || srcUri == "daemon" ? fmt("copying path '%s' to '%s'", srcStore->printStorePath(storePath), dstUri) : dstUri == "local" || dstUri == "daemon" @@ -584,7 +581,7 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore, StringSink sink; srcStore->narFromPath({storePath}, sink); auto info2 = make_ref<ValidPathInfo>(*info); - info2->narHash = hashString(htSHA256, *sink.s); + info2->narHash = hashString(HashType::SHA256, *sink.s); if (!info->narSize) info2->narSize = sink.s->size(); if (info->ultimate) info2->ultimate = false; info = info2; @@ -626,7 +623,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st if (missing.empty()) return; - Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + Activity act(*logger, Verbosity::Info, ActivityType::CopyPaths, fmt("copying %d paths", missing.size())); std::atomic<size_t> nrDone{0}; std::atomic<size_t> nrFailed{0}; @@ -652,7 +649,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st auto info = srcStore->queryPathInfo(srcStore->parseStorePath(storePath)); bytesExpected += info->narSize; - act.setExpected(actCopyPath, bytesExpected); + act.setExpected(ActivityType::CopyPath, bytesExpected); return srcStore->printStorePathSet(info->references); }, @@ -671,7 +668,7 @@ void copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & st nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", storePathS, e.what())); + logger->log(Verbosity::Error, fmt("could not copy %s: %s", storePathS, e.what())); showProgress(); return; } @@ -693,21 +690,6 @@ void copyClosure(ref<Store> srcStore, ref<Store> dstStore, } -ValidPathInfo::ValidPathInfo(const ValidPathInfo & other) - : path(other.path.clone()) - , deriver(other.deriver ? other.deriver->clone(): std::optional<StorePath>{}) - , narHash(other.narHash) - , references(cloneStorePathSet(other.references)) - , registrationTime(other.registrationTime) - , narSize(other.narSize) - , id(other.id) - , ultimate(other.ultimate) - , sigs(other.sigs) - , ca(other.ca) -{ -} - - std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, bool hashGiven) { std::string path; @@ -717,7 +699,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre if (hashGiven) { string s; getline(str, s); - info.narHash = Hash(s, htSHA256); + info.narHash = Hash(s, HashType::SHA256); getline(str, s); if (!string2Int(s, info.narSize)) throw Error("number expected"); } @@ -760,7 +742,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32) + ";" + + narHash.to_string(Base::Base32) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -778,24 +760,23 @@ bool ValidPathInfo::isContentAddressed(const Store & store) const printError("warning: path '%s' claims to be content-addressed but isn't", store.printStorePath(path)); }; - if (hasPrefix(ca, "text:")) { - Hash hash(std::string(ca, 5)); - if (store.makeTextPath(path.name(), hash, references) == path) + if (! ca) {} + + else if (auto p = std::get_if<TextHash>(&*ca)) { + if (store.makeTextPath(path.name(), p->hash, references) == path) return true; else warn(); } - else if (hasPrefix(ca, "fixed:")) { - FileIngestionMethod recursive { ca.compare(6, 2, "r:") == 0 }; - Hash hash(std::string(ca, recursive == FileIngestionMethod::Recursive ? 8 : 6)); + else if (auto p = std::get_if<FileSystemHash>(&*ca)) { auto refs = cloneStorePathSet(references); bool hasSelfReference = false; if (refs.count(path)) { hasSelfReference = true; refs.erase(path); } - if (store.makeFixedOutputPath(recursive, hash, path.name(), refs, hasSelfReference) == path) + if (store.makeFixedOutputPath(p->method, p->hash, path.name(), refs, hasSelfReference) == path) return true; else warn(); @@ -832,14 +813,6 @@ Strings ValidPathInfo::shortRefs() const } -std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) -{ - return "fixed:" - + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") - + hash.to_string(); -} - - } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b1e25fc7d..7836c04dc 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -2,12 +2,14 @@ #include "path.hh" #include "hash.hh" +#include "file-hash.hh" #include "serialise.hh" #include "crypto.hh" #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" #include "config.hh" +#include "derivations.hh" #include <atomic> #include <limits> @@ -17,6 +19,7 @@ #include <memory> #include <string> #include <chrono> +#include <variant> namespace nix { @@ -31,15 +34,12 @@ MakeError(SubstituterDisabled, Error); MakeError(NotInStore, Error); -struct BasicDerivation; -struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; class JSONPlaceholder; -enum RepairFlag : bool { NoRepair = false, Repair = true }; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; @@ -111,7 +111,6 @@ struct SubstitutablePathInfo typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; - struct ValidPathInfo { StorePath path; @@ -140,21 +139,11 @@ struct ValidPathInfo that a particular output path was produced by a derivation; the path then implies the contents.) - Ideally, the content-addressability assertion would just be a - Boolean, and the store path would be computed from - the name component, ‘narHash’ and ‘references’. However, - 1) we've accumulated several types of content-addressed paths - over the years; and 2) fixed-output derivations support - multiple hash algorithms and serialisation methods (flat file - vs NAR). Thus, ‘ca’ has one of the following forms: - - * ‘text:sha256:<sha256 hash of file contents>’: For paths - computed by makeTextPath() / addTextToStore(). - - * ‘fixed:<r?>:<ht>:<h>’: For paths computed by - makeFixedOutputPath() / addToStore(). + Ideally, the content-addressability assertion would just be a Boolean, + and the store path would be computed from the name component, ‘narHash’ + and ‘references’. However, we support many types of content addresses. */ - std::string ca; + std::optional<ContentAddress> ca; bool operator == (const ValidPathInfo & i) const { @@ -189,8 +178,9 @@ struct ValidPathInfo Strings shortRefs() const; - ValidPathInfo(StorePath && path) : path(std::move(path)) { } - explicit ValidPathInfo(const ValidPathInfo & other); + ValidPathInfo(StorePath && path) : path(std::move(path)) { }; + ValidPathInfo(const StorePath & path) : path(path) { }; + ValidPathInfo(const ValidPathInfo & other) = default; virtual ~ValidPathInfo() { } }; @@ -359,7 +349,7 @@ public: path and the cryptographic hash of the contents of srcPath. */ std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; + HashType hashAlgo = HashType::SHA256, PathFilter & filter = defaultPathFilter) const; /* Preparatory part of addTextToStore(). @@ -457,12 +447,12 @@ public: The function object `filter' can be used to exclude files (see libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; // FIXME: remove? virtual StorePath addToStoreFromDump(const string & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair) { throw Error("addToStoreFromDump() is not supported by this store"); } @@ -556,7 +546,7 @@ public: each path is included. */ void pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - Base hashBase = Base32, + Base hashBase = Base::Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /* Return the size of the closure of the specified path, that is, @@ -842,12 +832,6 @@ std::optional<ValidPathInfo> decodeValidPathInfo( std::istream & str, bool hashGiven = false); - -/* Compute the content-addressability assertion (ValidPathInfo::ca) - for paths created by makeFixedOutputPath() / addToStore(). */ -std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash); - - /* Split URI into protocol+hierarchy part and its parameter set. */ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri); |