diff options
Diffstat (limited to 'src')
42 files changed, 346 insertions, 119 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 00340b787..d2ea6c956 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -241,7 +241,7 @@ connected: uploadLock = -1; - BasicDerivation drv(readDerivation(*store, store->realStoreDir + "/" + std::string(drvPath->to_string()))); + auto drv = store->readDerivation(*drvPath); drv.inputSrcs = store->parseStorePathSet(inputs); auto result = sshStore->buildDerivation(*drvPath, drv); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 083c7d398..eb3116d81 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -688,7 +688,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & j : refs) { drv.inputSrcs.insert(j.clone()); if (j.isDerivation()) - drv.inputDrvs[j.clone()] = state.store->queryDerivationOutputNames(j); + drv.inputDrvs[j.clone()] = state.store->readDerivation(j).outputNames(); } } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 1464aa8b4..97254aa04 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -23,7 +23,7 @@ void emitTreeAttrs( assert(tree.info.narHash); mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI)); + tree.info.narHash.to_string(SRI, true)); if (input->getRev()) { mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); @@ -140,7 +140,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, : hashFile(htSHA256, path); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", - *url, expectedHash->to_string(), hash.to_string()); + *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } if (state.allowedPaths) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 91b897e62..11cac4c55 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -47,7 +47,7 @@ Attrs Input::toAttrs() const { auto attrs = toAttrsInternal(); if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); + attrs.emplace("narHash", narHash->to_string(SRI, true)); attrs.emplace("type", type()); return attrs; } @@ -67,7 +67,7 @@ std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) if (narHash && narHash != input->narHash) throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); + to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true)); return {std::move(tree), input}; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 937f86bc6..7966da314 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -196,9 +196,9 @@ struct TarballInput : Input // NAR hashes are preferred over file hashes since tar/zip files // don't have a canonical representation. if (narHash) - url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); + url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); else if (hash) - url2.query.insert_or_assign("hash", hash->to_string(SRI)); + url2.query.insert_or_assign("hash", hash->to_string(SRI, true)); return url2; } @@ -207,7 +207,7 @@ struct TarballInput : Input Attrs attrs; attrs.emplace("url", url.to_string()); if (hash) - attrs.emplace("hash", hash->to_string(SRI)); + attrs.emplace("hash", hash->to_string(SRI, true)); return attrs; } diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 51e199ea5..051668e53 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,5 +1,6 @@ #include "common-args.hh" #include "globals.hh" +#include "loggers.hh" namespace nix { @@ -39,6 +40,14 @@ MixCommonArgs::MixCommonArgs(const string & programName) }); addFlag({ + .longName = "log-format", + .description = "format of log output; \"raw\", \"internal-json\", \"bar\" " + "or \"bar-with-logs\"", + .labels = {"format"}, + .handler = {[](std::string format) { setLogFormat(format); }}, + }); + + addFlag({ .longName = "max-jobs", .shortName = 'j', .description = "maximum number of parallel builds", diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc new file mode 100644 index 000000000..c44bb6408 --- /dev/null +++ b/src/libmain/loggers.cc @@ -0,0 +1,52 @@ +#include "loggers.hh" +#include "progress-bar.hh" + +namespace nix { + +LogFormat defaultLogFormat = LogFormat::raw; + +LogFormat parseLogFormat(const std::string & logFormatStr) { + if (logFormatStr == "raw") + return LogFormat::raw; + else if (logFormatStr == "raw-with-logs") + return LogFormat::rawWithLogs; + else if (logFormatStr == "internal-json") + return LogFormat::internalJson; + else if (logFormatStr == "bar") + return LogFormat::bar; + else if (logFormatStr == "bar-with-logs") + return LogFormat::barWithLogs; + throw Error("option 'log-format' has an invalid value '%s'", logFormatStr); +} + +Logger * makeDefaultLogger() { + switch (defaultLogFormat) { + case LogFormat::raw: + return makeSimpleLogger(false); + case LogFormat::rawWithLogs: + return makeSimpleLogger(true); + case LogFormat::internalJson: + return makeJSONLogger(*makeSimpleLogger()); + case LogFormat::bar: + return makeProgressBar(); + case LogFormat::barWithLogs: + return makeProgressBar(true); + default: + abort(); + } +} + +void setLogFormat(const std::string & logFormatStr) { + setLogFormat(parseLogFormat(logFormatStr)); +} + +void setLogFormat(const LogFormat & logFormat) { + defaultLogFormat = logFormat; + createDefaultLogger(); +} + +void createDefaultLogger() { + logger = makeDefaultLogger(); +} + +} diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh new file mode 100644 index 000000000..cada03110 --- /dev/null +++ b/src/libmain/loggers.hh @@ -0,0 +1,20 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +enum class LogFormat { + raw, + rawWithLogs, + internalJson, + bar, + barWithLogs, +}; + +void setLogFormat(const std::string & logFormatStr); +void setLogFormat(const LogFormat & logFormat); + +void createDefaultLogger(); + +} diff --git a/src/nix/progress-bar.cc b/src/libmain/progress-bar.cc index c67701098..b287de8a3 100644 --- a/src/nix/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -106,7 +106,7 @@ public: updateThread.join(); } - void stop() + void stop() override { auto state(state_.lock()); if (!state->active) return; @@ -119,6 +119,10 @@ public: quitCV.notify_one(); } + bool isVerbose() override { + return printBuildLogs; + } + void log(Verbosity lvl, const FormatOrString & fs) override { auto state(state_.lock()); @@ -457,11 +461,17 @@ public: } }; -void startProgressBar(bool printBuildLogs) +Logger * makeProgressBar(bool printBuildLogs) { - logger = new ProgressBar( + return new ProgressBar( printBuildLogs, - isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb"); + isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" + ); +} + +void startProgressBar(bool printBuildLogs) +{ + logger = makeProgressBar(printBuildLogs); } void stopProgressBar() diff --git a/src/nix/progress-bar.hh b/src/libmain/progress-bar.hh index 4d61175c2..7f0dafecf 100644 --- a/src/nix/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -4,6 +4,8 @@ namespace nix { +Logger * makeProgressBar(bool printBuildLogs = false); + void startProgressBar(bool printBuildLogs = false); void stopProgressBar(); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 70d1f0186..3bbb5cf93 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -2,6 +2,7 @@ #include "shared.hh" #include "store-api.hh" #include "util.hh" +#include "loggers.hh" #include <algorithm> #include <cctype> @@ -169,7 +170,7 @@ LegacyArgs::LegacyArgs(const std::string & programName, .longName = "no-build-output", .shortName = 'Q', .description = "do not show build output", - .handler = {&settings.verboseBuild, false}, + .handler = {[&]() {setLogFormat(LogFormat::raw); }}, }); addFlag({ diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f5c132a83..3ab6220e3 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1642,7 +1642,7 @@ void DerivationGoal::buildDone() worker.store.printStorePath(drvPath), statusToString(status)); - if (!settings.verboseBuild && !logTail.empty()) { + if (!logger->isVerbose() && !logTail.empty()) { msg += (format("; last %d log lines:") % logTail.size()).str(); for (auto & line : logTail) msg += "\n " + line; @@ -1691,11 +1691,7 @@ void DerivationGoal::buildDone() } void flushLine() { - if (settings.verboseBuild) { - printError("post-build-hook: " + currentLine); - } else { - act.result(resPostBuildLogLine, currentLine); - } + act.result(resPostBuildLogLine, currentLine); currentLine.clear(); } @@ -2727,9 +2723,6 @@ struct RestrictedStore : public LocalFSStore StorePathSet queryDerivationOutputs(const StorePath & path) override { throw Error("queryDerivationOutputs"); } - StringSet queryDerivationOutputNames(const StorePath & path) override - { throw Error("queryDerivationOutputNames"); } - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } @@ -3718,7 +3711,7 @@ void DerivationGoal::registerOutputs() worker.hashMismatch = true; delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", - worker.store.printStorePath(dest), h.to_string(SRI), h2.to_string(SRI))); + worker.store.printStorePath(dest), h.to_string(SRI, true), h2.to_string(SRI, true))); Path actualDest = worker.store.Store::toRealPath(dest); @@ -4155,13 +4148,8 @@ void DerivationGoal::flushLine() ; else { - if (settings.verboseBuild && - (settings.printRepeatedBuilds || curRound == 1)) - printError(currentLogLine); - else { - logTail.push_back(currentLogLine); - if (logTail.size() > settings.logLines) logTail.pop_front(); - } + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); act->result(resBuildLogLine, currentLogLine); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5cff170dd..877c70a03 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -329,8 +329,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopQueryDerivationOutputNames: { auto path = store->parseStorePath(readString(from)); logger->startWork(); - StringSet names; - names = store->queryDerivationOutputNames(path); + auto names = store->readDerivation(path).outputNames(); logger->stopWork(); to << names; break; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c68e7b16b..e268c65ff 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -204,6 +204,12 @@ Derivation readDerivation(const Store & store, const Path & drvPath) Derivation Store::derivationFromPath(const StorePath & drvPath) { ensurePath(drvPath); + return readDerivation(drvPath); +} + + +Derivation Store::readDerivation(const StorePath & drvPath) +{ auto accessor = getFSAccessor(); try { return parseDerivation(*this, accessor->readFile(printStorePath(drvPath))); @@ -378,7 +384,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput if (h == drvHashes.end()) { assert(store.isValidPath(i.first)); h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store, - readDerivation(store, store.toRealPath(i.first)), false)).first; + store.readDerivation(i.first), false)).first; } inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second); } @@ -410,6 +416,15 @@ StorePathSet BasicDerivation::outputPaths() const } +StringSet BasicDerivation::outputNames() const +{ + StringSet names; + for (auto & i : outputs) + names.insert(i.first); + return names; +} + + Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv) { drv.outputs.clear(); diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index b1224b93b..88aed66bf 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -58,6 +58,8 @@ struct BasicDerivation /* Return the output paths of a derivation. */ StorePathSet outputPaths() const; + /* Return the output names of a derivation. */ + StringSet outputNames() const; }; struct Derivation : BasicDerivation diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index f0d01a240..54471d4a3 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -57,7 +57,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashAndWriteSink.currentHash(); if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(), hash.to_string()); + printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); hashAndWriteSink << exportMagic diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index da95fd3ae..2fbcafff8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -271,7 +271,7 @@ public: "listed in 'trusted-public-keys'."}; Setting<StringSet> extraPlatforms{this, - std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{}, + std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, "extra-platforms", "Additional platforms that can be built on the local system. " "These may be supported natively (e.g. armv7 on some aarch64 CPUs " diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 80851b591..c2c45b4c2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -580,7 +580,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmtRegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -595,7 +595,7 @@ uint64_t LocalStore::addValidPath(State & state, efficiently query whether a path is an output of some derivation. */ if (info.path.isDerivation()) { - auto drv = readDerivation(*this, realStoreDir + "/" + std::string(info.path.to_string())); + auto drv = readDerivation(info.path); /* Verify that the output paths in the derivation are correct (i.e., follow the scheme for computing output paths from @@ -680,7 +680,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmtUpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16)) + (info.narHash.to_string(Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (info.ca, !info.ca.empty()) @@ -785,23 +785,6 @@ StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) } -StringSet LocalStore::queryDerivationOutputNames(const StorePath & path) -{ - return retrySQLite<StringSet>([&]() { - auto state(_state.lock()); - - auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() - (queryValidPathId(*state, path))); - - StringSet outputNames; - while (useQueryDerivationOutputs.next()) - outputNames.insert(useQueryDerivationOutputs.getStr(0)); - - return outputNames; - }); -} - - std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart) { if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); @@ -928,8 +911,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) for (auto & i : infos) if (i.path.isDerivation()) { // FIXME: inefficient; we already loaded the derivation in addValidPath(). - checkDerivationOutputs(i.path, - readDerivation(*this, realStoreDir + "/" + std::string(i.path.to_string()))); + checkDerivationOutputs(i.path, readDerivation(i.path)); } /* Do a topological sort of the paths. This will throw an @@ -1022,7 +1004,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(), hashResult.first.to_string()); + printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", @@ -1155,7 +1137,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s, info.narHash = narHash; info.narSize = sink.s->size(); info.references = cloneStorePathSet(references); - info.ca = "text:" + hash.to_string(); + info.ca = "text:" + hash.to_string(Base32, true); registerValidPath(info); } @@ -1273,7 +1255,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(), current.first.to_string()); + printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c1e75390c..e17cc45ae 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -135,8 +135,6 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const StorePath & path) override; - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; diff --git a/src/libexpr/names.cc b/src/libstore/names.cc index d1c8a6101..d1c8a6101 100644 --- a/src/libexpr/names.cc +++ b/src/libstore/names.cc diff --git a/src/libexpr/names.hh b/src/libstore/names.hh index 00e14b8c7..00e14b8c7 100644 --- a/src/libexpr/names.hh +++ b/src/libstore/names.hh diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 442541330..e8cf1d177 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -230,9 +230,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string()) + (info->narHash.to_string(Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1375094b5..6b16be08a 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -87,10 +87,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash.type == htSHA256); - res += "FileHash: " + fileHash.to_string(Base32) + "\n"; + res += "FileHash: " + fileHash.to_string(Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32) + "\n"; + res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 8ac382e9d..3f4b72b9c 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -150,7 +150,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); + debug(format("'%1%' has hash '%2%'") % path % hash.to_string(Base32, true)); /* Check if this is a known hash. */ Path linkPath = linksDir + "/" + hash.to_string(Base32, false); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e6..735f59a91 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -418,15 +418,6 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -PathSet RemoteStore::queryDerivationOutputNames(const StorePath & path) -{ - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputNames << printStorePath(path); - conn.processStderr(); - return readStrings<PathSet>(conn->from); -} - - std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3c86b4524..80c8e9f11 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -51,8 +51,6 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - StringSet queryDerivationOutputNames(const StorePath & path) override; - std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 095363d0c..0645fca84 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -142,7 +142,7 @@ StorePath Store::makeStorePath(const string & type, const Hash & hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ - string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + std::string(name); + string s = type + ":" + hash.to_string(Base16, true) + ":" + storeDir + ":" + std::string(name); auto h = compressHash(hashString(htSHA256, s), 20); return StorePath::make(h.hash, name); } @@ -186,7 +186,7 @@ StorePath Store::makeFixedOutputPath( hashString(htSHA256, "fixed:out:" + (recursive == FileIngestionMethod::Recursive ? (string) "r:" : "") - + hash.to_string(Base16) + ":"), + + hash.to_string(Base16, true) + ":"), name); } } @@ -461,7 +461,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store auto info = queryPathInfo(storePath); jsonPath - .attr("narHash", info->narHash.to_string(hashBase)) + .attr("narHash", info->narHash.to_string(hashBase, true)) .attr("narSize", info->narSize); { @@ -504,7 +504,7 @@ void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & store if (!narInfo->url.empty()) jsonPath.attr("url", narInfo->url); if (narInfo->fileHash) - jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); + jsonPath.attr("downloadHash", narInfo->fileHash.to_string(Base32, true)); if (narInfo->fileSize) jsonPath.attr("downloadSize", narInfo->fileSize); if (showClosureSize) @@ -760,7 +760,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32) + ";" + + narHash.to_string(Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -836,7 +836,7 @@ std::string makeFixedOutputCA(FileIngestionMethod recursive, const Hash & hash) { return "fixed:" + (recursive == FileIngestionMethod::Recursive ? (std::string) "r:" : "") - + hash.to_string(); + + hash.to_string(Base32, true); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b1e25fc7d..5ef506326 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -430,10 +430,6 @@ public: virtual StorePathSet queryDerivationOutputs(const StorePath & path) { unsupported("queryDerivationOutputs"); } - /* Query the output names of the derivation denoted by `path'. */ - virtual StringSet queryDerivationOutputNames(const StorePath & path) - { unsupported("queryDerivationOutputNames"); } - /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ virtual std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) = 0; @@ -587,6 +583,9 @@ public: ensurePath(). */ Derivation derivationFromPath(const StorePath & drvPath); + /* Read a derivation (which must already be valid). */ + Derivation readDerivation(const StorePath & drvPath); + /* Place in `out' the set of all store paths in the file system closure of `storePath'; that is, all paths than can be directly or indirectly reached from it. `out' is not cleared. If diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 857d54d99..ac42457fc 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -36,7 +36,7 @@ typedef enum { wopClearFailedPaths = 25, wopQueryPathInfo = 26, wopImportPaths = 27, // obsolete - wopQueryDerivationOutputNames = 28, + wopQueryDerivationOutputNames = 28, // obsolete wopQueryPathFromHashPart = 29, wopQuerySubstitutablePathInfos = 30, wopQueryValidPaths = 31, diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 449cb1b86..5080539b2 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -79,7 +79,7 @@ struct Hash /* Return a string representation of the hash, in base-16, base-32 or base-64. By default, this is prefixed by the hash type (e.g. "sha256:"). */ - std::string to_string(Base base = Base32, bool includeType = true) const; + std::string to_string(Base base, bool includeType) const; std::string gitRev() const { diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 3cc4ef8f1..15cbc1589 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -18,7 +18,7 @@ void setCurActivity(const ActivityId activityId) curActivity = activityId; } -Logger * logger = makeDefaultLogger(); +Logger * logger = makeSimpleLogger(true); void Logger::warn(const std::string & msg) { @@ -35,13 +35,19 @@ class SimpleLogger : public Logger public: bool systemd, tty; + bool printBuildLogs; - SimpleLogger() + SimpleLogger(bool printBuildLogs) + : printBuildLogs(printBuildLogs) { systemd = getEnv("IN_SYSTEMD") == "1"; tty = isatty(STDERR_FILENO); } + bool isVerbose() override { + return printBuildLogs; + } + void log(Verbosity lvl, const FormatOrString & fs) override { if (lvl > verbosity) return; @@ -70,6 +76,18 @@ public: if (lvl <= verbosity && !s.empty()) log(lvl, s + "..."); } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + if (type == resBuildLogLine && printBuildLogs) { + auto lastLine = fields[0].s; + printError(lastLine); + } + else if (type == resPostBuildLogLine && printBuildLogs) { + auto lastLine = fields[0].s; + printError("post-build-hook: " + lastLine); + } + } }; Verbosity verbosity = lvlInfo; @@ -94,9 +112,9 @@ void writeToStderr(const string & s) } } -Logger * makeDefaultLogger() +Logger * makeSimpleLogger(bool printBuildLogs) { - return new SimpleLogger(); + return new SimpleLogger(printBuildLogs); } std::atomic<uint64_t> nextId{(uint64_t) getpid() << 32}; @@ -114,6 +132,10 @@ struct JSONLogger : Logger JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { } + bool isVerbose() override { + return true; + } + void addFields(nlohmann::json & json, const Fields & fields) { if (fields.empty()) return; diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 18c24d508..e3d91e01f 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -63,6 +63,11 @@ public: virtual ~Logger() { } + virtual void stop() { }; + + // Whether the logger prints the whole build log + virtual bool isVerbose() { return false; } + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; void log(const FormatOrString & fs) @@ -141,7 +146,7 @@ struct PushActivity extern Logger * logger; -Logger * makeDefaultLogger(); +Logger * makeSimpleLogger(bool printBuildLogs = true); Logger * makeJSONLogger(Logger & prevLogger); diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index 7cb439817..5334b046e 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -11,28 +11,28 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; auto hash = hashString(HashType::htMD5, s1); - ASSERT_EQ(hash.to_string(Base::Base16), "md5:d41d8cd98f00b204e9800998ecf8427e"); + ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; auto hash = hashString(HashType::htMD5, s2); - ASSERT_EQ(hash.to_string(Base::Base16), "md5:900150983cd24fb0d6963f7d28e17f72"); + ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } TEST(hashString, testKnownSHA256Hashes1) { @@ -40,7 +40,7 @@ namespace nix { auto s = "abc"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -48,7 +48,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -56,7 +56,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); @@ -67,7 +67,7 @@ namespace nix { auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16), + ASSERT_EQ(hash.to_string(Base::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); diff --git a/src/libutil/tests/pool.cc b/src/libutil/tests/pool.cc new file mode 100644 index 000000000..127e42dda --- /dev/null +++ b/src/libutil/tests/pool.cc @@ -0,0 +1,127 @@ +#include "pool.hh" +#include <gtest/gtest.h> + +namespace nix { + + struct TestResource + { + + TestResource() { + static int counter = 0; + num = counter++; + } + + int dummyValue = 1; + bool good = true; + int num; + }; + + /* ---------------------------------------------------------------------------- + * Pool + * --------------------------------------------------------------------------*/ + + TEST(Pool, freshPoolHasZeroCountAndSpecifiedCapacity) { + auto isGood = [](const ref<TestResource> & r) { return r->good; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + + ASSERT_EQ(pool.count(), 0); + ASSERT_EQ(pool.capacity(), 1); + } + + TEST(Pool, freshPoolCanGetAResource) { + auto isGood = [](const ref<TestResource> & r) { return r->good; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + ASSERT_EQ(pool.count(), 0); + + TestResource r = *(pool.get()); + + ASSERT_EQ(pool.count(), 1); + ASSERT_EQ(pool.capacity(), 1); + ASSERT_EQ(r.dummyValue, 1); + ASSERT_EQ(r.good, true); + } + + TEST(Pool, capacityCanBeIncremented) { + auto isGood = [](const ref<TestResource> & r) { return r->good; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + ASSERT_EQ(pool.capacity(), 1); + pool.incCapacity(); + ASSERT_EQ(pool.capacity(), 2); + } + + TEST(Pool, capacityCanBeDecremented) { + auto isGood = [](const ref<TestResource> & r) { return r->good; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + ASSERT_EQ(pool.capacity(), 1); + pool.decCapacity(); + ASSERT_EQ(pool.capacity(), 0); + } + + TEST(Pool, flushBadDropsOutOfScopeResources) { + auto isGood = [](const ref<TestResource> & r) { return false; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + + { + auto _r = pool.get(); + ASSERT_EQ(pool.count(), 1); + } + + pool.flushBad(); + ASSERT_EQ(pool.count(), 0); + } + + // Test that the resources we allocate are being reused when they are still good. + TEST(Pool, reuseResource) { + auto isGood = [](const ref<TestResource> & r) { return true; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + + // Compare the instance counter between the two handles. We expect them to be equal + // as the pool should hand out the same (still) good one again. + int counter = -1; + { + Pool<TestResource>::Handle h = pool.get(); + counter = h->num; + } // the first handle goes out of scope + + { // the second handle should contain the same resource (with the same counter value) + Pool<TestResource>::Handle h = pool.get(); + ASSERT_EQ(h->num, counter); + } + } + + // Test that the resources we allocate are being thrown away when they are no longer good. + TEST(Pool, badResourceIsNotReused) { + auto isGood = [](const ref<TestResource> & r) { return false; }; + auto createResource = []() { return make_ref<TestResource>(); }; + + Pool<TestResource> pool = Pool<TestResource>((size_t)1, createResource, isGood); + + // Compare the instance counter between the two handles. We expect them + // to *not* be equal as the pool should hand out a new instance after + // the first one was returned. + int counter = -1; + { + Pool<TestResource>::Handle h = pool.get(); + counter = h->num; + } // the first handle goes out of scope + + { + // the second handle should contain a different resource (with a + //different counter value) + Pool<TestResource>::Handle h = pool.get(); + ASSERT_NE(h->num, counter); + } + } +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 71db92d77..e0a99152b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -989,7 +989,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) { auto wrapper = [&]() { if (!options.allowVfork) - logger = makeDefaultLogger(); + logger = makeSimpleLogger(); try { #if __linux__ if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 0a058a31b..8649de5e9 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -472,6 +472,8 @@ static void _main(int argc, char * * argv) restoreSignals(); + logger->stop(); + execvp(shell->c_str(), argPtrs.data()); throw SysError("executing shell '%s'", *shell); @@ -521,6 +523,8 @@ static void _main(int argc, char * * argv) if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) store2->addPermRoot(store->parseStorePath(symlink.second), absPath(symlink.first), true); + logger->stop(); + for (auto & path : outPaths) std::cout << path << '\n'; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index d62febaff..f7b04eb2b 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1446,6 +1446,8 @@ static int _main(int argc, char * * argv) globals.state->printStats(); + logger->stop(); + return 0; } } diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc index 5a686c8cd..b645bdc1b 100644 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ b/src/nix-prefetch-url/nix-prefetch-url.cc @@ -8,7 +8,7 @@ #include "attr-path.hh" #include "finally.hh" #include "../nix/legacy.hh" -#include "../nix/progress-bar.hh" +#include "progress-bar.hh" #include "tarfile.hh" #include <iostream> diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 3a3060ad8..ef8fb0951 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -373,7 +373,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32)); + cout << fmt("%s\n", info->narHash.to_string(Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -728,7 +728,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError( "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(path), info->narHash.to_string(), current.first.to_string()); + store->printStorePath(path), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); status = 1; } } @@ -856,7 +856,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << (info->narHash ? info->narHash.to_string() : "") << info->ca << info->sigs; + out << (info->narHash ? info->narHash.to_string(Base32, true) : "") << info->ca << info->sigs; } catch (InvalidPath &) { } } @@ -1098,6 +1098,8 @@ static int _main(int argc, char * * argv) op(opFlags, opArgs); + logger->stop(); + return 0; } } diff --git a/src/nix/main.cc b/src/nix/main.cc index 1120ba5ef..203901168 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -10,6 +10,7 @@ #include "progress-bar.hh" #include "filetransfer.hh" #include "finally.hh" +#include "loggers.hh" #include <sys/types.h> #include <sys/socket.h> @@ -90,7 +91,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs .longName = "print-build-logs", .shortName = 'L', .description = "print full build logs on stderr", - .handler = {&printBuildLogs, true}, + .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, }); addFlag({ @@ -165,6 +166,10 @@ void mainWrapped(int argc, char * * argv) verbosity = lvlWarn; settings.verboseBuild = false; + setLogFormat("bar"); + + Finally f([] { logger->stop(); }); + NixArgs args; args.parseCmdline(argvToStrings(argc, argv)); @@ -178,10 +183,6 @@ void mainWrapped(int argc, char * * argv) && args.command->first != "upgrade-nix") settings.requireExperimentalFeature("nix-command"); - Finally f([]() { stopProgressBar(); }); - - startProgressBar(args.printBuildLogs); - if (args.useNet && !haveInternet()) { warn("you don't have Internet access; disabling some network-dependent features"); args.useNet = false; diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 22c569f3c..2d31894c2 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -61,11 +61,9 @@ struct CmdShowDerivation : InstallablesCommand for (auto & drvPath : drvPaths) { if (!drvPath.isDerivation()) continue; - auto drvPathS = store->printStorePath(drvPath); + auto drvObj(jsonRoot.object(store->printStorePath(drvPath))); - auto drvObj(jsonRoot.object(drvPathS)); - - auto drv = readDerivation(*store, drvPathS); + auto drv = store->readDerivation(drvPath); { auto outputsObj(drvObj.object("outputs")); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index cf1fa6a99..287dad101 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -101,7 +101,7 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError( "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(info->path), info->narHash.to_string(), hash.first.to_string()); + store->printStorePath(info->path), info->narHash.to_string(Base32, true), hash.first.to_string(Base32, true)); } } |