From d38f860c3ef001a456d4d447f89219de5e3c830c Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 10 Jun 2020 11:20:52 +0200 Subject: Add a way to get all the outputs of a derivation with their label Generalize `queryDerivationOutputNames` and `queryDerivationOutputs` by adding a `queryDerivationOutputMap` that returns the map `outputName=>outputPath` (not that this is not equivalent to merging the results of `queryDerivationOutputs` and `queryDerivationOutputNames` as sets don't preserve the order, so we would end up with an incorrect mapping). squash! Add a way to get all the outputs of a derivation with their label Rename StorePathMap to OutputPathMap --- src/libstore/build.cc | 4 ++-- src/libstore/daemon.cc | 9 +++++++++ src/libstore/local-store.cc | 11 +++++++---- src/libstore/local-store.hh | 2 +- src/libstore/path.hh | 1 + src/libstore/remote-store.cc | 38 ++++++++++++++++++++++++++++++++++++++ src/libstore/remote-store.hh | 1 + src/libstore/store-api.cc | 10 ++++++++++ src/libstore/store-api.hh | 7 +++++-- src/libstore/worker-protocol.hh | 6 ++++-- 10 files changed, 78 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0c25897f8..bb668def5 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -2750,8 +2750,8 @@ struct RestrictedStore : public LocalFSStore void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - StorePathSet queryDerivationOutputs(const StorePath & path) override - { throw Error("queryDerivationOutputs"); } + OutputPathMap queryDerivationOutputMap(const StorePath & path) override + { throw Error("queryDerivationOutputMap"); } std::optional queryPathFromHashPart(const std::string & hashPart) override { throw Error("queryPathFromHashPart"); } diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 842aef20c..6042bd610 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -347,6 +347,15 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case wopQueryDerivationOutputMap: { + auto path = store->parseStorePath(readString(from)); + logger->startWork(); + OutputPathMap outputs = store->queryDerivationOutputMap(path); + logger->stopWork(); + writeOutputPathMap(*store, to, outputs); + break; + } + case wopQueryDeriver: { auto path = store->parseStorePath(readString(from)); logger->startWork(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0dfbed9fc..eed225349 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -774,17 +774,20 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) } -StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path) +OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path) { - return retrySQLite([&]() { + return retrySQLite([&]() { auto state(_state.lock()); auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() (queryValidPathId(*state, path))); - StorePathSet outputs; + OutputPathMap outputs; while (useQueryDerivationOutputs.next()) - outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1))); + outputs.emplace( + useQueryDerivationOutputs.getStr(0), + parseStorePath(useQueryDerivationOutputs.getStr(1)) + ); return outputs; }); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e17cc45ae..ff36cb00e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -133,7 +133,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - StorePathSet queryDerivationOutputs(const StorePath & path) override; + OutputPathMap queryDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 4f79843fe..e43a8b50c 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -62,6 +62,7 @@ public: typedef std::set StorePathSet; typedef std::vector StorePaths; +typedef std::map OutputPathMap; /* Extension of derivations in the Nix store. */ const std::string drvExtension = ".drv"; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b7cc7a5fc..dc82dbdd9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,6 +38,32 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } +std::map readOutputPathMap(const Store & store, Source & from) +{ + std::map pathMap; + auto rawInput = readStrings(from); + auto curInput = rawInput.begin(); + while (curInput != rawInput.end()) { + string thisKey = *curInput; + curInput = std::next(curInput); + if (curInput == rawInput.end()) { + throw Error("Got an odd number of elements from the daemon when trying to read a map… that's odd"); + } + string thisValue = *curInput; + curInput = std::next(curInput); + pathMap.emplace(thisKey, store.parseStorePath(thisValue)); + } + return pathMap; +} + +void writeOutputPathMap(const Store & store, Sink & out, const std::map & pathMap) +{ + out << 2*pathMap.size(); + for (auto & i : pathMap) { + out << i.first; + out << store.printStorePath(i.second); + } +} /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -412,12 +438,24 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) { auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) { + return Store::queryDerivationOutputs(path); + } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); return readStorePaths(*this, conn->from); } +OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path) +{ + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return readOutputPathMap(*this, conn->from); + +} + std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 80c8e9f11..fb2052752 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -51,6 +51,7 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; + OutputPathMap queryDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index e4a4ae11e..c0a8bc9f6 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -242,6 +242,16 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } +StorePathSet Store::queryDerivationOutputs(const StorePath & path) +{ + auto outputMap = this->queryDerivationOutputMap(path); + StorePathSet outputPaths; + for (auto & i: outputMap) { + outputPaths.emplace(std::move(i.second)); + } + return outputPaths; +} + bool Store::isValidPath(const StorePath & storePath) { std::string hashPart(storePath.hashPart()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 25d78c297..b122e05d6 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -418,8 +418,11 @@ public: virtual StorePathSet queryValidDerivers(const StorePath & path) { return {}; }; /* Query the outputs of the derivation denoted by `path'. */ - virtual StorePathSet queryDerivationOutputs(const StorePath & path) - { unsupported("queryDerivationOutputs"); } + virtual StorePathSet queryDerivationOutputs(const StorePath & path); + + /* Query the mapping outputName=>outputPath for the given derivation */ + virtual OutputPathMap queryDerivationOutputMap(const StorePath & path) + { unsupported("queryDerivationOutputMap"); } /* Query the full store path given the hash part of a valid store path, or empty if the path doesn't exist. */ diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ac42457fc..8b538f6da 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -6,7 +6,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x115 +#define PROTOCOL_VERSION 0x116 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -30,7 +30,7 @@ typedef enum { wopSetOptions = 19, wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, - wopQueryDerivationOutputs = 22, + wopQueryDerivationOutputs = 22, // obsolete wopQueryAllValidPaths = 23, wopQueryFailedPaths = 24, wopClearFailedPaths = 25, @@ -49,6 +49,7 @@ typedef enum { wopNarFromPath = 38, wopAddToStoreNar = 39, wopQueryMissing = 40, + wopQueryDerivationOutputMap = 41, } WorkerOp; @@ -69,5 +70,6 @@ template T readStorePaths(const Store & store, Source & from); void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths); +void writeOutputPathMap(const Store & store, Sink & out, const OutputPathMap & paths); } -- cgit v1.2.3 From 2b834d48aae35c5050895fac540ca55387925d0f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jun 2020 22:45:41 +0200 Subject: NAR parser: Fix missing name field check Discovered by @Kloenk. --- src/libutil/archive.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6a8484705..51c88537e 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -262,7 +262,7 @@ static void parse(ParseSink & sink, Source & source, const Path & path) names[name] = 0; } } else if (s == "node") { - if (s.empty()) throw badArchive("entry name missing"); + if (name.empty()) throw badArchive("entry name missing"); parse(sink, source, path + "/" + name); } else throw badArchive("unknown field " + s); -- cgit v1.2.3 From e72a16a339092359e58ac7626ea89d4182b26642 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 30 Jun 2020 11:00:51 -0600 Subject: check for a null symbol --- src/libutil/error.hh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 1e6102ce1..6982e30aa 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -67,7 +67,11 @@ struct ErrPos { { line = pos.line; column = pos.column; - file = pos.file; + // is file symbol null? + if (pos.file.set()) + file = pos.file; + else + file = ""; return *this; } -- cgit v1.2.3 From a0705e0dd152f2a19d096d02024723a5614c7726 Mon Sep 17 00:00:00 2001 From: Ben Burdette Date: Tue, 30 Jun 2020 11:01:46 -0600 Subject: invalid pos check --- src/libutil/tests/logging.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'src') diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 6a6fb4ac3..6a58b9425 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -289,4 +289,22 @@ namespace nix { "what about this " ANSI_YELLOW "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL); } + + /* ---------------------------------------------------------------------------- + * ErrPos + * --------------------------------------------------------------------------*/ + + TEST(errpos, invalidPos) { + + // contains an invalid symbol, which we should not dereference! + Pos invalid; + + // constructing without access violation. + ErrPos ep(invalid); + + // assignment without access violation. + ep = invalid; + + } + } -- cgit v1.2.3 From 38ccf2e24168ab5c0d2a7cf6b5bdec1b0784ebcd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Jul 2020 15:31:34 +0200 Subject: Cleanup --- src/libstore/remote-store.cc | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index dc82dbdd9..8c648d671 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -42,15 +42,12 @@ std::map readOutputPathMap(const Store & store, Source & from { std::map pathMap; auto rawInput = readStrings(from); + if (rawInput.size() % 2) + throw Error("got an odd number of elements from the daemon when trying to read a output path map"); auto curInput = rawInput.begin(); while (curInput != rawInput.end()) { - string thisKey = *curInput; - curInput = std::next(curInput); - if (curInput == rawInput.end()) { - throw Error("Got an odd number of elements from the daemon when trying to read a map… that's odd"); - } - string thisValue = *curInput; - curInput = std::next(curInput); + auto thisKey = *curInput++; + auto thisValue = *curInput++; pathMap.emplace(thisKey, store.parseStorePath(thisValue)); } return pathMap; -- cgit v1.2.3 From b5e42536977d84359f5fea3f42cfefdd46799eb1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Jul 2020 18:24:11 +0200 Subject: Fix abort in 'nix develop' --- src/libutil/hash.cc | 2 +- src/nix/develop.cc | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 01fae3044..1a3e7c5d8 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -370,7 +370,7 @@ string printHashType(HashType ht) default: // illegal hash type enum value internally, as opposed to external input // which should be validated with nice error message. - abort(); + assert(false); } } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 037987313..289e3bed3 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -135,13 +135,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) drv.inputSrcs.insert(std::move(getEnvShPath)); Hash h = hashDerivationModulo(*store, drv, true); auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { - .path = shellOutPath, - .hash = FixedOutputHash { - .method = FileIngestionMethod::Flat, - .hash = Hash { }, - }, - }); + drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath }); drv.env["out"] = store->printStorePath(shellOutPath); auto shellDrvPath2 = writeDerivation(store, drv, drvName); -- cgit v1.2.3 From 223fbe644a4cde45269c01e971331d9414971d46 Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 2 Jul 2020 17:18:03 +0200 Subject: Shorten the path to the test root Fix a socket length failure on the OSX builders --- src/libstore/globals.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index bee94cbd8..683fa5196 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -35,7 +35,7 @@ Settings::Settings() , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))) , nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) - , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) + , nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { buildUsersGroup = getuid() == 0 ? "nixbld" : ""; lockCPU = getEnv("NIX_AFFINITY_HACK") == "1"; -- cgit v1.2.3 From 6f8fd3a3f22de24223bba05635aaa62107d92767 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Jul 2020 14:50:07 +0200 Subject: Shut up a clang warning --- src/libfetchers/cache.hh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src') diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index d76ab1233..3db4f081c 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -6,6 +6,8 @@ namespace nix::fetchers { struct Cache { + virtual ~Cache() { } + virtual void add( ref store, const Attrs & inAttrs, -- cgit v1.2.3 From c3c7aedbb5ac869b7c454e90683f77b9c527a75a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Jul 2020 14:58:58 +0200 Subject: nix develop: Fix bad regex This was accepted by libstdc++ but not libc++. https://hydra.nixos.org/build/123569154 --- src/nix/develop.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 289e3bed3..eb93f56fc 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -50,7 +50,7 @@ BuildEnvironment readEnvironment(const Path & path) R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; static std::string indexedArrayRegex = - R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re"; + R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; static std::regex varRegex( "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n"); -- cgit v1.2.3