From 79c169d1c615211af69c1cbc6218f7465a4f81ed Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 12 Jun 2020 09:49:09 -0500 Subject: Allow substituting from different storeDir Substituters can substitute from one store dir to another with a little bit of help. The store api just needs to have a CA so it can recompute the store path based on the new store dir. We can only do this for fixed output derivations with no references, though. --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5c36693e6..956980f49 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -309,7 +309,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos) + SubstitutablePathInfos & infos, std::map pathsCA) { if (paths.empty()) return; -- cgit v1.2.3 From 5e631e3304aaf00b912c12025debfd35942a0ca9 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Wed, 17 Jun 2020 15:03:05 -0400 Subject: Add StorePathCAMap for querySubstitutablePathInfos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I’m not 100% sure this is wanted since it kind of makes everything have to know about ca even if they don’t really want to. But it also make things easier in dealing with looking up ca. --- src/libstore/remote-store.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 396629a32..4538ccd4f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -308,18 +308,17 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) } -void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, - SubstitutablePathInfos & infos, std::map pathsCA) +void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) { - if (paths.empty()) return; + if (pathsMap.empty()) return; auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { - for (auto & i : paths) { + for (auto & i : pathsMap) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i); + conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; @@ -329,12 +328,15 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathSet & paths, info.references = readStorePaths(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i, std::move(info)); + infos.insert_or_assign(i.first, std::move(info)); } } else { conn->to << wopQuerySubstitutablePathInfos; + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); writeStorePaths(*this, conn->to, paths); conn.processStderr(); size_t count = readNum(conn->from); -- cgit v1.2.3 From 507aa48739f23f9a16c8b7079bfa6fc1806be78e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Jun 2020 18:41:33 +0000 Subject: WIP: Make Hash always store a valid hash type --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f5f2ab7fd..c206a4538 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -462,7 +462,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << wopAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash->to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << info.ca -- cgit v1.2.3 From f2a6cee334255ced72a70f527cf3c283b4586e42 Mon Sep 17 00:00:00 2001 From: Matthew Bauer Date: Fri, 19 Jun 2020 18:06:19 -0400 Subject: Update worker protocol to support sending storepath maps We need to also send the ca to daemon in addition to the path. --- src/libstore/remote-store.cc | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4538ccd4f..f6158685c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -38,6 +38,27 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } +std::map> readStorePathCAMap(const Store & store, Source & from) +{ + StorePathCAMap paths; + auto count = readNum(from); + while (count--) { + auto path = readString(from); + auto ca = readString(from); + paths.insert_or_assign(store.parseStorePath(path), !ca.empty() ? std::optional(ca) : std::nullopt); + } + return paths; +} + +void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths) +{ + out << paths.size(); + for (auto & i : paths) { + out << store.printStorePath(i.first); + out << (i.second ? *i.second : ""); + } +} + /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -334,10 +355,13 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } else { conn->to << wopQuerySubstitutablePathInfos; - StorePathSet paths; - for (auto & path : pathsMap) - paths.insert(path.first); - writeStorePaths(*this, conn->to, paths); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); + writeStorePaths(*this, conn->to, paths); + } else + writeStorePathCAMap(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { -- cgit v1.2.3 From 263ccdd48923b730fd7e6f687583160d7b24039b Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 1 Jul 2020 18:34:18 -0400 Subject: Rename two hash constructors to proper functions --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 305a47340..890d96388 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -375,7 +375,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, info = std::make_shared(StorePath(path)); auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->narHash = Hash(readString(conn->from), htSHA256); + info->narHash = Hash::parseAny(readString(conn->from), htSHA256); info->references = readStorePaths(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { -- cgit v1.2.3 From 545bb2ed03001cd7a80a90f73eb500f396c043a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Jul 2020 17:37:44 +0200 Subject: Remove 'accessor' from addToStore() This is only used by hydra-queue-runner and it's better to implement it there. --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 118aadf7e..9af4364b7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -466,7 +466,7 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) + RepairFlag repair, CheckSigsFlag checkSigs) { auto conn(getConnection()); -- cgit v1.2.3 From 0aa79dcc6f4e67891cdd9500ae4ce85e4189a951 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Fri, 17 Jul 2020 17:24:02 -0400 Subject: Remove StoreType abstraction and delegate regStore to each Store implementation. The generic regStore implementation will only be for the ambiguous shorthands, like "" and "auto". This also could get us close to simplifying the daemon command. --- src/libstore/remote-store.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 9af4364b7..a9fbf9f82 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -843,14 +843,18 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -static std::string uriScheme = "unix://"; +static std::string_view uriScheme = "unix://"; static RegisterStoreImplementation regStore([]( const std::string & uri, const Store::Params & params) -> std::shared_ptr { - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared(std::string(uri, uriScheme.size()), params); + if (hasPrefix(uri, uriScheme)) + return std::make_shared(std::string(uri, uriScheme.size()), params); + else if (uri == "daemon") + return std::make_shared(params); + else + return nullptr; }); } -- cgit v1.2.3 From 2c7557481b4c9d5113a65cc7a75c8acc18031f4e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 24 Jul 2020 21:02:51 +0000 Subject: `queryDerivationOutputMap` no longer assumes all outputs have a mapping This assumption is broken by CA derivations. Making a PR now to do the breaking daemon change as soon as possible (if it is already too late, we can bump protocol intead). --- src/libstore/remote-store.cc | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 9af4364b7..3b3cdbc02 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -39,30 +39,21 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -std::map readOutputPathMap(const Store & store, Source & from) -{ - std::map pathMap; - auto rawInput = readStrings(from); - if (rawInput.size() % 2) - throw Error("got an odd number of elements from the daemon when trying to read a output path map"); - auto curInput = rawInput.begin(); - while (curInput != rawInput.end()) { - auto thisKey = *curInput++; - auto thisValue = *curInput++; - pathMap.emplace(thisKey, store.parseStorePath(thisValue)); - } - return pathMap; + +StorePath read(const Store & store, Source & from, Proxy _) +{ + auto path = readString(from); + return store.parseStorePath(path); } -void writeOutputPathMap(const Store & store, Sink & out, const std::map & pathMap) + +void write(const Store & store, Sink & out, const StorePath & storePath) { - out << 2*pathMap.size(); - for (auto & i : pathMap) { - out << i.first; - out << store.printStorePath(i.second); - } + auto path = store.printStorePath(storePath); + out << path; } + /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : Store(params) @@ -445,12 +436,12 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -OutputPathMap RemoteStore::queryDerivationOutputMap(const StorePath & path) +std::map> RemoteStore::queryDerivationOutputMap(const StorePath & path) { auto conn(getConnection()); conn->to << wopQueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return readOutputPathMap(*this, conn->from); + return read(*this, conn->from, Proxy>> {}); } -- cgit v1.2.3 From 4c0077a07d2ee5362bd8ffb0ea9ee053c759303d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2020 00:48:39 +0200 Subject: Fix RemoteStore::addToStore() latency Since 6185d25e523a3cd223dd6f6aca10cf6ff15b4823, this was very latency-bound since it required a round-trip for every 32 KiB. So for example copying a 514 MiB closure over a virtual ethernet device with a articial delay of just 1 ms took 343s. Now it takes 2.7s. Fixes #3372. --- src/libstore/remote-store.cc | 81 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8d01c6667..4ed81d9d8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -503,9 +503,84 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; - if (!tunnel) copyNAR(source, conn->to); - conn.processStderr(0, tunnel ? &source : nullptr); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + + std::exception_ptr ex; + + struct FramedSink : BufferedSink + { + ConnectionHandle & conn; + std::exception_ptr & ex; + + FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) + { } + + ~FramedSink() + { + try { + conn->to << 0; + conn->to.flush(); + } catch (...) { + ignoreException(); + } + } + + void write(const unsigned char * data, size_t len) override + { + /* Don't send more data if the remote has + encountered an error. */ + if (ex) { + auto ex2 = ex; + ex = nullptr; + std::rethrow_exception(ex2); + } + conn->to << len; + conn->to(data, len); + }; + }; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + conn.processStderr(0, nullptr); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink(conn, ex); + copyNAR(source, sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + + } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + conn.processStderr(0, &source); + } else { + copyNAR(source, conn->to); + conn.processStderr(0, nullptr); + } } } -- cgit v1.2.3 From f63839bfa402f73e001a5ee412c3f6a6b95d1a38 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 29 Jul 2020 18:04:35 +0200 Subject: Cleanup --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4ed81d9d8..616081aa1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -545,7 +545,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, std::thread stderrThread([&]() { try { - conn.processStderr(0, nullptr); + conn.processStderr(); } catch (...) { ex = std::current_exception(); } -- cgit v1.2.3 From 3f6e88a5527dcc4d58e3147f78388a88eb8896e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Jul 2020 13:10:49 +0200 Subject: unsigned long long -> uint64_t --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 616081aa1..611f70783 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -782,7 +782,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s void RemoteStore::queryMissing(const std::vector & targets, StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown, - unsigned long long & downloadSize, unsigned long long & narSize) + uint64_t & downloadSize, uint64_t & narSize) { { auto conn(getConnection()); -- cgit v1.2.3 From 2f2ae993dc6d35e9c0e66e893e5d615116d42917 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 19:02:05 +0000 Subject: WIP systematize more of the worker protocol This refactor should *not* change the wire protocol. --- src/libstore/remote-store.cc | 100 +++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 61 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1200ab200..de50b3e2e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -23,66 +23,44 @@ namespace nix { -template<> StorePathSet readStorePaths(const Store & store, Source & from) -{ - StorePathSet paths; - for (auto & i : readStrings(from)) - paths.insert(store.parseStorePath(i)); - return paths; -} - - -void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths) +void write(const Store & store, Sink & out, const StorePathSet & paths) { out << paths.size(); for (auto & i : paths) out << store.printStorePath(i); } -StorePath read(const Store & store, Source & from, Proxy _) + +std::string read(const Store & store, Source & from, Proxy _) { - auto path = readString(from); - return store.parseStorePath(path); + return readString(from); } -StorePathCAMap readStorePathCAMap(const Store & store, Source & from) +void write(const Store & store, Sink & out, const std::string & str) { - StorePathCAMap paths; - auto count = readNum(from); - while (count--) - paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); - return paths; + out << str; } -void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & paths) + +StorePath read(const Store & store, Source & from, Proxy _) { - out << paths.size(); - for (auto & i : paths) { - out << store.printStorePath(i.first); - out << renderContentAddress(i.second); - } + return store.parseStorePath(readString(from)); } -std::map readOutputPathMap(const Store & store, Source & from) +void write(const Store & store, Sink & out, const StorePath & storePath) { - std::map pathMap; - auto rawInput = readStrings(from); - if (rawInput.size() % 2) - throw Error("got an odd number of elements from the daemon when trying to read a output path map"); - auto curInput = rawInput.begin(); - while (curInput != rawInput.end()) { - auto thisKey = *curInput++; - auto thisValue = *curInput++; - pathMap.emplace(thisKey, store.parseStorePath(thisValue)); - } - return pathMap; + out << store.printStorePath(storePath); } -void write(const Store & store, Sink & out, const StorePath & storePath) +ContentAddress read(const Store & store, Source & from, Proxy _) +{ + return parseContentAddress(readString(from)); +} + +void write(const Store & store, Sink & out, const ContentAddress & ca) { - auto path = store.printStorePath(storePath); - out << path; + out << renderContentAddress(ca); } @@ -319,9 +297,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - writeStorePaths(*this, conn->to, paths); + write(*this, conn->to, paths); conn.processStderr(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } } @@ -331,7 +309,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } @@ -348,9 +326,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - writeStorePaths(*this, conn->to, paths); + write(*this, conn->to, paths); conn.processStderr(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } } @@ -372,7 +350,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths(*this, conn->from); + info.references = read(*this, conn->from, Proxy {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -385,9 +363,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - writeStorePaths(*this, conn->to, paths); + write(*this, conn->to, paths); } else - writeStorePathCAMap(*this, conn->to, pathsMap); + write(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -395,7 +373,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = readStorePaths(*this, conn->from); + info.references = read(*this, conn->from, Proxy {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -428,7 +406,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash(readString(conn->from), htSHA256); - info->references = readStorePaths(*this, conn->from); + info->references = read(*this, conn->from, Proxy {}); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -447,7 +425,7 @@ void RemoteStore::queryReferrers(const StorePath & path, auto conn(getConnection()); conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : readStorePaths(*this, conn->from)) + for (auto & i : read(*this, conn->from, Proxy {})) referrers.insert(i); } @@ -457,7 +435,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) auto conn(getConnection()); conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } @@ -469,7 +447,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return readStorePaths(*this, conn->from); + return read(*this, conn->from, Proxy {}); } @@ -508,7 +486,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - writeStorePaths(*this, sink, info.references); + write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -518,7 +496,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = readStorePaths(*this, conn->from); + auto importedPaths = read(*this, conn->from, Proxy {}); assert(importedPaths.size() <= 1); } @@ -527,7 +505,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash->to_string(Base16, false); - writeStorePaths(*this, conn->to, info.references); + write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -660,7 +638,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, auto conn(getConnection()); conn->to << wopAddTextToStore << name << s; - writeStorePaths(*this, conn->to, references); + write(*this, conn->to, references); conn.processStderr(); return parseStorePath(readString(conn->from)); @@ -762,7 +740,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->to << wopCollectGarbage << options.action; - writeStorePaths(*this, conn->to, options.pathsToDelete); + write(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -824,9 +802,9 @@ void RemoteStore::queryMissing(const std::vector & targets ss.push_back(p.to_string(*this)); conn->to << ss; conn.processStderr(); - willBuild = readStorePaths(*this, conn->from); - willSubstitute = readStorePaths(*this, conn->from); - unknown = readStorePaths(*this, conn->from); + willBuild = read(*this, conn->from, Proxy {}); + willSubstitute = read(*this, conn->from, Proxy {}); + unknown = read(*this, conn->from, Proxy {}); conn->from >> downloadSize >> narSize; return; } -- cgit v1.2.3 From 1bab8a321f246c1c202268851b0e706ff5030d2e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 21:56:42 +0000 Subject: Remove unneeded definition Template instantiations will cover this case fine. --- src/libstore/remote-store.cc | 8 -------- 1 file changed, 8 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index de50b3e2e..5de8f95a7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -23,14 +23,6 @@ namespace nix { -void write(const Store & store, Sink & out, const StorePathSet & paths) -{ - out << paths.size(); - for (auto & i : paths) - out << store.printStorePath(i); -} - - std::string read(const Store & store, Source & from, Proxy _) { return readString(from); -- cgit v1.2.3 From 45b6fdb22b05352108b6470ae16d611431c06666 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 22:10:13 +0000 Subject: Remove unused functions --- src/libstore/remote-store.cc | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1200ab200..dc2efefeb 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -63,22 +63,6 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & } } -std::map readOutputPathMap(const Store & store, Source & from) -{ - std::map pathMap; - auto rawInput = readStrings(from); - if (rawInput.size() % 2) - throw Error("got an odd number of elements from the daemon when trying to read a output path map"); - auto curInput = rawInput.begin(); - while (curInput != rawInput.end()) { - auto thisKey = *curInput++; - auto thisValue = *curInput++; - pathMap.emplace(thisKey, store.parseStorePath(thisValue)); - } - return pathMap; -} - - void write(const Store & store, Sink & out, const StorePath & storePath) { auto path = store.printStorePath(storePath); -- cgit v1.2.3 From 1dfcbebc95f1c74763770bb2bd7afdebe8804845 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 22:28:10 +0000 Subject: Organize and format code a bit --- src/libstore/remote-store.cc | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index dc2efefeb..772979066 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -31,7 +31,6 @@ template<> StorePathSet readStorePaths(const Store & store, Source & from) return paths; } - void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths) { out << paths.size(); @@ -39,11 +38,6 @@ void writeStorePaths(const Store & store, Sink & out, const StorePathSet & paths out << store.printStorePath(i); } -StorePath read(const Store & store, Source & from, Proxy _) -{ - auto path = readString(from); - return store.parseStorePath(path); -} StorePathCAMap readStorePathCAMap(const Store & store, Source & from) { @@ -63,10 +57,17 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & } } + +StorePath read(const Store & store, Source & from, Proxy _) +{ + auto path = readString(from); + return store.parseStorePath(path); +} + void write(const Store & store, Sink & out, const StorePath & storePath) { - auto path = store.printStorePath(storePath); - out << path; + auto path = store.printStorePath(storePath); + out << path; } -- cgit v1.2.3 From 16c98bf57c52dee59c06e0e8911800943a468962 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 4 Aug 2020 22:36:31 +0000 Subject: Get rid of some unneeded temporaries --- src/libstore/remote-store.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 772979066..63c19a450 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -60,14 +60,12 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & StorePath read(const Store & store, Source & from, Proxy _) { - auto path = readString(from); - return store.parseStorePath(path); + return store.parseStorePath(readString(from)); } void write(const Store & store, Sink & out, const StorePath & storePath) { - auto path = store.printStorePath(storePath); - out << path; + out << store.printStorePath(storePath); } -- cgit v1.2.3 From 1d71028f4d43f8e3ade559f7187f697c7bb9d709 Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Wed, 5 Aug 2020 14:42:48 -0400 Subject: Remove optionality in ValidPathInfo::narInfo --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 33d1e431b..b5e4d9fa6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -521,7 +521,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << wopAddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash->to_string(Base16, false); + << info.narHash.to_string(Base16, false); writeStorePaths(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) -- cgit v1.2.3 From ed96e603e116123c1e44f5108b67b472b2c96538 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Aug 2020 19:44:08 +0000 Subject: Proxy -> Phantom to match Rust Sorry, Haskell. --- src/libstore/remote-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 377c81ff4..2c0857466 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -58,7 +58,7 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & } -StorePath read(const Store & store, Source & from, Proxy _) +StorePath read(const Store & store, Source & from, Phantom _) { return store.parseStorePath(readString(from)); } @@ -461,7 +461,7 @@ std::map> RemoteStore::queryDerivationOutp auto conn(getConnection()); conn->to << wopQueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return read(*this, conn->from, Proxy>> {}); + return read(*this, conn->from, Phantom>> {}); } -- cgit v1.2.3 From 6c66331d5c34d97092a9b5e2832fed73e0da3c87 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Aug 2020 20:37:48 +0000 Subject: WIP: Put the worker protocol `read` and `write` in a namespace to disambig --- src/libstore/remote-store.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 2c0857466..96e0aabba 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -58,6 +58,8 @@ void writeStorePathCAMap(const Store & store, Sink & out, const StorePathCAMap & } +namespace worker_proto { + StorePath read(const Store & store, Source & from, Phantom _) { return store.parseStorePath(readString(from)); @@ -68,6 +70,8 @@ void write(const Store & store, Sink & out, const StorePath & storePath) out << store.printStorePath(storePath); } +} + /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -461,7 +465,7 @@ std::map> RemoteStore::queryDerivationOutp auto conn(getConnection()); conn->to << wopQueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return read(*this, conn->from, Phantom>> {}); + return worker_proto::read(*this, conn->from, Phantom>> {}); } -- cgit v1.2.3 From e89b5bd0bfeb4dfdd8fe7e6929544cb9ceb8a505 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 6 Aug 2020 18:31:48 +0000 Subject: Minimize the usage of `Hash::dummy` --- src/libstore/remote-store.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b5e4d9fa6..8dcc1d710 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -419,10 +419,10 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - info = std::make_shared(StorePath(path)); auto deriver = readString(conn->from); + auto narHash = Hash::parseAny(readString(conn->from), htSHA256); + info = std::make_shared(path, narHash); if (deriver != "") info->deriver = parseStorePath(deriver); - info->narHash = Hash::parseAny(readString(conn->from), htSHA256); info->references = readStorePaths(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { -- cgit v1.2.3 From 9ab07e99f527d1fa3adfa02839da477a1528d64b Mon Sep 17 00:00:00 2001 From: Carlo Nucera Date: Thu, 6 Aug 2020 18:04:13 -0400 Subject: Use template structs instead of phantoms --- src/libstore/remote-store.cc | 61 ++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 33 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ac776b95a..48450eb67 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -22,40 +22,36 @@ namespace nix { -namespace worker_proto { - -std::string read(const Store & store, Source & from, Phantom _) +std::string WorkerProto::read(const Store & store, Source & from) { return readString(from); } -void write(const Store & store, Sink & out, const std::string & str) +void WorkerProto::write(const Store & store, Sink & out, const std::string & str) { out << str; } -StorePath read(const Store & store, Source & from, Phantom _) +StorePath WorkerProto::read(const Store & store, Source & from) { return store.parseStorePath(readString(from)); } -void write(const Store & store, Sink & out, const StorePath & storePath) +void WorkerProto::write(const Store & store, Sink & out, const StorePath & storePath) { out << store.printStorePath(storePath); } -ContentAddress read(const Store & store, Source & from, Phantom _) +ContentAddress WorkerProto::read(const Store & store, Source & from) { return parseContentAddress(readString(from)); } -void write(const Store & store, Sink & out, const ContentAddress & ca) +void WorkerProto::write(const Store & store, Sink & out, const ContentAddress & ca) { out << renderContentAddress(ca); } -} - /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -290,9 +286,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - nix::worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } } @@ -302,7 +298,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return nix::worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -319,9 +315,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - nix::worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } } @@ -343,7 +339,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references = WorkerProto::read(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -356,9 +352,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, conn->to, paths); } else - worker_proto::write(*this, conn->to, pathsMap); + WorkerProto::write(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -366,7 +362,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom {}); + info.references = WorkerProto::read(*this, conn->from); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -399,7 +395,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); info->narHash = Hash::parseAny(readString(conn->from), htSHA256); - info->references = worker_proto::read(*this, conn->from, Phantom {}); + info->references = WorkerProto::read(*this, conn->from); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -418,7 +414,7 @@ void RemoteStore::queryReferrers(const StorePath & path, auto conn(getConnection()); conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) + for (auto & i : WorkerProto::read(*this, conn->from)) referrers.insert(i); } @@ -428,7 +424,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) auto conn(getConnection()); conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -440,7 +436,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom {}); + return WorkerProto::read(*this, conn->from); } @@ -449,8 +445,7 @@ std::map> RemoteStore::queryDerivationOutp auto conn(getConnection()); conn->to << wopQueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom>> {}); - + return WorkerProto>>::read(*this, conn->from); } std::optional RemoteStore::queryPathFromHashPart(const std::string & hashPart) @@ -479,7 +474,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + WorkerProto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -489,7 +484,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); + auto importedPaths = WorkerProto::read(*this, conn->from); assert(importedPaths.size() <= 1); } @@ -498,7 +493,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash->to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + WorkerProto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -631,7 +626,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, auto conn(getConnection()); conn->to << wopAddTextToStore << name << s; - worker_proto::write(*this, conn->to, references); + WorkerProto::write(*this, conn->to, references); conn.processStderr(); return parseStorePath(readString(conn->from)); @@ -733,7 +728,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->to << wopCollectGarbage << options.action; - worker_proto::write(*this, conn->to, options.pathsToDelete); + WorkerProto::write(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -795,9 +790,9 @@ void RemoteStore::queryMissing(const std::vector & targets ss.push_back(p.to_string(*this)); conn->to << ss; conn.processStderr(); - willBuild = worker_proto::read(*this, conn->from, Phantom {}); - willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); - unknown = worker_proto::read(*this, conn->from, Phantom {}); + willBuild = WorkerProto::read(*this, conn->from); + willSubstitute = WorkerProto::read(*this, conn->from); + unknown = WorkerProto::read(*this, conn->from); conn->from >> downloadSize >> narSize; return; } -- cgit v1.2.3 From 47644e49ca252441fcc3ae57f5a01e7fc579ba8c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 7 Aug 2020 17:05:14 +0000 Subject: Specialize `std::optional` so this is backwards compatible While I am cautious to break parametricity, I think it's OK in this cases---we're not about to try to do some crazy polymorphic protocol anytime soon. --- src/libstore/remote-store.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 96e0aabba..005415666 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -70,6 +70,20 @@ void write(const Store & store, Sink & out, const StorePath & storePath) out << store.printStorePath(storePath); } + +template<> +std::optional read(const Store & store, Source & from, Phantom> _) +{ + auto s = readString(from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +template<> +void write(const Store & store, Sink & out, const std::optional & storePathOpt) +{ + out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + } -- cgit v1.2.3 From a83694c7a1212567ec2f032f84c0d72722bcf5ea Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 19 Aug 2020 19:34:47 +0000 Subject: Use `RemoteStore` to open connection for proxying daemon Removes duplicate websocket opening code, and also means we should be able to to ssh-ssh-... daemon relays, not just uds-uds-... ones. --- src/libstore/remote-store.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 553069b89..ff7149643 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -86,7 +86,16 @@ RemoteStore::RemoteStore(const Params & params) : Store(params) , connections(make_ref>( std::max(1, (int) maxConnections), - [this]() { return openConnectionWrapper(); }, + [this]() { + auto conn = openConnectionWrapper(); + try { + initConnection(*conn); + } catch (...) { + failed = true; + throw; + } + return conn; + }, [this](const ref & r) { return r->to.good() @@ -169,8 +178,6 @@ ref UDSRemoteStore::openConnection() conn->startTime = std::chrono::steady_clock::now(); - initConnection(*conn); - return conn; } -- cgit v1.2.3 From 9a9d834dc7bde0a4eafa2f7412e7a2f0df8c3262 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 20 Aug 2020 14:12:51 +0000 Subject: Rename drv output querying functions - `queryDerivationOutputMapAssumeTotal` -> `queryPartialDerivationOutputMap` - `queryDerivationOutputMapAssumeTotal` -> `queryDerivationOutputMap` --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 005415666..3ee907d1a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -474,7 +474,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -std::map> RemoteStore::queryDerivationOutputMap(const StorePath & path) +std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) { auto conn(getConnection()); conn->to << wopQueryDerivationOutputMap << printStorePath(path); -- cgit v1.2.3 From 422affe1027058c7807adec0dac1aee09b65ec4c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 21 Aug 2020 19:19:14 +0000 Subject: tabs -> spaces Sorry I let the tab sneak in there in the first place. --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index c2c6eff66..adba17c5e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -74,7 +74,7 @@ void write(const Store & store, Sink & out, const StorePath & storePath) template<> std::optional read(const Store & store, Source & from, Phantom> _) { - auto s = readString(from); + auto s = readString(from); return s == "" ? std::optional {} : store.parseStorePath(s); } -- cgit v1.2.3 From a0f19d9f3a009961869a077922f15986ce05738e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 27 Aug 2020 14:48:08 +0200 Subject: RemoteStore::addToStore(): Fix race between stderrThread and NAR writer As pointed out by @B4dM4n, the call to to.flush() on stderrThread is unsafe because the NAR writer thread is also writing to 'to'. Fixes #3943. --- src/libstore/remote-store.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index adba17c5e..e4a4ef5af 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -284,9 +284,9 @@ struct ConnectionHandle RemoteStore::Connection * operator -> () { return &*handle; } - void processStderr(Sink * sink = 0, Source * source = 0) + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) { - auto ex = handle->processStderr(sink, source); + auto ex = handle->processStderr(sink, source, flush); if (ex) { daemonException = true; std::rethrow_exception(ex); @@ -535,6 +535,8 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + conn->to.flush(); + std::exception_ptr ex; struct FramedSink : BufferedSink @@ -574,7 +576,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, std::thread stderrThread([&]() { try { - conn.processStderr(); + conn.processStderr(nullptr, nullptr, false); } catch (...) { ex = std::current_exception(); } @@ -884,9 +886,10 @@ static Logger::Fields readFields(Source & from) } -std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source) +std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush) { - to.flush(); + if (flush) + to.flush(); while (true) { -- cgit v1.2.3 From 525b38eee8fac48eb2a82fb78fa0a933a9eee2a4 Mon Sep 17 00:00:00 2001 From: aszlig Date: Sun, 13 Sep 2020 02:40:51 +0200 Subject: Fix unspecified behaviour in readStorePathCAMap When deploying a Hydra instance with current Nix master, most builds would not run because of errors like this: queue monitor: error: --- Error --- hydra-queue-runner error: --- UsageError --- nix-daemon not a content address because it is not in the form ':': /nix/store/...-somedrv The last error message is from parseContentAddress, which expects a colon-separated string, however what we got here is a store path. Looking at the worker protocol, the following message sent to the Nix daemon caused the error above: 0x1E -> wopQuerySubstitutablePathInfos 0x01 -> Number of paths 0x16 -> Length of string "/nix/store/...-somedrv" 0x00 -> Length of string "" Looking at writeStorePathCAMap, the store path is indeed the first field that's transmitted. However, readStorePathCAMap expects it to be the *second* field *on my machine*, since expression evaluation order is a classic form of unspecified behaviour[1] in C++. This has been introduced in https://github.com/NixOS/nix/pull/3689, specifically in commit 66a62b3189c8c9b0965850e6b3c9b0fda0b50fd8. [1]: https://en.wikipedia.org/wiki/Unspecified_behavior#Order_of_evaluation_of_subexpressions Signed-off-by: aszlig --- src/libstore/remote-store.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e4a4ef5af..8bcf92301 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -43,8 +43,11 @@ StorePathCAMap readStorePathCAMap(const Store & store, Source & from) { StorePathCAMap paths; auto count = readNum(from); - while (count--) - paths.insert_or_assign(store.parseStorePath(readString(from)), parseContentAddressOpt(readString(from))); + while (count--) { + auto path = store.parseStorePath(readString(from)); + auto ca = parseContentAddressOpt(readString(from)); + paths.insert_or_assign(path, ca); + } return paths; } -- cgit v1.2.3 From 057c6203b5337cb4833958f4a4f29f6587f4eb2f Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 7 Sep 2020 11:26:09 +0200 Subject: gracefully handle old daemon versions Add a fallback path in `queryPartialDerivationOutputMap` for daemons that don't support it. Also upstreams a couple methods from `SSHStore` to `RemoteStore` as this is needed to handle the fallback path. --- src/libstore/remote-store.cc | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8bcf92301..dc61951d3 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,5 +1,6 @@ #include "serialise.hh" #include "util.hh" +#include "remote-fs-accessor.hh" #include "remote-store.hh" #include "worker-protocol.hh" #include "archive.hh" @@ -479,10 +480,26 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) { - auto conn(getConnection()); - conn->to << wopQueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom>> {}); + if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return worker_proto::read(*this, conn->from, Phantom>> {}); + } else { + // Fallback for old daemon versions. + // For floating-CA derivations (and their co-dependencies) this is an + // under-approximation as it only returns the paths that can be inferred + // from the derivation itself (and not the ones that are known because + // the have been built), but as old stores don't handle floating-CA + // derivations this shouldn't matter + auto derivation = readDerivation(path); + auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); + std::map> ret; + for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { + ret.emplace(outputName, outputAndPath.second); + } + return ret; + } } @@ -871,6 +888,18 @@ RemoteStore::Connection::~Connection() } } +void RemoteStore::narFromPath(const StorePath & path, Sink & sink) +{ + auto conn(connections->get()); + conn->to << wopNarFromPath << printStorePath(path); + conn->processStderr(); + copyNAR(conn->from, sink); +} + +ref RemoteStore::getFSAccessor() +{ + return make_ref(ref(shared_from_this())); +} static Logger::Fields readFields(Source & from) { -- cgit v1.2.3 From 7d5bdf8b5679cc7b2b9b4d9caf5af9ca52211336 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 8 Sep 2020 14:50:23 +0200 Subject: Make the store plugins more introspectable Directly register the store classes rather than a function to build an instance of them. This gives the possibility to introspect static members of the class or choose different ways of instantiating them. --- src/libstore/remote-store.cc | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index dc61951d3..824b6b140 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -982,14 +982,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -static std::string uriScheme = "unix://"; - -static RegisterStoreImplementation regStore([]( - const std::string & uri, const Store::Params & params) - -> std::shared_ptr -{ - if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; - return std::make_shared(std::string(uri, uriScheme.size()), params); -}); +[[maybe_unused]] static RegisterStoreImplementation regStore(); } -- cgit v1.2.3 From fa32560169ffa55b9e0b6d5f04c3792acf0c544a Mon Sep 17 00:00:00 2001 From: regnat Date: Wed, 9 Sep 2020 11:18:12 +0200 Subject: Fix the registration of stores --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 824b6b140..5f455a62a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -982,6 +982,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -[[maybe_unused]] static RegisterStoreImplementation regStore(); +static RegisterStoreImplementation regStore; } -- cgit v1.2.3 From 22afa8fb4dd7e459faf531dd8a9c3580d02c7e2a Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 10 Sep 2020 10:55:51 +0200 Subject: Separate store configs from the implems Rework the `Store` hierarchy so that there's now one hierarchy for the store configs and one for the implementations (where each implementation extends the corresponding config). So a class hierarchy like ``` StoreConfig-------->Store | | v v SubStoreConfig----->SubStore | | v v SubSubStoreConfig-->SubSubStore ``` (with virtual inheritance to prevent DDD). The advantage of this architecture is that we can now introspect the configuration of a store without having to instantiate the store itself --- src/libstore/remote-store.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5f455a62a..619e9a7f1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -94,6 +94,7 @@ void write(const Store & store, Sink & out, const std::optional & sto /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : Store(params) + , RemoteStoreConfig(params) , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { return openConnectionWrapper(); }, @@ -124,6 +125,7 @@ ref RemoteStore::openConnectionWrapper() UDSRemoteStore::UDSRemoteStore(const Params & params) : Store(params) + , UDSRemoteStoreConfig(params) , LocalFSStore(params) , RemoteStore(params) { @@ -131,11 +133,9 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) - : Store(params) - , LocalFSStore(params) - , RemoteStore(params) - , path(socket_path) + : UDSRemoteStore(params) { + path.emplace(socket_path); } @@ -982,6 +982,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regStore; } -- cgit v1.2.3 From 5895184df44e86ae55270390402c8263b0f24ae2 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 11 Sep 2020 11:06:18 +0200 Subject: Correctly call all the parent contructors of the stores Using virtual inheritance means that only the default constructors of the parent classes will be called, which isn't what we want --- src/libstore/remote-store.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 619e9a7f1..9bbb69b99 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -124,7 +124,10 @@ ref RemoteStore::openConnectionWrapper() UDSRemoteStore::UDSRemoteStore(const Params & params) - : Store(params) + : StoreConfig(params) + , Store(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) , UDSRemoteStoreConfig(params) , LocalFSStore(params) , RemoteStore(params) -- cgit v1.2.3 From 7f103dcddd4cf3c0748b7a379a117f3262c4c5f7 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 11 Sep 2020 11:11:05 +0200 Subject: Properly filter the stores according to their declared uriSchemes When opening a store, only try the stores whose `uriSchemes()` include the current one --- src/libstore/remote-store.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 9bbb69b99..561502cbe 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -135,7 +135,10 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) } -UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) +UDSRemoteStore::UDSRemoteStore( + [[maybe_unused]] const std::string scheme, + std::string socket_path, + const Params & params) : UDSRemoteStore(params) { path.emplace(socket_path); -- cgit v1.2.3 From f24f0888f93b0881e2cefa1ceb4c8fc187c94f21 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 14 Sep 2020 11:18:45 +0200 Subject: Document the new store hierarchy --- src/libstore/remote-store.cc | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 561502cbe..4c8ffe71e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -126,9 +126,6 @@ ref RemoteStore::openConnectionWrapper() UDSRemoteStore::UDSRemoteStore(const Params & params) : StoreConfig(params) , Store(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(params) , LocalFSStore(params) , RemoteStore(params) { -- cgit v1.2.3 From 888f7afe9fe1a69c27bcc15a3e0fb0be26a11172 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 14 Sep 2020 14:12:47 +0200 Subject: Fix build issues with gcc --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4c8ffe71e..1abe236f7 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -133,7 +133,7 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) UDSRemoteStore::UDSRemoteStore( - [[maybe_unused]] const std::string scheme, + const std::string scheme, std::string socket_path, const Params & params) : UDSRemoteStore(params) -- cgit v1.2.3 From 14b30b3f3d5af75c210a15cb128e67c0eff66149 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 17:36:16 +0200 Subject: Move FramedSource and FramedSink, extract withFramedSink --- src/libstore/remote-store.cc | 151 +++++++++++++++++++++++-------------------- 1 file changed, 80 insertions(+), 71 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e92b94975..993b05dc0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -306,6 +306,8 @@ struct ConnectionHandle std::rethrow_exception(ex); } } + + void withFramedSink(std::function fun); }; @@ -564,78 +566,9 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << repair << !checkSigs; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { - - conn->to.flush(); - - std::exception_ptr ex; - - struct FramedSink : BufferedSink - { - ConnectionHandle & conn; - std::exception_ptr & ex; - - FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) - { } - - ~FramedSink() - { - try { - conn->to << 0; - conn->to.flush(); - } catch (...) { - ignoreException(); - } - } - - void write(const unsigned char * data, size_t len) override - { - /* Don't send more data if the remote has - encountered an error. */ - if (ex) { - auto ex2 = ex; - ex = nullptr; - std::rethrow_exception(ex2); - } - conn->to << len; - conn->to(data, len); - }; - }; - - /* Handle log messages / exceptions from the remote on a - separate thread. */ - std::thread stderrThread([&]() - { - try { - conn.processStderr(nullptr, nullptr, false); - } catch (...) { - ex = std::current_exception(); - } - }); - - Finally joinStderrThread([&]() - { - if (stderrThread.joinable()) { - stderrThread.join(); - if (ex) { - try { - std::rethrow_exception(ex); - } catch (...) { - ignoreException(); - } - } - } - }); - - { - FramedSink sink(conn, ex); + conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); - sink.flush(); - } - - stderrThread.join(); - if (ex) - std::rethrow_exception(ex); - + }); } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { conn.processStderr(0, &source); } else { @@ -992,6 +925,82 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } + +struct FramedSink : nix::BufferedSink +{ + ConnectionHandle & conn; + std::exception_ptr & ex; + + FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) + { } + + ~FramedSink() + { + try { + conn->to << 0; + conn->to.flush(); + } catch (...) { + ignoreException(); + } + } + + void write(const unsigned char * data, size_t len) override + { + /* Don't send more data if the remote has + encountered an error. */ + if (ex) { + auto ex2 = ex; + ex = nullptr; + std::rethrow_exception(ex2); + } + conn->to << len; + conn->to(data, len); + }; +}; + +void +ConnectionHandle::withFramedSink(std::function fun) { + (*this)->to.flush(); + + std::exception_ptr ex; + + /* Handle log messages / exceptions from the remote on a + separate thread. */ + std::thread stderrThread([&]() + { + try { + processStderr(nullptr, nullptr, false); + } catch (...) { + ex = std::current_exception(); + } + }); + + Finally joinStderrThread([&]() + { + if (stderrThread.joinable()) { + stderrThread.join(); + if (ex) { + try { + std::rethrow_exception(ex); + } catch (...) { + ignoreException(); + } + } + } + }); + + { + FramedSink sink(*this, ex); + fun(sink); + sink.flush(); + } + + stderrThread.join(); + if (ex) + std::rethrow_exception(ex); + +} + static RegisterStoreImplementation regStore; } -- cgit v1.2.3 From e34fe47d0ce19fc7657970fb0e610bffbc3e43f0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 19:27:11 +0200 Subject: Overhaul wopAddToStore --- src/libstore/remote-store.cc | 103 ++++++++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 40 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 993b05dc0..14ad4767d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -526,6 +526,69 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } +StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashType, RepairFlag repair) +{ + if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon"); + StorePathSet references; + + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + + conn->to + << wopAddToStore + << name + << renderContentAddressMethod(FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }); + writeStorePaths(*this, conn->to, references); + + conn.withFramedSink([&](Sink & sink) { + dump.drainInto(sink); + }); + + return parseStorePath(readString(conn->from)); + + } + else { + if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); + + conn->to + << wopAddToStore + << name + << ((hashType == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (method == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(hashType); + + try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + if (method == FileIngestionMethod::Recursive) { + dump.drainInto(conn->to); + } else { + std::string contents = dump.drain(); + dumpString(contents, conn->to); + } + } + conn->to.warn = false; + conn.processStderr(); + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + conn.processStderr(); + } catch (EndOfFile & e) { } + throw; + } + + return parseStorePath(readString(conn->from)); + } +} + + void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { @@ -579,46 +642,6 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, } -StorePath RemoteStore::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) -{ - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - - Path srcPath(absPath(_srcPath)); - - conn->to - << wopAddToStore - << name - << ((hashAlgo == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (method == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(hashAlgo); - - try { - conn->to.written = 0; - conn->to.warn = true; - connections->incCapacity(); - { - Finally cleanup([&]() { connections->decCapacity(); }); - dumpPath(srcPath, conn->to, filter); - } - conn->to.warn = false; - conn.processStderr(); - } catch (SysError & e) { - /* Daemon closed while we were sending the path. Probably OOM - or I/O error. */ - if (e.errNo == EPIPE) - try { - conn.processStderr(); - } catch (EndOfFile & e) { } - throw; - } - - return parseStorePath(readString(conn->from)); -} - - StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { -- cgit v1.2.3 From c602ebfb34de3626fa0b9110face6ea4b171ac0f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 20:19:15 +0200 Subject: Refactor wopAddToStore to make wopAddTextToStore obsolete --- src/libstore/remote-store.cc | 91 ++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 41 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 14ad4767d..95e6a3291 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -526,12 +526,8 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } -StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashType, RepairFlag repair) +StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references) { - if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon"); - StorePathSet references; - auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { @@ -539,7 +535,7 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, conn->to << wopAddToStore << name - << renderContentAddressMethod(FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }); + << renderContentAddressMethod(caMethod); writeStorePaths(*this, conn->to, references); conn.withFramedSink([&](Sink & sink) { @@ -552,42 +548,60 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); - conn->to - << wopAddToStore - << name - << ((hashType == htSHA256 && method == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (method == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(hashType); + std::visit(overloaded { + [&](TextHashMethod thm) -> void { + std::string s = dump.drain(); + conn->to << wopAddTextToStore << name << s; + writeStorePaths(*this, conn->to, references); + conn.processStderr(); + }, + [&](FixedOutputHashMethod fohm) -> void { + conn->to + << wopAddToStore + << name + << ((fohm.hashType == htSHA256 && fohm.fileIngestionMethod == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << (fohm.fileIngestionMethod == FileIngestionMethod::Recursive ? 1 : 0) + << printHashType(fohm.hashType); - try { - conn->to.written = 0; - conn->to.warn = true; - connections->incCapacity(); - { - Finally cleanup([&]() { connections->decCapacity(); }); - if (method == FileIngestionMethod::Recursive) { - dump.drainInto(conn->to); - } else { - std::string contents = dump.drain(); - dumpString(contents, conn->to); - } - } - conn->to.warn = false; - conn.processStderr(); - } catch (SysError & e) { - /* Daemon closed while we were sending the path. Probably OOM - or I/O error. */ - if (e.errNo == EPIPE) try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + if (fohm.fileIngestionMethod == FileIngestionMethod::Recursive) { + dump.drainInto(conn->to); + } else { + std::string contents = dump.drain(); + dumpString(contents, conn->to); + } + } + conn->to.warn = false; conn.processStderr(); - } catch (EndOfFile & e) { } - throw; - } + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + conn.processStderr(); + } catch (EndOfFile & e) { } + throw; + } + } + }, caMethod); return parseStorePath(readString(conn->from)); } } +StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, + FileIngestionMethod method, HashType hashType, RepairFlag repair) +{ + if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon"); + StorePathSet references; + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references); +} + void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) @@ -646,13 +660,8 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { if (repair) throw Error("repairing is not supported when building through the Nix daemon"); - - auto conn(getConnection()); - conn->to << wopAddTextToStore << name << s; - writeStorePaths(*this, conn->to, references); - - conn.processStderr(); - return parseStorePath(readString(conn->from)); + StringSource source(s); + return addCAToStore(source, name, TextHashMethod{}, references); } -- cgit v1.2.3 From 8279178b078103f816bf85f5a153a4845d30cc83 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 17 Sep 2020 22:01:35 +0200 Subject: Move FramedSink next to FramedSource --- src/libstore/remote-store.cc | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 95e6a3291..cfbf23ac4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -957,39 +957,6 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } - -struct FramedSink : nix::BufferedSink -{ - ConnectionHandle & conn; - std::exception_ptr & ex; - - FramedSink(ConnectionHandle & conn, std::exception_ptr & ex) : conn(conn), ex(ex) - { } - - ~FramedSink() - { - try { - conn->to << 0; - conn->to.flush(); - } catch (...) { - ignoreException(); - } - } - - void write(const unsigned char * data, size_t len) override - { - /* Don't send more data if the remote has - encountered an error. */ - if (ex) { - auto ex2 = ex; - ex = nullptr; - std::rethrow_exception(ex2); - } - conn->to << len; - conn->to(data, len); - }; -}; - void ConnectionHandle::withFramedSink(std::function fun) { (*this)->to.flush(); @@ -1022,7 +989,7 @@ ConnectionHandle::withFramedSink(std::function fun) { }); { - FramedSink sink(*this, ex); + FramedSink sink((*this)->to, ex); fun(sink); sink.flush(); } -- cgit v1.2.3 From 7c682640857106f18d7020c9c75ea39b1ef8dd2c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 18 Sep 2020 10:06:34 +0200 Subject: wopAddToStore: add RepairFlag --- src/libstore/remote-store.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cfbf23ac4..54d70e3e1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -526,7 +526,7 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } -StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references) +StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair) { auto conn(getConnection()); @@ -537,6 +537,7 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA << name << renderContentAddressMethod(caMethod); writeStorePaths(*this, conn->to, references); + conn->to << repair; conn.withFramedSink([&](Sink & sink) { dump.drainInto(sink); @@ -597,9 +598,8 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method, HashType hashType, RepairFlag repair) { - if (repair) throw Error("repairing is not supported when adding to store through the Nix daemon"); StorePathSet references; - return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references); + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair); } @@ -659,9 +659,8 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { - if (repair) throw Error("repairing is not supported when building through the Nix daemon"); StringSource source(s); - return addCAToStore(source, name, TextHashMethod{}, references); + return addCAToStore(source, name, TextHashMethod{}, references, repair); } -- cgit v1.2.3 From fa08db5c4c92ed4674cfacd131d1b45906a9b146 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 18 Sep 2020 11:22:13 +0200 Subject: wopAddToStore: return ValidPathInfo A ValidPathInfo is created anyway. By returning it we can save a roundtrip and we have a nicer interface. --- src/libstore/remote-store.cc | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 54d70e3e1..3936ddb1d 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -421,11 +421,27 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } +ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) { + auto deriver = readString(conn->from); + auto narHash = Hash::parseAny(readString(conn->from), htSHA256); + auto info = make_ref(path, narHash); + if (deriver != "") info->deriver = parseStorePath(deriver); + info->references = readStorePaths(*this, conn->from); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings(conn->from); + info->ca = parseContentAddressOpt(readString(conn->from)); + } + return info; +} + + void RemoteStore::queryPathInfoUncached(const StorePath & path, Callback> callback) noexcept { try { - std::shared_ptr info; + std::shared_ptr info; { auto conn(getConnection()); conn->to << wopQueryPathInfo << printStorePath(path); @@ -441,17 +457,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, bool valid; conn->from >> valid; if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } - auto deriver = readString(conn->from); - auto narHash = Hash::parseAny(readString(conn->from), htSHA256); - info = std::make_shared(path, narHash); - if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = readStorePaths(*this, conn->from); - conn->from >> info->registrationTime >> info->narSize; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { - conn->from >> info->ultimate; - info->sigs = readStrings(conn->from); - info->ca = parseContentAddressOpt(readString(conn->from)); - } + info = readValidPathInfo(conn, path); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -526,7 +532,7 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } -StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair) +ref RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair) { auto conn(getConnection()); @@ -543,8 +549,8 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA dump.drainInto(sink); }); - return parseStorePath(readString(conn->from)); - + auto path = parseStorePath(readString(conn->from)); + return readValidPathInfo(conn, path); } else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); @@ -591,7 +597,8 @@ StorePath RemoteStore::addCAToStore(Source & dump, const string & name, ContentA } }, caMethod); - return parseStorePath(readString(conn->from)); + auto path = parseStorePath(readString(conn->from)); + return queryPathInfo(path); } } @@ -599,7 +606,7 @@ StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method, HashType hashType, RepairFlag repair) { StorePathSet references; - return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair); + return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; } @@ -660,7 +667,7 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) { StringSource source(s); - return addCAToStore(source, name, TextHashMethod{}, references, repair); + return addCAToStore(source, name, TextHashMethod{}, references, repair)->path; } -- cgit v1.2.3 From d51ba430473368b29abfd1a8c4655da74b3a780c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Sep 2020 18:40:11 +0200 Subject: Move Callback into its own header This gets rid of the inclusion of in util.hh, cutting compilation time by ~20s (CPU time). Issue #4045. --- src/libstore/remote-store.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index e92b94975..27535f1d0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -10,6 +10,7 @@ #include "pool.hh" #include "finally.hh" #include "logging.hh" +#include "callback.hh" #include #include -- cgit v1.2.3 From 35a0ac183858ecb03e313e088562c84fe211e20d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Sep 2020 11:40:19 +0200 Subject: Style fixes --- src/libstore/remote-store.cc | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 02a8b72b9..6f1f9769b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -422,7 +422,8 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } -ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) { +ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, const StorePath & path) +{ auto deriver = readString(conn->from); auto narHash = Hash::parseAny(readString(conn->from), htSHA256); auto info = make_ref(path, narHash); @@ -533,7 +534,12 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & } -ref RemoteStore::addCAToStore(Source & dump, const string & name, ContentAddressMethod caMethod, StorePathSet references, RepairFlag repair) +ref RemoteStore::addCAToStore( + Source & dump, + const string & name, + ContentAddressMethod caMethod, + const StorePathSet & references, + RepairFlag repair) { auto conn(getConnection()); @@ -603,6 +609,7 @@ ref RemoteStore::addCAToStore(Source & dump, const string & } } + StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, FileIngestionMethod method, HashType hashType, RepairFlag repair) { @@ -964,8 +971,8 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -void -ConnectionHandle::withFramedSink(std::function fun) { +void ConnectionHandle::withFramedSink(std::function fun) +{ (*this)->to.flush(); std::exception_ptr ex; -- cgit v1.2.3 From 980edd1f3a31eefe297d073f6a7cff099f21eb4a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Sep 2020 15:28:20 +0200 Subject: RemoteStore::addCAToStore(): Don't hold connection while calling queryPathInfo() This leads to a deadlock if we're at the connection limit. --- src/libstore/remote-store.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6f1f9769b..be5eb4736 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -541,7 +541,8 @@ ref RemoteStore::addCAToStore( const StorePathSet & references, RepairFlag repair) { - auto conn(getConnection()); + std::optional conn_(getConnection()); + auto & conn = *conn_; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { @@ -605,6 +606,8 @@ ref RemoteStore::addCAToStore( } }, caMethod); auto path = parseStorePath(readString(conn->from)); + // Release our connection to prevent a deadlock in queryPathInfo(). + conn_.reset(); return queryPathInfo(path); } } -- cgit v1.2.3 From 45a0ed82f089158a79c8c25ef844c55e4a74fc35 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Sep 2020 00:39:06 +0000 Subject: Revert "Use template structs instead of phantoms" This reverts commit 9ab07e99f527d1fa3adfa02839da477a1528d64b. --- src/libstore/remote-store.cc | 70 +++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 33 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 273466137..43853db4e 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -24,61 +24,65 @@ namespace nix { -std::string WorkerProto::read(const Store & store, Source & from) +namespace worker_proto { + +std::string read(const Store & store, Source & from, Phantom _) { return readString(from); } -void WorkerProto::write(const Store & store, Sink & out, const std::string & str) +void write(const Store & store, Sink & out, const std::string & str) { out << str; } -StorePath WorkerProto::read(const Store & store, Source & from) +StorePath read(const Store & store, Source & from, Phantom _) { return store.parseStorePath(readString(from)); } -void WorkerProto::write(const Store & store, Sink & out, const StorePath & storePath) +void write(const Store & store, Sink & out, const StorePath & storePath) { out << store.printStorePath(storePath); } -ContentAddress WorkerProto::read(const Store & store, Source & from) +ContentAddress read(const Store & store, Source & from, Phantom _) { return parseContentAddress(readString(from)); } -void WorkerProto::write(const Store & store, Sink & out, const ContentAddress & ca) +void write(const Store & store, Sink & out, const ContentAddress & ca) { out << renderContentAddress(ca); } -std::optional WorkerProto>::read(const Store & store, Source & from) +std::optional read(const Store & store, Source & from, Phantom> _) { auto s = readString(from); return s == "" ? std::optional {} : store.parseStorePath(s); } -void WorkerProto>::write(const Store & store, Sink & out, const std::optional & storePathOpt) +void write(const Store & store, Sink & out, const std::optional & storePathOpt) { out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); } -std::optional WorkerProto>::read(const Store & store, Source & from) +std::optional read(const Store & store, Source & from, Phantom> _) { return parseContentAddressOpt(readString(from)); } -void WorkerProto>::write(const Store & store, Sink & out, const std::optional & caOpt) +void write(const Store & store, Sink & out, const std::optional & caOpt) { out << (caOpt ? renderContentAddress(*caOpt) : ""); } +} + /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) @@ -325,9 +329,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - WorkerProto::write(*this, conn->to, paths); + nix::worker_proto::write(*this, conn->to, paths); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom {}); } } @@ -337,7 +341,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return nix::worker_proto::read(*this, conn->from, Phantom {}); } @@ -354,9 +358,9 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - WorkerProto::write(*this, conn->to, paths); + nix::worker_proto::write(*this, conn->to, paths); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom {}); } } @@ -378,7 +382,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::read(*this, conn->from); + info.references = worker_proto::read(*this, conn->from, Phantom {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -391,9 +395,9 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - WorkerProto::write(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); } else - WorkerProto::write(*this, conn->to, pathsMap); + worker_proto::write(*this, conn->to, pathsMap); conn.processStderr(); size_t count = readNum(conn->from); for (size_t n = 0; n < count; n++) { @@ -401,7 +405,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = WorkerProto::read(*this, conn->from); + info.references = worker_proto::read(*this, conn->from, Phantom {}); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -416,7 +420,7 @@ ref RemoteStore::readValidPathInfo(ConnectionHandle & conn, auto narHash = Hash::parseAny(readString(conn->from), htSHA256); auto info = make_ref(path, narHash); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::read(*this, conn->from); + info->references = worker_proto::read(*this, conn->from, Phantom {}); conn->from >> info->registrationTime >> info->narSize; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { conn->from >> info->ultimate; @@ -460,7 +464,7 @@ void RemoteStore::queryReferrers(const StorePath & path, auto conn(getConnection()); conn->to << wopQueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : WorkerProto::read(*this, conn->from)) + for (auto & i : worker_proto::read(*this, conn->from, Phantom {})) referrers.insert(i); } @@ -470,7 +474,7 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) auto conn(getConnection()); conn->to << wopQueryValidDerivers << printStorePath(path); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom {}); } @@ -482,7 +486,7 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } conn->to << wopQueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return WorkerProto::read(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom {}); } @@ -492,7 +496,7 @@ std::map> RemoteStore::queryPartialDerivat auto conn(getConnection()); conn->to << wopQueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return WorkerProto>>::read(*this, conn->from); + return worker_proto::read(*this, conn->from, Phantom>> {}); } else { // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an @@ -537,7 +541,7 @@ ref RemoteStore::addCAToStore( << wopAddToStore << name << renderContentAddressMethod(caMethod); - WorkerProto::write(*this, conn->to, references); + worker_proto::write(*this, conn->to, references); conn->to << repair; conn.withFramedSink([&](Sink & sink) { @@ -554,7 +558,7 @@ ref RemoteStore::addCAToStore( [&](TextHashMethod thm) -> void { std::string s = dump.drain(); conn->to << wopAddTextToStore << name << s; - WorkerProto::write(*this, conn->to, references); + worker_proto::write(*this, conn->to, references); conn.processStderr(); }, [&](FixedOutputHashMethod fohm) -> void { @@ -623,7 +627,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, sink, info.references); + worker_proto::write(*this, sink, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -633,7 +637,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = WorkerProto::read(*this, conn->from); + auto importedPaths = worker_proto::read(*this, conn->from, Phantom {}); assert(importedPaths.size() <= 1); } @@ -642,7 +646,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, conn->to, info.references); + worker_proto::write(*this, conn->to, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -764,7 +768,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) conn->to << wopCollectGarbage << options.action; - WorkerProto::write(*this, conn->to, options.pathsToDelete); + worker_proto::write(*this, conn->to, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -826,9 +830,9 @@ void RemoteStore::queryMissing(const std::vector & targets ss.push_back(p.to_string(*this)); conn->to << ss; conn.processStderr(); - willBuild = WorkerProto::read(*this, conn->from); - willSubstitute = WorkerProto::read(*this, conn->from); - unknown = WorkerProto::read(*this, conn->from); + willBuild = worker_proto::read(*this, conn->from, Phantom {}); + willSubstitute = worker_proto::read(*this, conn->from, Phantom {}); + unknown = worker_proto::read(*this, conn->from, Phantom {}); conn->from >> downloadSize >> narSize; return; } -- cgit v1.2.3 From b7597016529cebdc3c9432a101c1f8d9227713cc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Sep 2020 00:41:18 +0000 Subject: nix::worker_proto -> worker_proto --- src/libstore/remote-store.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 43853db4e..049d4e954 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -329,7 +329,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute return res; } else { conn->to << wopQueryValidPaths; - nix::worker_proto::write(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); } @@ -341,7 +341,7 @@ StorePathSet RemoteStore::queryAllValidPaths() auto conn(getConnection()); conn->to << wopQueryAllValidPaths; conn.processStderr(); - return nix::worker_proto::read(*this, conn->from, Phantom {}); + return worker_proto::read(*this, conn->from, Phantom {}); } @@ -358,7 +358,7 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) return res; } else { conn->to << wopQuerySubstitutablePaths; - nix::worker_proto::write(*this, conn->to, paths); + worker_proto::write(*this, conn->to, paths); conn.processStderr(); return worker_proto::read(*this, conn->from, Phantom {}); } -- cgit v1.2.3 From 85c8be6286e8f5464813a8a29b0b78d892498745 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Oct 2020 13:36:55 +0200 Subject: Remove static variable name clashes This was useful for an experiment with building Nix as a single compilation unit. It's not very useful otherwise but also doesn't hurt... --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 049d4e954..40ef72293 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1008,6 +1008,6 @@ void ConnectionHandle::withFramedSink(std::function fun) } -static RegisterStoreImplementation regStore; +static RegisterStoreImplementation regUDSRemoteStore; } -- cgit v1.2.3 From c43e882f54b5f09278dfb0d315239531c0676b4e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Oct 2020 17:13:54 +0200 Subject: Serialize exceptions from the daemon to the client --- src/libstore/remote-store.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'src/libstore/remote-store.cc') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 40ef72293..23b1942ce 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -925,9 +925,13 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * } else if (msg == STDERR_ERROR) { - string error = readString(from); - unsigned int status = readInt(from); - return std::make_exception_ptr(Error(status, error)); + if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { + return std::make_exception_ptr(readError(from)); + } else { + string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); + } } else if (msg == STDERR_NEXT) -- cgit v1.2.3