diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/libexpr/get-drvs.hh | 7 | ||||
-rw-r--r-- | src/libexpr/primops.cc | 6 | ||||
-rw-r--r-- | src/libstore/build.cc | 46 | ||||
-rw-r--r-- | src/libstore/globals.cc | 1 | ||||
-rw-r--r-- | src/libstore/local-store.cc | 83 | ||||
-rw-r--r-- | src/libstore/local-store.hh | 16 | ||||
-rw-r--r-- | src/libstore/misc.cc | 106 | ||||
-rw-r--r-- | src/libstore/optimise-store.cc | 2 | ||||
-rw-r--r-- | src/libstore/remote-store.cc | 98 | ||||
-rw-r--r-- | src/libstore/remote-store.hh | 10 | ||||
-rw-r--r-- | src/libstore/store-api.hh | 32 | ||||
-rw-r--r-- | src/libstore/worker-protocol.hh | 7 | ||||
-rw-r--r-- | src/nix-env/nix-env.cc | 28 | ||||
-rw-r--r-- | src/nix-store/nix-store.cc | 2 | ||||
-rw-r--r-- | src/nix-worker/nix-worker.cc | 62 |
15 files changed, 368 insertions, 138 deletions
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 2d260c57b..1e5d0a817 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -32,6 +32,8 @@ private: bool metaInfoRead; MetaInfo meta; + + bool failed; // set if we get an AssertionError public: string name; @@ -41,7 +43,7 @@ public: /* !!! make this private */ Bindings * attrs; - DrvInfo() : metaInfoRead(false), attrs(0) { }; + DrvInfo() : metaInfoRead(false), failed(false), attrs(0) { }; string queryDrvPath(EvalState & state) const; string queryOutPath(EvalState & state) const; @@ -59,6 +61,9 @@ public: } void setMetaInfo(const MetaInfo & meta); + + void setFailed() { failed = true; }; + bool hasFailed() { return failed; }; }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5d5f0bfb3..f20c2f287 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -51,6 +51,12 @@ static void prim_import(EvalState & state, Value * * args, Value & v) % path % ctx); if (isDerivation(ctx)) try { + /* For performance, prefetch all substitute info. */ + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + queryMissing(*store, singleton<PathSet>(ctx), + willBuild, willSubstitute, unknown, downloadSize, narSize); + /* !!! If using a substitute, we only need to fetch the selected output of this derivation. */ store->buildPaths(singleton<PathSet>(ctx)); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 26268f6dd..4bef6f02d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -94,7 +94,7 @@ typedef map<Path, WeakGoalPtr> WeakGoalMap; class Goal : public boost::enable_shared_from_this<Goal> { public: - typedef enum {ecBusy, ecSuccess, ecFailed} ExitCode; + typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters} ExitCode; protected: @@ -111,6 +111,10 @@ protected: /* Number of goals we are/were waiting for that have failed. */ unsigned int nrFailed; + /* Number of substitution goals we are/were waiting for that + failed because there are no substituters. */ + unsigned int nrNoSubstituters; + /* Name of this goal for debugging purposes. */ string name; @@ -119,7 +123,7 @@ protected: Goal(Worker & worker) : worker(worker) { - nrFailed = 0; + nrFailed = nrNoSubstituters = 0; exitCode = ecBusy; } @@ -306,7 +310,9 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result) trace(format("waitee `%1%' done; %2% left") % waitee->name % waitees.size()); - if (result == ecFailed) ++nrFailed; + if (result == ecFailed || result == ecNoSubstituters) ++nrFailed; + + if (result == ecNoSubstituters) ++nrNoSubstituters; if (waitees.empty() || (result == ecFailed && !keepGoing)) { @@ -330,7 +336,7 @@ void Goal::amDone(ExitCode result) { trace("done"); assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed); + assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters); exitCode = result; foreach (WeakGoals::iterator, i, waiters) { GoalPtr goal = i->lock(); @@ -736,6 +742,8 @@ HookInstance::~HookInstance() typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; +class SubstitutionGoal; + class DerivationGoal : public Goal { private: @@ -986,10 +994,8 @@ void DerivationGoal::haveDerivation() /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - foreach (PathSet::iterator, i, invalidOutputs) - /* Don't bother creating a substitution goal if there are no - substitutes. */ - if (queryBoolSetting("build-use-substitutes", true) && worker.store.hasSubstitutes(*i)) + if (queryBoolSetting("build-use-substitutes", true)) + foreach (PathSet::iterator, i, invalidOutputs) addWaitee(worker.makeSubstitutionGoal(*i)); if (waitees.empty()) /* to prevent hang (no wake-up event) */ @@ -1003,10 +1009,10 @@ void DerivationGoal::outputsSubstituted() { trace("all outputs substituted (maybe)"); - if (nrFailed > 0 && !tryFallback) + if (nrFailed > 0 && nrFailed > nrNoSubstituters && !tryFallback) throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath); - nrFailed = 0; + nrFailed = nrNoSubstituters = 0; if (checkPathValidity(false).size() == 0) { amDone(ecSuccess); @@ -2258,6 +2264,9 @@ private: /* The current substituter. */ Path sub; + /* Whether any substituter can realise this path */ + bool hasSubstitute; + /* Path info returned by the substituter's query info operation. */ SubstitutablePathInfo info; @@ -2299,6 +2308,7 @@ public: SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker) : Goal(worker) + , hasSubstitute(false) { this->storePath = storePath; state = &SubstitutionGoal::init; @@ -2362,17 +2372,23 @@ void SubstitutionGoal::tryNext() /* None left. Terminate this goal and let someone else deal with it. */ debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath); - amDone(ecFailed); + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + amDone(hasSubstitute ? ecFailed : ecNoSubstituters); return; } sub = subs.front(); subs.pop_front(); - if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) { - tryNext(); - return; - } + SubstitutablePathInfos infos; + PathSet dummy(singleton<PathSet>(storePath)); + worker.store.querySubstitutablePathInfos(sub, dummy, infos); + SubstitutablePathInfos::iterator k = infos.find(storePath); + if (k == infos.end()) { tryNext(); return; } + info = k->second; + hasSubstitute = true; /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 5c22f1406..9636bf49d 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -157,6 +157,7 @@ void setDefaultsFromEnvironment() if (subs == "default") { substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl"); substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl"); + substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl"); } else substituters = tokenizeString(subs, ":"); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 30398a244..aa21774df 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -748,7 +748,16 @@ bool LocalStore::isValidPath(const Path & path) } -PathSet LocalStore::queryValidPaths() +PathSet LocalStore::queryValidPaths(const PathSet & paths) +{ + PathSet res; + foreach (PathSet::const_iterator, i, paths) + if (isValidPath(*i)) res.insert(*i); + return res; +} + + +PathSet LocalStore::queryAllValidPaths() { SQLiteStmt stmt; stmt.create(db, "select path from ValidPaths"); @@ -947,50 +956,66 @@ template<class T> T getIntLine(int fd) } -bool LocalStore::hasSubstitutes(const Path & path) +PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) { + PathSet res; foreach (Paths::iterator, i, substituters) { + if (res.size() == paths.size()) break; RunningSubstituter & run(runningSubstituters[*i]); startSubstituter(*i, run); - writeLine(run.to, "have\n" + path); - if (getIntLine<int>(run.from)) return true; + string s = "have "; + foreach (PathSet::const_iterator, i, paths) + if (res.find(*i) == res.end()) { s += *i; s += " "; } + writeLine(run.to, s); + while (true) { + Path path = readLine(run.from); + if (path == "") break; + res.insert(path); + } } - - return false; + return res; } -bool LocalStore::querySubstitutablePathInfo(const Path & substituter, - const Path & path, SubstitutablePathInfo & info) +void LocalStore::querySubstitutablePathInfos(const Path & substituter, + PathSet & paths, SubstitutablePathInfos & infos) { RunningSubstituter & run(runningSubstituters[substituter]); startSubstituter(substituter, run); - writeLine(run.to, "info\n" + path); + string s = "info "; + foreach (PathSet::const_iterator, i, paths) + if (infos.find(*i) == infos.end()) { s += *i; s += " "; } + writeLine(run.to, s); - if (!getIntLine<int>(run.from)) return false; - - info.deriver = readLine(run.from); - if (info.deriver != "") assertStorePath(info.deriver); - int nrRefs = getIntLine<int>(run.from); - while (nrRefs--) { - Path p = readLine(run.from); - assertStorePath(p); - info.references.insert(p); + while (true) { + Path path = readLine(run.from); + if (path == "") break; + assert(paths.find(path) != paths.end()); + paths.erase(path); + SubstitutablePathInfo & info(infos[path]); + info.deriver = readLine(run.from); + if (info.deriver != "") assertStorePath(info.deriver); + int nrRefs = getIntLine<int>(run.from); + while (nrRefs--) { + Path p = readLine(run.from); + assertStorePath(p); + info.references.insert(p); + } + info.downloadSize = getIntLine<long long>(run.from); + info.narSize = getIntLine<long long>(run.from); } - info.downloadSize = getIntLine<long long>(run.from); - info.narSize = getIntLine<long long>(run.from); - - return true; } -bool LocalStore::querySubstitutablePathInfo(const Path & path, - SubstitutablePathInfo & info) +void LocalStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) { - foreach (Paths::iterator, i, substituters) - if (querySubstitutablePathInfo(*i, path, info)) return true; - return false; + PathSet todo = paths; + foreach (Paths::iterator, i, substituters) { + if (todo.empty()) break; + querySubstitutablePathInfos(*i, todo, infos); + } } @@ -1134,7 +1159,7 @@ Path LocalStore::addToStore(const Path & _srcPath, method for very large paths, but `copyPath' is mainly used for small files. */ StringSink sink; - if (recursive) + if (recursive) dumpPath(srcPath, sink, filter); else sink.s = readFile(srcPath); @@ -1465,7 +1490,7 @@ void LocalStore::verifyStore(bool checkContents) /* Check whether all valid paths actually exist. */ printMsg(lvlInfo, "checking path existence..."); - PathSet validPaths2 = queryValidPaths(), validPaths, done; + PathSet validPaths2 = queryAllValidPaths(), validPaths, done; foreach (PathSet::iterator, i, validPaths2) verifyPath(*i, store, done, validPaths); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 65ee029c2..07d8198ec 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -99,7 +99,9 @@ public: bool isValidPath(const Path & path); - PathSet queryValidPaths(); + PathSet queryValidPaths(const PathSet & paths); + + PathSet queryAllValidPaths(); ValidPathInfo queryPathInfo(const Path & path); @@ -123,15 +125,13 @@ public: Path queryPathFromHashPart(const string & hashPart); - PathSet querySubstitutablePaths(); - - bool hasSubstitutes(const Path & path); + PathSet querySubstitutablePaths(const PathSet & paths); - bool querySubstitutablePathInfo(const Path & path, - SubstitutablePathInfo & info); + void querySubstitutablePathInfos(const Path & substituter, + PathSet & paths, SubstitutablePathInfos & infos); - bool querySubstitutablePathInfo(const Path & substituter, - const Path & path, SubstitutablePathInfo & info); + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos); Path addToStore(const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 093499936..aa5f6ff72 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -55,45 +55,97 @@ void queryMissing(StoreAPI & store, const PathSet & targets, PathSet todo(targets.begin(), targets.end()), done; + bool useSubstitutes = queryBoolSetting("build-use-substitutes", true); + + /* Getting substitute info has high latency when using the binary + cache substituter. Thus it's essential to do substitute + queries in parallel as much as possible. To accomplish this + we do the following: + + - For all paths still to be processed (‘todo’), we add all + paths for which we need info to the set ‘query’. For an + unbuilt derivation this is the output paths; otherwise, it's + the path itself. + + - We get info about all paths in ‘query’ in parallel. + + - We process the results and add new items to ‘todo’ if + necessary. E.g. if a path is substitutable, then we need to + get info on its references. + + - Repeat until ‘todo’ is empty. + */ + while (!todo.empty()) { - Path p = *(todo.begin()); - todo.erase(p); - if (done.find(p) != done.end()) continue; - done.insert(p); - - if (isDerivation(p)) { - if (!store.isValidPath(p)) { - unknown.insert(p); - continue; + + PathSet query, todoDrv, todoNonDrv; + + foreach (PathSet::iterator, i, todo) { + if (done.find(*i) != done.end()) continue; + done.insert(*i); + + if (isDerivation(*i)) { + if (!store.isValidPath(*i)) { + // FIXME: we could try to substitute p. + unknown.insert(*i); + continue; + } + Derivation drv = derivationFromPath(store, *i); + + PathSet invalid; + foreach (DerivationOutputs::iterator, j, drv.outputs) + if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path); + if (invalid.empty()) continue; + + todoDrv.insert(*i); + if (useSubstitutes) query.insert(invalid.begin(), invalid.end()); + } + + else { + if (store.isValidPath(*i)) continue; + query.insert(*i); + todoNonDrv.insert(*i); } - Derivation drv = derivationFromPath(store, p); + } + + todo.clear(); + + SubstitutablePathInfos infos; + store.querySubstitutablePathInfos(query, infos); + + foreach (PathSet::iterator, i, todoDrv) { + // FIXME: cache this + Derivation drv = derivationFromPath(store, *i); bool mustBuild = false; - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (!store.isValidPath(i->second.path) && - !(queryBoolSetting("build-use-substitutes", true) && store.hasSubstitutes(i->second.path))) - mustBuild = true; + if (useSubstitutes) { + foreach (DerivationOutputs::iterator, j, drv.outputs) + if (!store.isValidPath(j->second.path) && + infos.find(j->second.path) == infos.end()) + mustBuild = true; + } else + mustBuild = true; if (mustBuild) { - willBuild.insert(p); + willBuild.insert(*i); todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); foreach (DerivationInputs::iterator, i, drv.inputDrvs) todo.insert(i->first); - } else + } else foreach (DerivationOutputs::iterator, i, drv.outputs) - todo.insert(i->second.path); + todoNonDrv.insert(i->second.path); } - - else { - if (store.isValidPath(p)) continue; - SubstitutablePathInfo info; - if (store.querySubstitutablePathInfo(p, info)) { - willSubstitute.insert(p); - downloadSize += info.downloadSize; - narSize += info.narSize; - todo.insert(info.references.begin(), info.references.end()); + + foreach (PathSet::iterator, i, todoNonDrv) { + done.insert(*i); + SubstitutablePathInfos::iterator info = infos.find(*i); + if (info != infos.end()) { + willSubstitute.insert(*i); + downloadSize += info->second.downloadSize; + narSize += info->second.narSize; + todo.insert(info->second.references.begin(), info->second.references.end()); } else - unknown.insert(p); + unknown.insert(*i); } } } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 2ca98f46d..a486e66ef 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -178,7 +178,7 @@ void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats) { HashToPath hashToPath; - PathSet paths = queryValidPaths(); + PathSet paths = queryAllValidPaths(); foreach (PathSet::iterator, i, paths) { addTempRoot(*i); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cbb70b2fd..35530acab 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -217,42 +217,96 @@ bool RemoteStore::isValidPath(const Path & path) } -PathSet RemoteStore::queryValidPaths() +PathSet RemoteStore::queryValidPaths(const PathSet & paths) { openConnection(); - writeInt(wopQueryValidPaths, to); + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + PathSet res; + foreach (PathSet::const_iterator, i, paths) + if (isValidPath(*i)) res.insert(*i); + return res; + } else { + writeInt(wopQueryValidPaths, to); + writeStrings(paths, to); + processStderr(); + return readStorePaths<PathSet>(from); + } +} + + +PathSet RemoteStore::queryAllValidPaths() +{ + openConnection(); + writeInt(wopQueryAllValidPaths, to); processStderr(); return readStorePaths<PathSet>(from); } -bool RemoteStore::hasSubstitutes(const Path & path) +PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) { openConnection(); - writeInt(wopHasSubstitutes, to); - writeString(path, to); - processStderr(); - unsigned int reply = readInt(from); - return reply != 0; + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + PathSet res; + foreach (PathSet::const_iterator, i, paths) { + writeInt(wopHasSubstitutes, to); + writeString(*i, to); + processStderr(); + if (readInt(from)) res.insert(*i); + } + return res; + } else { + writeInt(wopQuerySubstitutablePaths, to); + writeStrings(paths, to); + processStderr(); + return readStorePaths<PathSet>(from); + } } -bool RemoteStore::querySubstitutablePathInfo(const Path & path, - SubstitutablePathInfo & info) +void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) { + if (paths.empty()) return; + openConnection(); - if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return false; - writeInt(wopQuerySubstitutablePathInfo, to); - writeString(path, to); - processStderr(); - unsigned int reply = readInt(from); - if (reply == 0) return false; - info.deriver = readString(from); - if (info.deriver != "") assertStorePath(info.deriver); - info.references = readStorePaths<PathSet>(from); - info.downloadSize = readLongLong(from); - info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; - return true; + + if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return; + + if (GET_PROTOCOL_MINOR(daemonVersion) < 12) { + + foreach (PathSet::const_iterator, i, paths) { + SubstitutablePathInfo info; + writeInt(wopQuerySubstitutablePathInfo, to); + writeString(*i, to); + processStderr(); + unsigned int reply = readInt(from); + if (reply == 0) continue; + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths<PathSet>(from); + info.downloadSize = readLongLong(from); + info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; + infos[*i] = info; + } + + } else { + + writeInt(wopQuerySubstitutablePathInfos, to); + writeStrings(paths, to); + processStderr(); + unsigned int count = readInt(from); + for (unsigned int n = 0; n < count; n++) { + Path path = readStorePath(from); + SubstitutablePathInfo & info(infos[path]); + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths<PathSet>(from); + info.downloadSize = readLongLong(from); + info.narSize = readLongLong(from); + } + + } } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f0e5dbf76..68db0640a 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -27,7 +27,9 @@ public: bool isValidPath(const Path & path); - PathSet queryValidPaths(); + PathSet queryValidPaths(const PathSet & paths); + + PathSet queryAllValidPaths(); ValidPathInfo queryPathInfo(const Path & path); @@ -45,10 +47,10 @@ public: Path queryPathFromHashPart(const string & hashPart); - bool hasSubstitutes(const Path & path); + PathSet querySubstitutablePaths(const PathSet & paths); - bool querySubstitutablePathInfo(const Path & path, - SubstitutablePathInfo & info); + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos); Path addToStore(const Path & srcPath, bool recursive = true, HashType hashAlgo = htSHA256, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 0ab15c380..5d8c09f5a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -86,6 +86,8 @@ struct SubstitutablePathInfo unsigned long long narSize; /* 0 = unknown */ }; +typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos; + struct ValidPathInfo { @@ -108,20 +110,23 @@ public: virtual ~StoreAPI() { } - /* Checks whether a path is valid. */ + /* Check whether a path is valid. */ virtual bool isValidPath(const Path & path) = 0; - /* Query the set of valid paths. */ - virtual PathSet queryValidPaths() = 0; + /* Query which of the given paths is valid. */ + virtual PathSet queryValidPaths(const PathSet & paths) = 0; + + /* Query the set of all valid paths. */ + virtual PathSet queryAllValidPaths() = 0; /* Query information about a valid path. */ virtual ValidPathInfo queryPathInfo(const Path & path) = 0; - /* Queries the hash of a valid path. */ + /* Query the hash of a valid path. */ virtual Hash queryPathHash(const Path & path) = 0; - /* Queries the set of outgoing FS references for a store path. - The result is not cleared. */ + /* Query the set of outgoing FS references for a store path. The + result is not cleared. */ virtual void queryReferences(const Path & path, PathSet & references) = 0; @@ -144,13 +149,14 @@ public: path, or "" if the path doesn't exist. */ virtual Path queryPathFromHashPart(const string & hashPart) = 0; - /* Query whether a path has substitutes. */ - virtual bool hasSubstitutes(const Path & path) = 0; - - /* Query the references, deriver and download size of a - substitutable path. */ - virtual bool querySubstitutablePathInfo(const Path & path, - SubstitutablePathInfo & info) = 0; + /* Query which of the given paths have substitutes. */ + virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0; + + /* Query substitute info (i.e. references, derivers and download + sizes) of a set of paths. If a path does not have substitute + info, it's omitted from the resulting ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) = 0; /* Copy the contents of a path to the store and register the validity the resulting path. The resulting path is returned. diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index b08410fa1..d4ac0ea16 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x10b +#define PROTOCOL_VERSION 0x10c #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -34,13 +34,16 @@ typedef enum { wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, wopQueryDerivationOutputs = 22, - wopQueryValidPaths = 23, + wopQueryAllValidPaths = 23, wopQueryFailedPaths = 24, wopClearFailedPaths = 25, wopQueryPathInfo = 26, wopImportPaths = 27, wopQueryDerivationOutputNames = 28, wopQueryPathFromHashPart = 29, + wopQuerySubstitutablePathInfos = 30, + wopQueryValidPaths = 31, + wopQuerySubstitutablePaths = 32, } WorkerOp; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 7aa6276e3..f06f23dad 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -211,9 +211,12 @@ static int comparePriorities(EvalState & state, static bool isPrebuilt(EvalState & state, const DrvInfo & elem) { + assert(false); +#if 0 return store->isValidPath(elem.queryOutPath(state)) || store->hasSubstitutes(elem.queryOutPath(state)); +#endif } @@ -929,6 +932,22 @@ static void opQuery(Globals & globals, installed.insert(i->queryOutPath(globals.state)); } + + /* Query which paths have substitutes. */ + PathSet validPaths, substitutablePaths; + if (printStatus) { + PathSet paths; + foreach (vector<DrvInfo>::iterator, i, elems2) + try { + paths.insert(i->queryOutPath(globals.state)); + } catch (AssertionError & e) { + printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name); + i->setFailed(); + } + validPaths = store->queryValidPaths(paths); + substitutablePaths = store->querySubstitutablePaths(paths); + } + /* Print the desired columns, or XML output. */ Table table; @@ -938,6 +957,8 @@ static void opQuery(Globals & globals, foreach (vector<DrvInfo>::iterator, i, elems2) { try { + if (i->hasFailed()) continue; + startNest(nest, lvlDebug, format("outputting query result `%1%'") % i->attrPath); if (globals.prebuiltOnly && !isPrebuilt(globals.state, *i)) continue; @@ -949,9 +970,10 @@ static void opQuery(Globals & globals, XMLAttrs attrs; if (printStatus) { - bool hasSubs = store->hasSubstitutes(i->queryOutPath(globals.state)); - bool isInstalled = installed.find(i->queryOutPath(globals.state)) != installed.end(); - bool isValid = store->isValidPath(i->queryOutPath(globals.state)); + Path outPath = i->queryOutPath(globals.state); + bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); + bool isInstalled = installed.find(outPath) != installed.end(); + bool isValid = validPaths.find(outPath) != validPaths.end(); if (xmlOutput) { attrs["installed"] = isInstalled ? "1" : "0"; attrs["valid"] = isValid ? "1" : "0"; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 23863525f..47c76693f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -465,7 +465,7 @@ static void opDumpDB(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); if (!opArgs.empty()) throw UsageError("no arguments expected"); - PathSet validPaths = store->queryValidPaths(); + PathSet validPaths = store->queryAllValidPaths(); foreach (PathSet::iterator, i, validPaths) cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true); } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 74a619c71..f2ca0a892 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -297,15 +297,33 @@ static void performOp(unsigned int clientVersion, break; } + case wopQueryValidPaths: { + PathSet paths = readStorePaths<PathSet>(from); + startWork(); + PathSet res = store->queryValidPaths(paths); + stopWork(); + writeStrings(res, to); + break; + } + case wopHasSubstitutes: { Path path = readStorePath(from); startWork(); - bool result = store->hasSubstitutes(path); + PathSet res = store->querySubstitutablePaths(singleton<PathSet>(path)); stopWork(); - writeInt(result, to); + writeInt(res.find(path) != res.end(), to); break; } + case wopQuerySubstitutablePaths: { + PathSet paths = readStorePaths<PathSet>(from); + startWork(); + PathSet res = store->querySubstitutablePaths(paths); + stopWork(); + writeStrings(res, to); + break; + } + case wopQueryPathHash: { Path path = readStorePath(from); startWork(); @@ -538,23 +556,43 @@ static void performOp(unsigned int clientVersion, case wopQuerySubstitutablePathInfo: { Path path = absPath(readString(from)); startWork(); - SubstitutablePathInfo info; - bool res = store->querySubstitutablePathInfo(path, info); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos(singleton<PathSet>(path), infos); stopWork(); - writeInt(res ? 1 : 0, to); - if (res) { - writeString(info.deriver, to); - writeStrings(info.references, to); - writeLongLong(info.downloadSize, to); + SubstitutablePathInfos::iterator i = infos.find(path); + if (i == infos.end()) + writeInt(0, to); + else { + writeInt(1, to); + writeString(i->second.deriver, to); + writeStrings(i->second.references, to); + writeLongLong(i->second.downloadSize, to); if (GET_PROTOCOL_MINOR(clientVersion) >= 7) - writeLongLong(info.narSize, to); + writeLongLong(i->second.narSize, to); } break; } - case wopQueryValidPaths: { + case wopQuerySubstitutablePathInfos: { + PathSet paths = readStorePaths<PathSet>(from); + startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos(paths, infos); + stopWork(); + writeInt(infos.size(), to); + foreach (SubstitutablePathInfos::iterator, i, infos) { + writeString(i->first, to); + writeString(i->second.deriver, to); + writeStrings(i->second.references, to); + writeLongLong(i->second.downloadSize, to); + writeLongLong(i->second.narSize, to); + } + break; + } + + case wopQueryAllValidPaths: { startWork(); - PathSet paths = store->queryValidPaths(); + PathSet paths = store->queryAllValidPaths(); stopWork(); writeStrings(paths, to); break; |