aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libexpr/get-drvs.hh7
-rw-r--r--src/libexpr/primops.cc6
-rw-r--r--src/libstore/build.cc46
-rw-r--r--src/libstore/globals.cc1
-rw-r--r--src/libstore/local-store.cc83
-rw-r--r--src/libstore/local-store.hh16
-rw-r--r--src/libstore/misc.cc106
-rw-r--r--src/libstore/optimise-store.cc2
-rw-r--r--src/libstore/remote-store.cc98
-rw-r--r--src/libstore/remote-store.hh10
-rw-r--r--src/libstore/store-api.hh32
-rw-r--r--src/libstore/worker-protocol.hh7
-rw-r--r--src/nix-env/nix-env.cc28
-rw-r--r--src/nix-store/nix-store.cc2
-rw-r--r--src/nix-worker/nix-worker.cc62
15 files changed, 368 insertions, 138 deletions
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index 2d260c57b..1e5d0a817 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -32,6 +32,8 @@ private:
bool metaInfoRead;
MetaInfo meta;
+
+ bool failed; // set if we get an AssertionError
public:
string name;
@@ -41,7 +43,7 @@ public:
/* !!! make this private */
Bindings * attrs;
- DrvInfo() : metaInfoRead(false), attrs(0) { };
+ DrvInfo() : metaInfoRead(false), failed(false), attrs(0) { };
string queryDrvPath(EvalState & state) const;
string queryOutPath(EvalState & state) const;
@@ -59,6 +61,9 @@ public:
}
void setMetaInfo(const MetaInfo & meta);
+
+ void setFailed() { failed = true; };
+ bool hasFailed() { return failed; };
};
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 5d5f0bfb3..f20c2f287 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -51,6 +51,12 @@ static void prim_import(EvalState & state, Value * * args, Value & v)
% path % ctx);
if (isDerivation(ctx))
try {
+ /* For performance, prefetch all substitute info. */
+ PathSet willBuild, willSubstitute, unknown;
+ unsigned long long downloadSize, narSize;
+ queryMissing(*store, singleton<PathSet>(ctx),
+ willBuild, willSubstitute, unknown, downloadSize, narSize);
+
/* !!! If using a substitute, we only need to fetch
the selected output of this derivation. */
store->buildPaths(singleton<PathSet>(ctx));
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 26268f6dd..4bef6f02d 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -94,7 +94,7 @@ typedef map<Path, WeakGoalPtr> WeakGoalMap;
class Goal : public boost::enable_shared_from_this<Goal>
{
public:
- typedef enum {ecBusy, ecSuccess, ecFailed} ExitCode;
+ typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters} ExitCode;
protected:
@@ -111,6 +111,10 @@ protected:
/* Number of goals we are/were waiting for that have failed. */
unsigned int nrFailed;
+ /* Number of substitution goals we are/were waiting for that
+ failed because there are no substituters. */
+ unsigned int nrNoSubstituters;
+
/* Name of this goal for debugging purposes. */
string name;
@@ -119,7 +123,7 @@ protected:
Goal(Worker & worker) : worker(worker)
{
- nrFailed = 0;
+ nrFailed = nrNoSubstituters = 0;
exitCode = ecBusy;
}
@@ -306,7 +310,9 @@ void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
trace(format("waitee `%1%' done; %2% left") %
waitee->name % waitees.size());
- if (result == ecFailed) ++nrFailed;
+ if (result == ecFailed || result == ecNoSubstituters) ++nrFailed;
+
+ if (result == ecNoSubstituters) ++nrNoSubstituters;
if (waitees.empty() || (result == ecFailed && !keepGoing)) {
@@ -330,7 +336,7 @@ void Goal::amDone(ExitCode result)
{
trace("done");
assert(exitCode == ecBusy);
- assert(result == ecSuccess || result == ecFailed);
+ assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters);
exitCode = result;
foreach (WeakGoals::iterator, i, waiters) {
GoalPtr goal = i->lock();
@@ -736,6 +742,8 @@ HookInstance::~HookInstance()
typedef enum {rpAccept, rpDecline, rpPostpone} HookReply;
+class SubstitutionGoal;
+
class DerivationGoal : public Goal
{
private:
@@ -986,10 +994,8 @@ void DerivationGoal::haveDerivation()
/* We are first going to try to create the invalid output paths
through substitutes. If that doesn't work, we'll build
them. */
- foreach (PathSet::iterator, i, invalidOutputs)
- /* Don't bother creating a substitution goal if there are no
- substitutes. */
- if (queryBoolSetting("build-use-substitutes", true) && worker.store.hasSubstitutes(*i))
+ if (queryBoolSetting("build-use-substitutes", true))
+ foreach (PathSet::iterator, i, invalidOutputs)
addWaitee(worker.makeSubstitutionGoal(*i));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -1003,10 +1009,10 @@ void DerivationGoal::outputsSubstituted()
{
trace("all outputs substituted (maybe)");
- if (nrFailed > 0 && !tryFallback)
+ if (nrFailed > 0 && nrFailed > nrNoSubstituters && !tryFallback)
throw Error(format("some substitutes for the outputs of derivation `%1%' failed; try `--fallback'") % drvPath);
- nrFailed = 0;
+ nrFailed = nrNoSubstituters = 0;
if (checkPathValidity(false).size() == 0) {
amDone(ecSuccess);
@@ -2258,6 +2264,9 @@ private:
/* The current substituter. */
Path sub;
+ /* Whether any substituter can realise this path */
+ bool hasSubstitute;
+
/* Path info returned by the substituter's query info operation. */
SubstitutablePathInfo info;
@@ -2299,6 +2308,7 @@ public:
SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker)
: Goal(worker)
+ , hasSubstitute(false)
{
this->storePath = storePath;
state = &SubstitutionGoal::init;
@@ -2362,17 +2372,23 @@ void SubstitutionGoal::tryNext()
/* None left. Terminate this goal and let someone else deal
with it. */
debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath);
- amDone(ecFailed);
+ /* Hack: don't indicate failure if there were no substituters.
+ In that case the calling derivation should just do a
+ build. */
+ amDone(hasSubstitute ? ecFailed : ecNoSubstituters);
return;
}
sub = subs.front();
subs.pop_front();
- if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) {
- tryNext();
- return;
- }
+ SubstitutablePathInfos infos;
+ PathSet dummy(singleton<PathSet>(storePath));
+ worker.store.querySubstitutablePathInfos(sub, dummy, infos);
+ SubstitutablePathInfos::iterator k = infos.find(storePath);
+ if (k == infos.end()) { tryNext(); return; }
+ info = k->second;
+ hasSubstitute = true;
/* To maintain the closure invariant, we first have to realise the
paths referenced by this one. */
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 5c22f1406..9636bf49d 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -157,6 +157,7 @@ void setDefaultsFromEnvironment()
if (subs == "default") {
substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+ substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
} else
substituters = tokenizeString(subs, ":");
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 30398a244..aa21774df 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -748,7 +748,16 @@ bool LocalStore::isValidPath(const Path & path)
}
-PathSet LocalStore::queryValidPaths()
+PathSet LocalStore::queryValidPaths(const PathSet & paths)
+{
+ PathSet res;
+ foreach (PathSet::const_iterator, i, paths)
+ if (isValidPath(*i)) res.insert(*i);
+ return res;
+}
+
+
+PathSet LocalStore::queryAllValidPaths()
{
SQLiteStmt stmt;
stmt.create(db, "select path from ValidPaths");
@@ -947,50 +956,66 @@ template<class T> T getIntLine(int fd)
}
-bool LocalStore::hasSubstitutes(const Path & path)
+PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
{
+ PathSet res;
foreach (Paths::iterator, i, substituters) {
+ if (res.size() == paths.size()) break;
RunningSubstituter & run(runningSubstituters[*i]);
startSubstituter(*i, run);
- writeLine(run.to, "have\n" + path);
- if (getIntLine<int>(run.from)) return true;
+ string s = "have ";
+ foreach (PathSet::const_iterator, i, paths)
+ if (res.find(*i) == res.end()) { s += *i; s += " "; }
+ writeLine(run.to, s);
+ while (true) {
+ Path path = readLine(run.from);
+ if (path == "") break;
+ res.insert(path);
+ }
}
-
- return false;
+ return res;
}
-bool LocalStore::querySubstitutablePathInfo(const Path & substituter,
- const Path & path, SubstitutablePathInfo & info)
+void LocalStore::querySubstitutablePathInfos(const Path & substituter,
+ PathSet & paths, SubstitutablePathInfos & infos)
{
RunningSubstituter & run(runningSubstituters[substituter]);
startSubstituter(substituter, run);
- writeLine(run.to, "info\n" + path);
+ string s = "info ";
+ foreach (PathSet::const_iterator, i, paths)
+ if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
+ writeLine(run.to, s);
- if (!getIntLine<int>(run.from)) return false;
-
- info.deriver = readLine(run.from);
- if (info.deriver != "") assertStorePath(info.deriver);
- int nrRefs = getIntLine<int>(run.from);
- while (nrRefs--) {
- Path p = readLine(run.from);
- assertStorePath(p);
- info.references.insert(p);
+ while (true) {
+ Path path = readLine(run.from);
+ if (path == "") break;
+ assert(paths.find(path) != paths.end());
+ paths.erase(path);
+ SubstitutablePathInfo & info(infos[path]);
+ info.deriver = readLine(run.from);
+ if (info.deriver != "") assertStorePath(info.deriver);
+ int nrRefs = getIntLine<int>(run.from);
+ while (nrRefs--) {
+ Path p = readLine(run.from);
+ assertStorePath(p);
+ info.references.insert(p);
+ }
+ info.downloadSize = getIntLine<long long>(run.from);
+ info.narSize = getIntLine<long long>(run.from);
}
- info.downloadSize = getIntLine<long long>(run.from);
- info.narSize = getIntLine<long long>(run.from);
-
- return true;
}
-bool LocalStore::querySubstitutablePathInfo(const Path & path,
- SubstitutablePathInfo & info)
+void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos)
{
- foreach (Paths::iterator, i, substituters)
- if (querySubstitutablePathInfo(*i, path, info)) return true;
- return false;
+ PathSet todo = paths;
+ foreach (Paths::iterator, i, substituters) {
+ if (todo.empty()) break;
+ querySubstitutablePathInfos(*i, todo, infos);
+ }
}
@@ -1134,7 +1159,7 @@ Path LocalStore::addToStore(const Path & _srcPath,
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
- if (recursive)
+ if (recursive)
dumpPath(srcPath, sink, filter);
else
sink.s = readFile(srcPath);
@@ -1465,7 +1490,7 @@ void LocalStore::verifyStore(bool checkContents)
/* Check whether all valid paths actually exist. */
printMsg(lvlInfo, "checking path existence...");
- PathSet validPaths2 = queryValidPaths(), validPaths, done;
+ PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
foreach (PathSet::iterator, i, validPaths2)
verifyPath(*i, store, done, validPaths);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 65ee029c2..07d8198ec 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -99,7 +99,9 @@ public:
bool isValidPath(const Path & path);
- PathSet queryValidPaths();
+ PathSet queryValidPaths(const PathSet & paths);
+
+ PathSet queryAllValidPaths();
ValidPathInfo queryPathInfo(const Path & path);
@@ -123,15 +125,13 @@ public:
Path queryPathFromHashPart(const string & hashPart);
- PathSet querySubstitutablePaths();
-
- bool hasSubstitutes(const Path & path);
+ PathSet querySubstitutablePaths(const PathSet & paths);
- bool querySubstitutablePathInfo(const Path & path,
- SubstitutablePathInfo & info);
+ void querySubstitutablePathInfos(const Path & substituter,
+ PathSet & paths, SubstitutablePathInfos & infos);
- bool querySubstitutablePathInfo(const Path & substituter,
- const Path & path, SubstitutablePathInfo & info);
+ void querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos);
Path addToStore(const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 093499936..aa5f6ff72 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -55,45 +55,97 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
PathSet todo(targets.begin(), targets.end()), done;
+ bool useSubstitutes = queryBoolSetting("build-use-substitutes", true);
+
+ /* Getting substitute info has high latency when using the binary
+ cache substituter. Thus it's essential to do substitute
+ queries in parallel as much as possible. To accomplish this
+ we do the following:
+
+ - For all paths still to be processed (‘todo’), we add all
+ paths for which we need info to the set ‘query’. For an
+ unbuilt derivation this is the output paths; otherwise, it's
+ the path itself.
+
+ - We get info about all paths in ‘query’ in parallel.
+
+ - We process the results and add new items to ‘todo’ if
+ necessary. E.g. if a path is substitutable, then we need to
+ get info on its references.
+
+ - Repeat until ‘todo’ is empty.
+ */
+
while (!todo.empty()) {
- Path p = *(todo.begin());
- todo.erase(p);
- if (done.find(p) != done.end()) continue;
- done.insert(p);
-
- if (isDerivation(p)) {
- if (!store.isValidPath(p)) {
- unknown.insert(p);
- continue;
+
+ PathSet query, todoDrv, todoNonDrv;
+
+ foreach (PathSet::iterator, i, todo) {
+ if (done.find(*i) != done.end()) continue;
+ done.insert(*i);
+
+ if (isDerivation(*i)) {
+ if (!store.isValidPath(*i)) {
+ // FIXME: we could try to substitute p.
+ unknown.insert(*i);
+ continue;
+ }
+ Derivation drv = derivationFromPath(store, *i);
+
+ PathSet invalid;
+ foreach (DerivationOutputs::iterator, j, drv.outputs)
+ if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path);
+ if (invalid.empty()) continue;
+
+ todoDrv.insert(*i);
+ if (useSubstitutes) query.insert(invalid.begin(), invalid.end());
+ }
+
+ else {
+ if (store.isValidPath(*i)) continue;
+ query.insert(*i);
+ todoNonDrv.insert(*i);
}
- Derivation drv = derivationFromPath(store, p);
+ }
+
+ todo.clear();
+
+ SubstitutablePathInfos infos;
+ store.querySubstitutablePathInfos(query, infos);
+
+ foreach (PathSet::iterator, i, todoDrv) {
+ // FIXME: cache this
+ Derivation drv = derivationFromPath(store, *i);
bool mustBuild = false;
- foreach (DerivationOutputs::iterator, i, drv.outputs)
- if (!store.isValidPath(i->second.path) &&
- !(queryBoolSetting("build-use-substitutes", true) && store.hasSubstitutes(i->second.path)))
- mustBuild = true;
+ if (useSubstitutes) {
+ foreach (DerivationOutputs::iterator, j, drv.outputs)
+ if (!store.isValidPath(j->second.path) &&
+ infos.find(j->second.path) == infos.end())
+ mustBuild = true;
+ } else
+ mustBuild = true;
if (mustBuild) {
- willBuild.insert(p);
+ willBuild.insert(*i);
todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
foreach (DerivationInputs::iterator, i, drv.inputDrvs)
todo.insert(i->first);
- } else
+ } else
foreach (DerivationOutputs::iterator, i, drv.outputs)
- todo.insert(i->second.path);
+ todoNonDrv.insert(i->second.path);
}
-
- else {
- if (store.isValidPath(p)) continue;
- SubstitutablePathInfo info;
- if (store.querySubstitutablePathInfo(p, info)) {
- willSubstitute.insert(p);
- downloadSize += info.downloadSize;
- narSize += info.narSize;
- todo.insert(info.references.begin(), info.references.end());
+
+ foreach (PathSet::iterator, i, todoNonDrv) {
+ done.insert(*i);
+ SubstitutablePathInfos::iterator info = infos.find(*i);
+ if (info != infos.end()) {
+ willSubstitute.insert(*i);
+ downloadSize += info->second.downloadSize;
+ narSize += info->second.narSize;
+ todo.insert(info->second.references.begin(), info->second.references.end());
} else
- unknown.insert(p);
+ unknown.insert(*i);
}
}
}
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index 2ca98f46d..a486e66ef 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -178,7 +178,7 @@ void LocalStore::optimiseStore(bool dryRun, OptimiseStats & stats)
{
HashToPath hashToPath;
- PathSet paths = queryValidPaths();
+ PathSet paths = queryAllValidPaths();
foreach (PathSet::iterator, i, paths) {
addTempRoot(*i);
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index cbb70b2fd..35530acab 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -217,42 +217,96 @@ bool RemoteStore::isValidPath(const Path & path)
}
-PathSet RemoteStore::queryValidPaths()
+PathSet RemoteStore::queryValidPaths(const PathSet & paths)
{
openConnection();
- writeInt(wopQueryValidPaths, to);
+ if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+ PathSet res;
+ foreach (PathSet::const_iterator, i, paths)
+ if (isValidPath(*i)) res.insert(*i);
+ return res;
+ } else {
+ writeInt(wopQueryValidPaths, to);
+ writeStrings(paths, to);
+ processStderr();
+ return readStorePaths<PathSet>(from);
+ }
+}
+
+
+PathSet RemoteStore::queryAllValidPaths()
+{
+ openConnection();
+ writeInt(wopQueryAllValidPaths, to);
processStderr();
return readStorePaths<PathSet>(from);
}
-bool RemoteStore::hasSubstitutes(const Path & path)
+PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths)
{
openConnection();
- writeInt(wopHasSubstitutes, to);
- writeString(path, to);
- processStderr();
- unsigned int reply = readInt(from);
- return reply != 0;
+ if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+ PathSet res;
+ foreach (PathSet::const_iterator, i, paths) {
+ writeInt(wopHasSubstitutes, to);
+ writeString(*i, to);
+ processStderr();
+ if (readInt(from)) res.insert(*i);
+ }
+ return res;
+ } else {
+ writeInt(wopQuerySubstitutablePaths, to);
+ writeStrings(paths, to);
+ processStderr();
+ return readStorePaths<PathSet>(from);
+ }
}
-bool RemoteStore::querySubstitutablePathInfo(const Path & path,
- SubstitutablePathInfo & info)
+void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos)
{
+ if (paths.empty()) return;
+
openConnection();
- if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return false;
- writeInt(wopQuerySubstitutablePathInfo, to);
- writeString(path, to);
- processStderr();
- unsigned int reply = readInt(from);
- if (reply == 0) return false;
- info.deriver = readString(from);
- if (info.deriver != "") assertStorePath(info.deriver);
- info.references = readStorePaths<PathSet>(from);
- info.downloadSize = readLongLong(from);
- info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
- return true;
+
+ if (GET_PROTOCOL_MINOR(daemonVersion) < 3) return;
+
+ if (GET_PROTOCOL_MINOR(daemonVersion) < 12) {
+
+ foreach (PathSet::const_iterator, i, paths) {
+ SubstitutablePathInfo info;
+ writeInt(wopQuerySubstitutablePathInfo, to);
+ writeString(*i, to);
+ processStderr();
+ unsigned int reply = readInt(from);
+ if (reply == 0) continue;
+ info.deriver = readString(from);
+ if (info.deriver != "") assertStorePath(info.deriver);
+ info.references = readStorePaths<PathSet>(from);
+ info.downloadSize = readLongLong(from);
+ info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0;
+ infos[*i] = info;
+ }
+
+ } else {
+
+ writeInt(wopQuerySubstitutablePathInfos, to);
+ writeStrings(paths, to);
+ processStderr();
+ unsigned int count = readInt(from);
+ for (unsigned int n = 0; n < count; n++) {
+ Path path = readStorePath(from);
+ SubstitutablePathInfo & info(infos[path]);
+ info.deriver = readString(from);
+ if (info.deriver != "") assertStorePath(info.deriver);
+ info.references = readStorePaths<PathSet>(from);
+ info.downloadSize = readLongLong(from);
+ info.narSize = readLongLong(from);
+ }
+
+ }
}
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index f0e5dbf76..68db0640a 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -27,7 +27,9 @@ public:
bool isValidPath(const Path & path);
- PathSet queryValidPaths();
+ PathSet queryValidPaths(const PathSet & paths);
+
+ PathSet queryAllValidPaths();
ValidPathInfo queryPathInfo(const Path & path);
@@ -45,10 +47,10 @@ public:
Path queryPathFromHashPart(const string & hashPart);
- bool hasSubstitutes(const Path & path);
+ PathSet querySubstitutablePaths(const PathSet & paths);
- bool querySubstitutablePathInfo(const Path & path,
- SubstitutablePathInfo & info);
+ void querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos);
Path addToStore(const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 0ab15c380..5d8c09f5a 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -86,6 +86,8 @@ struct SubstitutablePathInfo
unsigned long long narSize; /* 0 = unknown */
};
+typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
+
struct ValidPathInfo
{
@@ -108,20 +110,23 @@ public:
virtual ~StoreAPI() { }
- /* Checks whether a path is valid. */
+ /* Check whether a path is valid. */
virtual bool isValidPath(const Path & path) = 0;
- /* Query the set of valid paths. */
- virtual PathSet queryValidPaths() = 0;
+ /* Query which of the given paths is valid. */
+ virtual PathSet queryValidPaths(const PathSet & paths) = 0;
+
+ /* Query the set of all valid paths. */
+ virtual PathSet queryAllValidPaths() = 0;
/* Query information about a valid path. */
virtual ValidPathInfo queryPathInfo(const Path & path) = 0;
- /* Queries the hash of a valid path. */
+ /* Query the hash of a valid path. */
virtual Hash queryPathHash(const Path & path) = 0;
- /* Queries the set of outgoing FS references for a store path.
- The result is not cleared. */
+ /* Query the set of outgoing FS references for a store path. The
+ result is not cleared. */
virtual void queryReferences(const Path & path,
PathSet & references) = 0;
@@ -144,13 +149,14 @@ public:
path, or "" if the path doesn't exist. */
virtual Path queryPathFromHashPart(const string & hashPart) = 0;
- /* Query whether a path has substitutes. */
- virtual bool hasSubstitutes(const Path & path) = 0;
-
- /* Query the references, deriver and download size of a
- substitutable path. */
- virtual bool querySubstitutablePathInfo(const Path & path,
- SubstitutablePathInfo & info) = 0;
+ /* Query which of the given paths have substitutes. */
+ virtual PathSet querySubstitutablePaths(const PathSet & paths) = 0;
+
+ /* Query substitute info (i.e. references, derivers and download
+ sizes) of a set of paths. If a path does not have substitute
+ info, it's omitted from the resulting ‘infos’ map. */
+ virtual void querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index b08410fa1..d4ac0ea16 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -8,7 +8,7 @@ namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
-#define PROTOCOL_VERSION 0x10b
+#define PROTOCOL_VERSION 0x10c
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@@ -34,13 +34,16 @@ typedef enum {
wopCollectGarbage = 20,
wopQuerySubstitutablePathInfo = 21,
wopQueryDerivationOutputs = 22,
- wopQueryValidPaths = 23,
+ wopQueryAllValidPaths = 23,
wopQueryFailedPaths = 24,
wopClearFailedPaths = 25,
wopQueryPathInfo = 26,
wopImportPaths = 27,
wopQueryDerivationOutputNames = 28,
wopQueryPathFromHashPart = 29,
+ wopQuerySubstitutablePathInfos = 30,
+ wopQueryValidPaths = 31,
+ wopQuerySubstitutablePaths = 32,
} WorkerOp;
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 7aa6276e3..f06f23dad 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -211,9 +211,12 @@ static int comparePriorities(EvalState & state,
static bool isPrebuilt(EvalState & state, const DrvInfo & elem)
{
+ assert(false);
+#if 0
return
store->isValidPath(elem.queryOutPath(state)) ||
store->hasSubstitutes(elem.queryOutPath(state));
+#endif
}
@@ -929,6 +932,22 @@ static void opQuery(Globals & globals,
installed.insert(i->queryOutPath(globals.state));
}
+
+ /* Query which paths have substitutes. */
+ PathSet validPaths, substitutablePaths;
+ if (printStatus) {
+ PathSet paths;
+ foreach (vector<DrvInfo>::iterator, i, elems2)
+ try {
+ paths.insert(i->queryOutPath(globals.state));
+ } catch (AssertionError & e) {
+ printMsg(lvlTalkative, format("skipping derivation named `%1%' which gives an assertion failure") % i->name);
+ i->setFailed();
+ }
+ validPaths = store->queryValidPaths(paths);
+ substitutablePaths = store->querySubstitutablePaths(paths);
+ }
+
/* Print the desired columns, or XML output. */
Table table;
@@ -938,6 +957,8 @@ static void opQuery(Globals & globals,
foreach (vector<DrvInfo>::iterator, i, elems2) {
try {
+ if (i->hasFailed()) continue;
+
startNest(nest, lvlDebug, format("outputting query result `%1%'") % i->attrPath);
if (globals.prebuiltOnly && !isPrebuilt(globals.state, *i)) continue;
@@ -949,9 +970,10 @@ static void opQuery(Globals & globals,
XMLAttrs attrs;
if (printStatus) {
- bool hasSubs = store->hasSubstitutes(i->queryOutPath(globals.state));
- bool isInstalled = installed.find(i->queryOutPath(globals.state)) != installed.end();
- bool isValid = store->isValidPath(i->queryOutPath(globals.state));
+ Path outPath = i->queryOutPath(globals.state);
+ bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end();
+ bool isInstalled = installed.find(outPath) != installed.end();
+ bool isValid = validPaths.find(outPath) != validPaths.end();
if (xmlOutput) {
attrs["installed"] = isInstalled ? "1" : "0";
attrs["valid"] = isValid ? "1" : "0";
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 23863525f..47c76693f 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -465,7 +465,7 @@ static void opDumpDB(Strings opFlags, Strings opArgs)
if (!opFlags.empty()) throw UsageError("unknown flag");
if (!opArgs.empty())
throw UsageError("no arguments expected");
- PathSet validPaths = store->queryValidPaths();
+ PathSet validPaths = store->queryAllValidPaths();
foreach (PathSet::iterator, i, validPaths)
cout << store->makeValidityRegistration(singleton<PathSet>(*i), true, true);
}
diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc
index 74a619c71..f2ca0a892 100644
--- a/src/nix-worker/nix-worker.cc
+++ b/src/nix-worker/nix-worker.cc
@@ -297,15 +297,33 @@ static void performOp(unsigned int clientVersion,
break;
}
+ case wopQueryValidPaths: {
+ PathSet paths = readStorePaths<PathSet>(from);
+ startWork();
+ PathSet res = store->queryValidPaths(paths);
+ stopWork();
+ writeStrings(res, to);
+ break;
+ }
+
case wopHasSubstitutes: {
Path path = readStorePath(from);
startWork();
- bool result = store->hasSubstitutes(path);
+ PathSet res = store->querySubstitutablePaths(singleton<PathSet>(path));
stopWork();
- writeInt(result, to);
+ writeInt(res.find(path) != res.end(), to);
break;
}
+ case wopQuerySubstitutablePaths: {
+ PathSet paths = readStorePaths<PathSet>(from);
+ startWork();
+ PathSet res = store->querySubstitutablePaths(paths);
+ stopWork();
+ writeStrings(res, to);
+ break;
+ }
+
case wopQueryPathHash: {
Path path = readStorePath(from);
startWork();
@@ -538,23 +556,43 @@ static void performOp(unsigned int clientVersion,
case wopQuerySubstitutablePathInfo: {
Path path = absPath(readString(from));
startWork();
- SubstitutablePathInfo info;
- bool res = store->querySubstitutablePathInfo(path, info);
+ SubstitutablePathInfos infos;
+ store->querySubstitutablePathInfos(singleton<PathSet>(path), infos);
stopWork();
- writeInt(res ? 1 : 0, to);
- if (res) {
- writeString(info.deriver, to);
- writeStrings(info.references, to);
- writeLongLong(info.downloadSize, to);
+ SubstitutablePathInfos::iterator i = infos.find(path);
+ if (i == infos.end())
+ writeInt(0, to);
+ else {
+ writeInt(1, to);
+ writeString(i->second.deriver, to);
+ writeStrings(i->second.references, to);
+ writeLongLong(i->second.downloadSize, to);
if (GET_PROTOCOL_MINOR(clientVersion) >= 7)
- writeLongLong(info.narSize, to);
+ writeLongLong(i->second.narSize, to);
}
break;
}
- case wopQueryValidPaths: {
+ case wopQuerySubstitutablePathInfos: {
+ PathSet paths = readStorePaths<PathSet>(from);
+ startWork();
+ SubstitutablePathInfos infos;
+ store->querySubstitutablePathInfos(paths, infos);
+ stopWork();
+ writeInt(infos.size(), to);
+ foreach (SubstitutablePathInfos::iterator, i, infos) {
+ writeString(i->first, to);
+ writeString(i->second.deriver, to);
+ writeStrings(i->second.references, to);
+ writeLongLong(i->second.downloadSize, to);
+ writeLongLong(i->second.narSize, to);
+ }
+ break;
+ }
+
+ case wopQueryAllValidPaths: {
startWork();
- PathSet paths = store->queryValidPaths();
+ PathSet paths = store->queryAllValidPaths();
stopWork();
writeStrings(paths, to);
break;