aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libstore/build.cc10
-rw-r--r--src/libstore/globals.cc5
-rw-r--r--src/libstore/local-store.cc60
-rw-r--r--src/libstore/local-store.hh7
-rw-r--r--src/libstore/misc.cc106
-rw-r--r--src/libstore/remote-store.cc13
-rw-r--r--src/libstore/remote-store.hh3
-rw-r--r--src/libstore/store-api.hh5
8 files changed, 154 insertions, 55 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index d5bbd540b..1c84e5b9f 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -2352,10 +2352,12 @@ void SubstitutionGoal::tryNext()
sub = subs.front();
subs.pop_front();
- if (!worker.store.querySubstitutablePathInfo(sub, storePath, info)) {
- tryNext();
- return;
- }
+ SubstitutablePathInfos infos;
+ PathSet dummy(singleton<PathSet>(storePath));
+ worker.store.querySubstitutablePathInfos(sub, dummy, infos);
+ SubstitutablePathInfos::iterator k = infos.find(storePath);
+ if (k == infos.end()) { tryNext(); return; }
+ info = k->second;
/* To maintain the closure invariant, we first have to realise the
paths referenced by this one. */
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 5c22f1406..f660ed68d 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -155,8 +155,9 @@ void setDefaultsFromEnvironment()
string subs = getEnv("NIX_SUBSTITUTERS", "default");
if (subs == "default") {
- substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
- substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+ //substituters.push_back(nixLibexecDir + "/nix/substituters/copy-from-other-stores.pl");
+ //substituters.push_back(nixLibexecDir + "/nix/substituters/download-using-manifests.pl");
+ substituters.push_back(nixLibexecDir + "/nix/substituters/download-from-binary-cache.pl");
} else
substituters = tokenizeString(subs, ":");
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 1ce62aeaf..b4ea4b748 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -936,37 +936,57 @@ bool LocalStore::hasSubstitutes(const Path & path)
}
-bool LocalStore::querySubstitutablePathInfo(const Path & substituter,
- const Path & path, SubstitutablePathInfo & info)
+void LocalStore::querySubstitutablePathInfos(const Path & substituter,
+ PathSet & paths, SubstitutablePathInfos & infos)
{
RunningSubstituter & run(runningSubstituters[substituter]);
startSubstituter(substituter, run);
- writeLine(run.to, "info\n" + path);
+ string s = "info ";
+ foreach (PathSet::const_iterator, i, paths)
+ if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
+ writeLine(run.to, s);
- if (!getIntLine<int>(run.from)) return false;
-
- info.deriver = readLine(run.from);
- if (info.deriver != "") assertStorePath(info.deriver);
- int nrRefs = getIntLine<int>(run.from);
- while (nrRefs--) {
- Path p = readLine(run.from);
- assertStorePath(p);
- info.references.insert(p);
+ while (true) {
+ Path path = readLine(run.from);
+ if (path == "") break;
+ assert(paths.find(path) != paths.end());
+ paths.erase(path);
+ SubstitutablePathInfo & info(infos[path]);
+ info.deriver = readLine(run.from);
+ if (info.deriver != "") assertStorePath(info.deriver);
+ int nrRefs = getIntLine<int>(run.from);
+ while (nrRefs--) {
+ Path p = readLine(run.from);
+ assertStorePath(p);
+ info.references.insert(p);
+ }
+ info.downloadSize = getIntLine<long long>(run.from);
+ info.narSize = getIntLine<long long>(run.from);
}
- info.downloadSize = getIntLine<long long>(run.from);
- info.narSize = getIntLine<long long>(run.from);
-
- return true;
}
bool LocalStore::querySubstitutablePathInfo(const Path & path,
SubstitutablePathInfo & info)
{
- foreach (Paths::iterator, i, substituters)
- if (querySubstitutablePathInfo(*i, path, info)) return true;
- return false;
+ SubstitutablePathInfos infos;
+ querySubstitutablePathInfos(singleton<PathSet>(path), infos);
+ SubstitutablePathInfos::iterator i = infos.find(path);
+ if (i == infos.end()) return false;
+ info = i->second;
+ return true;
+}
+
+
+void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos)
+{
+ PathSet todo = paths;
+ foreach (Paths::iterator, i, substituters) {
+ if (todo.empty()) break;
+ querySubstitutablePathInfos(*i, todo, infos);
+ }
}
@@ -1110,7 +1130,7 @@ Path LocalStore::addToStore(const Path & _srcPath,
method for very large paths, but `copyPath' is mainly used for
small files. */
StringSink sink;
- if (recursive)
+ if (recursive)
dumpPath(srcPath, sink, filter);
else
sink.s = readFile(srcPath);
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index aa8e8582f..c4d8be692 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -128,8 +128,11 @@ public:
bool querySubstitutablePathInfo(const Path & path,
SubstitutablePathInfo & info);
- bool querySubstitutablePathInfo(const Path & substituter,
- const Path & path, SubstitutablePathInfo & info);
+ void querySubstitutablePathInfos(const Path & substituter,
+ PathSet & paths, SubstitutablePathInfos & infos);
+
+ void querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos);
Path addToStore(const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index 093499936..aa5f6ff72 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -55,45 +55,97 @@ void queryMissing(StoreAPI & store, const PathSet & targets,
PathSet todo(targets.begin(), targets.end()), done;
+ bool useSubstitutes = queryBoolSetting("build-use-substitutes", true);
+
+ /* Getting substitute info has high latency when using the binary
+ cache substituter. Thus it's essential to do substitute
+ queries in parallel as much as possible. To accomplish this
+ we do the following:
+
+ - For all paths still to be processed (‘todo’), we add all
+ paths for which we need info to the set ‘query’. For an
+ unbuilt derivation this is the output paths; otherwise, it's
+ the path itself.
+
+ - We get info about all paths in ‘query’ in parallel.
+
+ - We process the results and add new items to ‘todo’ if
+ necessary. E.g. if a path is substitutable, then we need to
+ get info on its references.
+
+ - Repeat until ‘todo’ is empty.
+ */
+
while (!todo.empty()) {
- Path p = *(todo.begin());
- todo.erase(p);
- if (done.find(p) != done.end()) continue;
- done.insert(p);
-
- if (isDerivation(p)) {
- if (!store.isValidPath(p)) {
- unknown.insert(p);
- continue;
+
+ PathSet query, todoDrv, todoNonDrv;
+
+ foreach (PathSet::iterator, i, todo) {
+ if (done.find(*i) != done.end()) continue;
+ done.insert(*i);
+
+ if (isDerivation(*i)) {
+ if (!store.isValidPath(*i)) {
+ // FIXME: we could try to substitute p.
+ unknown.insert(*i);
+ continue;
+ }
+ Derivation drv = derivationFromPath(store, *i);
+
+ PathSet invalid;
+ foreach (DerivationOutputs::iterator, j, drv.outputs)
+ if (!store.isValidPath(j->second.path)) invalid.insert(j->second.path);
+ if (invalid.empty()) continue;
+
+ todoDrv.insert(*i);
+ if (useSubstitutes) query.insert(invalid.begin(), invalid.end());
+ }
+
+ else {
+ if (store.isValidPath(*i)) continue;
+ query.insert(*i);
+ todoNonDrv.insert(*i);
}
- Derivation drv = derivationFromPath(store, p);
+ }
+
+ todo.clear();
+
+ SubstitutablePathInfos infos;
+ store.querySubstitutablePathInfos(query, infos);
+
+ foreach (PathSet::iterator, i, todoDrv) {
+ // FIXME: cache this
+ Derivation drv = derivationFromPath(store, *i);
bool mustBuild = false;
- foreach (DerivationOutputs::iterator, i, drv.outputs)
- if (!store.isValidPath(i->second.path) &&
- !(queryBoolSetting("build-use-substitutes", true) && store.hasSubstitutes(i->second.path)))
- mustBuild = true;
+ if (useSubstitutes) {
+ foreach (DerivationOutputs::iterator, j, drv.outputs)
+ if (!store.isValidPath(j->second.path) &&
+ infos.find(j->second.path) == infos.end())
+ mustBuild = true;
+ } else
+ mustBuild = true;
if (mustBuild) {
- willBuild.insert(p);
+ willBuild.insert(*i);
todo.insert(drv.inputSrcs.begin(), drv.inputSrcs.end());
foreach (DerivationInputs::iterator, i, drv.inputDrvs)
todo.insert(i->first);
- } else
+ } else
foreach (DerivationOutputs::iterator, i, drv.outputs)
- todo.insert(i->second.path);
+ todoNonDrv.insert(i->second.path);
}
-
- else {
- if (store.isValidPath(p)) continue;
- SubstitutablePathInfo info;
- if (store.querySubstitutablePathInfo(p, info)) {
- willSubstitute.insert(p);
- downloadSize += info.downloadSize;
- narSize += info.narSize;
- todo.insert(info.references.begin(), info.references.end());
+
+ foreach (PathSet::iterator, i, todoNonDrv) {
+ done.insert(*i);
+ SubstitutablePathInfos::iterator info = infos.find(*i);
+ if (info != infos.end()) {
+ willSubstitute.insert(*i);
+ downloadSize += info->second.downloadSize;
+ narSize += info->second.narSize;
+ todo.insert(info->second.references.begin(), info->second.references.end());
} else
- unknown.insert(p);
+ unknown.insert(*i);
}
}
}
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 5e5561a6a..1cf67d373 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -256,6 +256,19 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path,
}
+void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos)
+{
+ if (paths.empty()) return;
+ printMsg(lvlError, format("QUERYING %1% (REMOTE)") % showPaths(paths));
+ foreach (PathSet::const_iterator, i, paths) {
+ SubstitutablePathInfo info;
+ if (querySubstitutablePathInfo(*i, info))
+ infos[*i] = info;
+ }
+}
+
+
ValidPathInfo RemoteStore::queryPathInfo(const Path & path)
{
openConnection();
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index e9f40da6d..1056a6115 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -48,6 +48,9 @@ public:
bool querySubstitutablePathInfo(const Path & path,
SubstitutablePathInfo & info);
+ void querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos);
+
Path addToStore(const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter);
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index bf3269f57..92b2ddb1e 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -86,6 +86,8 @@ struct SubstitutablePathInfo
unsigned long long narSize; /* 0 = unknown */
};
+typedef std::map<Path, SubstitutablePathInfo> SubstitutablePathInfos;
+
struct ValidPathInfo
{
@@ -147,6 +149,9 @@ public:
substitutable path. */
virtual bool querySubstitutablePathInfo(const Path & path,
SubstitutablePathInfo & info) = 0;
+
+ virtual void querySubstitutablePathInfos(const PathSet & paths,
+ SubstitutablePathInfos & infos) = 0;
/* Copy the contents of a path to the store and register the
validity the resulting path. The resulting path is returned.
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 14:21:12 +0000