aboutsummaryrefslogtreecommitdiff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/build.cc53
-rw-r--r--src/libstore/derivations.cc53
-rw-r--r--src/libstore/derivations.hh23
-rw-r--r--src/libstore/local-store.cc27
4 files changed, 146 insertions, 10 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index ba28e78c8..f5256bf87 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -984,6 +984,8 @@ private:
void tryLocalBuild();
void buildDone();
+ void resolvedFinished();
+
/* Is the build hook willing to perform the build? */
HookReply tryBuildHook();
@@ -1451,8 +1453,39 @@ void DerivationGoal::inputsRealised()
/* Determine the full set of input paths. */
/* First, the input derivations. */
- if (useDerivation)
- for (auto & [depDrvPath, wantedDepOutputs] : dynamic_cast<Derivation *>(drv.get())->inputDrvs) {
+ if (useDerivation) {
+ auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
+
+ if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
+ /* We are be able to resolve this derivation based on the
+ now-known results of dependencies. If so, we become a stub goal
+ aliasing that resolved derivation goal */
+ Derivation drvResolved { fullDrv.resolve(worker.store) };
+
+ auto pathResolved = writeDerivation(worker.store, drvResolved);
+ /* Add to memotable to speed up downstream goal's queries with the
+ original derivation. */
+ drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
+
+ auto msg = fmt("Resolved derivation: '%s' -> '%s'",
+ worker.store.printStorePath(drvPath),
+ worker.store.printStorePath(pathResolved));
+ act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg,
+ Logger::Fields {
+ worker.store.printStorePath(drvPath),
+ worker.store.printStorePath(pathResolved),
+ });
+
+ auto resolvedGoal = worker.makeDerivationGoal(
+ pathResolved, wantedOutputs,
+ buildMode == bmRepair ? bmRepair : bmNormal);
+ addWaitee(resolvedGoal);
+
+ state = &DerivationGoal::resolvedFinished;
+ return;
+ }
+
+ for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) {
/* Add the relevant output closures of the input derivation
`i' as input paths. Only add the closures of output paths
that are specified as inputs. */
@@ -1472,6 +1505,7 @@ void DerivationGoal::inputsRealised()
worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
}
}
+ }
/* Second, the input sources. */
worker.store.computeFSClosure(drv->inputSrcs, inputPaths);
@@ -1893,6 +1927,9 @@ void DerivationGoal::buildDone()
done(BuildResult::Built);
}
+void DerivationGoal::resolvedFinished() {
+ done(BuildResult::Built);
+}
HookReply DerivationGoal::tryBuildHook()
{
@@ -2065,7 +2102,7 @@ void linkOrCopy(const Path & from, const Path & to)
file (e.g. 32000 of ext3), which is quite possible after a
'nix-store --optimise'. FIXME: actually, why don't we just
bind-mount in this case?
-
+
It can also fail with EPERM in BeegFS v7 and earlier versions
which don't allow hard-links to other directories */
if (errno != EMLINK && errno != EPERM)
@@ -3850,7 +3887,7 @@ void DerivationGoal::registerOutputs()
something like that. */
canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);
- debug("scanning for references for output %1 in temp location '%1%'", outputName, actualPath);
+ debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
/* Pass blank Sink as we are not ready to hash data at this stage. */
NullSink blank;
@@ -4248,10 +4285,14 @@ void DerivationGoal::registerOutputs()
{
ValidPathInfos infos2;
for (auto & [outputName, newInfo] : infos) {
- /* FIXME: we will want to track this mapping in the DB whether or
- not we have a drv file. */
if (useDerivation)
worker.store.linkDeriverToPath(drvPath, outputName, newInfo.path);
+ else {
+ /* Once a floating CA derivations reaches this point, it must
+ already be resolved, drvPath the basic derivation path, and
+ a file existsing at that path for sake of the DB's foreign key. */
+ assert(drv->type() != DerivationType::CAFloating);
+ }
infos2.push_back(newInfo);
}
worker.store.registerValidPaths(infos2);
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index e5ec60811..ce57a5bb0 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -644,4 +644,57 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
return "/" + hashString(htSHA256, clearText).to_string(Base32, false);
}
+
+// N.B. Outputs are left unchanged
+static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
+
+ debug("Rewriting the derivation");
+
+ for (auto &rewrite: rewrites) {
+ debug("rewriting %s as %s", rewrite.first, rewrite.second);
+ }
+
+ drv.builder = rewriteStrings(drv.builder, rewrites);
+ for (auto & arg: drv.args) {
+ arg = rewriteStrings(arg, rewrites);
+ }
+
+ StringPairs newEnv;
+ for (auto & envVar: drv.env) {
+ auto envName = rewriteStrings(envVar.first, rewrites);
+ auto envValue = rewriteStrings(envVar.second, rewrites);
+ newEnv.emplace(envName, envValue);
+ }
+ drv.env = newEnv;
+}
+
+
+Sync<DrvPathResolutions> drvPathResolutions;
+
+BasicDerivation Derivation::resolve(Store & store) {
+ BasicDerivation resolved { *this };
+
+ // Input paths that we'll want to rewrite in the derivation
+ StringMap inputRewrites;
+
+ for (auto & input : inputDrvs) {
+ auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first);
+ StringSet newOutputNames;
+ for (auto & outputName : input.second) {
+ auto actualPathOpt = inputDrvOutputs.at(outputName);
+ if (!actualPathOpt)
+ throw Error("input drv '%s' wasn't yet built", store.printStorePath(input.first));
+ auto actualPath = *actualPathOpt;
+ inputRewrites.emplace(
+ downstreamPlaceholder(store, input.first, outputName),
+ store.printStorePath(actualPath));
+ resolved.inputSrcs.insert(std::move(actualPath));
+ }
+ }
+
+ rewriteDerivation(store, resolved, inputRewrites);
+
+ return resolved;
+}
+
}
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index c5c6e90ca..e4d85aa05 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -4,6 +4,7 @@
#include "types.hh"
#include "hash.hh"
#include "content-address.hh"
+#include "sync.hh"
#include <map>
#include <variant>
@@ -100,7 +101,7 @@ struct BasicDerivation
StringPairs env;
std::string name;
- BasicDerivation() { }
+ BasicDerivation() = default;
virtual ~BasicDerivation() { };
bool isBuiltin() const;
@@ -127,7 +128,15 @@ struct Derivation : BasicDerivation
std::string unparse(const Store & store, bool maskOutputs,
std::map<std::string, StringSet> * actualInputs = nullptr) const;
- Derivation() { }
+ /* Return the underlying basic derivation but with
+
+ 1. input drv outputs moved to input sources.
+
+ 2. placeholders replaced with realized input store paths. */
+ BasicDerivation resolve(Store & store);
+
+ Derivation() = default;
+ Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { }
};
@@ -186,6 +195,16 @@ typedef std::map<StorePath, DrvHashModulo> DrvHashes;
extern DrvHashes drvHashes; // FIXME: global, not thread-safe
+/* Memoisation of `readDerivation(..).resove()`. */
+typedef std::map<
+ StorePath,
+ std::optional<StorePath>
+> DrvPathResolutions;
+
+// FIXME: global, though at least thread-safe.
+// FIXME: arguably overlaps with hashDerivationModulo memo table.
+extern Sync<DrvPathResolutions> drvPathResolutions;
+
bool wantOutput(const string & output, const std::set<string> & wanted);
struct Source;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 0086bb13e..e51d127b3 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -803,13 +803,36 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
}
-std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path)
+std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
{
+ auto path = path_;
std::map<std::string, std::optional<StorePath>> outputs;
- BasicDerivation drv = readDerivation(path);
+ Derivation drv = readDerivation(path);
for (auto & [outName, _] : drv.outputs) {
outputs.insert_or_assign(outName, std::nullopt);
}
+ bool haveCached = false;
+ {
+ auto resolutions = drvPathResolutions.lock();
+ auto resolvedPathOptIter = resolutions->find(path);
+ if (resolvedPathOptIter != resolutions->end()) {
+ auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
+ if (resolvedPathOpt)
+ path = *resolvedPathOpt;
+ haveCached = true;
+ }
+ }
+ /* can't just use else-if instead of `!haveCached` because we need to unlock
+ `drvPathResolutions` before it is locked in `Derivation::resolve`. */
+ if (!haveCached && drv.type() == DerivationType::CAFloating) {
+ /* Resolve drv and use that path instead. */
+ auto pathResolved = writeDerivation(*this, drv.resolve(*this));
+ /* Store in memo table. */
+ /* FIXME: memo logic should not be local-store specific, should have
+ wrapper-method instead. */
+ drvPathResolutions.lock()->insert_or_assign(path, pathResolved);
+ path = std::move(pathResolved);
+ }
return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
auto state(_state.lock());
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 09:10:25 +0000