diff options
Diffstat (limited to 'src/libstore/build/local-derivation-goal.cc')
| -rw-r--r-- | src/libstore/build/local-derivation-goal.cc | 235 |
1 files changed, 79 insertions, 156 deletions
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ab37f9523..b4137839d 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -17,10 +17,7 @@ #include <regex> #include <queue> -#include <sys/types.h> -#include <sys/socket.h> #include <sys/un.h> -#include <netdb.h> #include <fcntl.h> #include <termios.h> #include <unistd.h> @@ -34,7 +31,6 @@ /* Includes required for chroot support. */ #if __linux__ -#include <sys/socket.h> #include <sys/ioctl.h> #include <net/if.h> #include <netinet/ip.h> @@ -70,12 +66,14 @@ void handleDiffHook( auto diffHook = settings.diffHook; if (diffHook != "" && settings.runDiffHook) { try { - RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); - diffHookOptions.searchPath = true; - diffHookOptions.uid = uid; - diffHookOptions.gid = gid; - diffHookOptions.chdir = "/"; - auto diffRes = runProgram(diffHookOptions); + auto diffRes = runProgram(RunOptions { + .program = diffHook, + .searchPath = true, + .args = {tryA, tryB, drvPath, tmpDir}, + .uid = uid, + .gid = gid, + .chdir = "/" + }); if (!statusOk(diffRes.first)) throw ExecError(diffRes.first, "diff-hook program '%1%' %2%", @@ -153,6 +151,7 @@ void LocalDerivationGoal::killChild() void LocalDerivationGoal::tryLocalBuild() { unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { + state = &DerivationGoal::tryToBuild; worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; @@ -284,7 +283,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; - if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 && + if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && @@ -336,23 +335,6 @@ int childEntry(void * arg) } -static std::once_flag dns_resolve_flag; - -static void preloadNSS() { - /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of - one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already - been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to - load its lookup libraries in the parent before any child gets a chance to. */ - std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { - if (res) freeaddrinfo(res); - } - }); -} - - static void linkOrCopy(const Path & from, const Path & to) { if (link(from.c_str(), to.c_str()) == -1) { @@ -381,9 +363,6 @@ void LocalDerivationGoal::startBuilder() settings.thisSystem, concatStringsSep<StringSet>(", ", worker.store.systemFeatures)); - if (drv->isBuiltin()) - preloadNSS(); - #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif @@ -409,7 +388,7 @@ void LocalDerivationGoal::startBuilder() } auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir) { + if (localStore.storeDir != localStore.realStoreDir.get()) { #if __linux__ useChroot = true; #else @@ -510,7 +489,7 @@ void LocalDerivationGoal::startBuilder() /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); + worker.store.exportReferences({storePath}, inputPaths), false, false)); } } @@ -577,7 +556,9 @@ void LocalDerivationGoal::startBuilder() throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps", worker.store.printStorePath(drvPath), i); - dirsInChroot[i] = i; + /* Allow files in __impureHostDeps to be missing; e.g. + macOS 11+ has no /usr/lib/libSystem*.dylib */ + dirsInChroot[i] = {i, true}; } #if __linux__ @@ -1110,113 +1091,28 @@ void LocalDerivationGoal::initEnv() } -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - void LocalDerivationGoal::writeStructuredAttrs() { - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get<std::string>())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } - } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get<float>(); - if (std::ceil(f) == f) - return std::to_string(value.get<int>()); + if (auto structAttrsJson = parsedDrv->prepareStructuredAttrs(worker.store, inputPaths)) { + auto json = structAttrsJson.value(); + nlohmann::json rewritten; + for (auto & [i, v] : json["outputs"].get<nlohmann::json::object_t>()) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + rewritten[i] = rewriteStrings(v, inputRewrites); } - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get<bool>() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } + json["outputs"] = rewritten; - else if (value.is_object()) { - std::string s2; - bool good = true; + auto jsonSh = writeStructuredAttrsShell(json); - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); + env["NIX_ATTRS_SH_FILE"] = tmpDir + "/.attrs.sh"; + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); + env["NIX_ATTRS_JSON_FILE"] = tmpDir + "/.attrs.json"; } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); } @@ -1359,13 +1255,20 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo std::optional<const Realisation> queryRealisation(const DrvOutput & id) override // XXX: This should probably be allowed if the realisation corresponds to // an allowed derivation - { throw Error("queryRealisation"); } + { + if (!goal.isAllowed(id)) + throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string()); + return next->queryRealisation(id); + } - void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override + void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override { + assert(!evalStore); + if (buildMode != bmNormal) throw Error("unsupported build mode"); StorePathSet newPaths; + std::set<Realisation> newRealisations; for (auto & req : paths) { if (!goal.isAllowed(req)) @@ -1378,16 +1281,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo auto p = std::get_if<DerivedPath::Built>(&path); if (!p) continue; auto & bfd = *p; + auto drv = readDerivation(bfd.drvPath); + auto drvHashes = staticOutputHashes(*this, drv); auto outputs = next->queryDerivationOutputMap(bfd.drvPath); for (auto & [outputName, outputPath] : outputs) - if (wantOutput(outputName, bfd.outputs)) + if (wantOutput(outputName, bfd.outputs)) { newPaths.insert(outputPath); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto thisRealisation = next->queryRealisation( + DrvOutput{drvHashes.at(outputName), outputName} + ); + assert(thisRealisation); + newRealisations.insert(*thisRealisation); + } + } } StorePathSet closure; next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); + for (auto & real : Realisation::closure(*next, newRealisations)) + goal.addedDrvOutputs.insert(real.id); } BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, @@ -1763,7 +1678,7 @@ void LocalDerivationGoal::runChild() /* N.B. it is realistic that these paths might not exist. It happens when testing Nix building fixed-output derivations within a pure derivation. */ - for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" }) + for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" }) if (pathExists(path)) ss.push_back(path); } @@ -2400,32 +2315,19 @@ void LocalDerivationGoal::registerOutputs() } auto got = caSink.finish().first; auto refs = rewriteRefs(); - HashModuloSink narSink { htSHA256, oldHashPart }; - dumpPath(actualPath, narSink); - auto narHashAndSize = narSink.finish(); - ValidPathInfo newInfo0 { - worker.store.makeFixedOutputPath( + + auto finalPath = worker.store.makeFixedOutputPath( outputHash.method, got, outputPathName(drv->name, outputName), refs.second, - refs.first), - narHashAndSize.first, - }; - newInfo0.narSize = narHashAndSize.second; - newInfo0.ca = FixedOutputHash { - .method = outputHash.method, - .hash = got, - }; - newInfo0.references = refs.second; - if (refs.first) - newInfo0.references.insert(newInfo0.path); - if (scratchPath != newInfo0.path) { + refs.first); + if (scratchPath != finalPath) { // Also rewrite the output path auto source = sinkToSource([&](Sink & nextSink) { StringSink sink; dumpPath(actualPath, sink); - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); + RewritingSink rsink2(oldHashPart, std::string(finalPath.hashPart()), nextSink); rsink2(*sink.s); rsink2.flush(); }); @@ -2435,6 +2337,21 @@ void LocalDerivationGoal::registerOutputs() movePath(tmpPath, actualPath); } + HashResult narHashAndSize = hashPath(htSHA256, actualPath); + ValidPathInfo newInfo0 { + finalPath, + narHashAndSize.first, + }; + + newInfo0.narSize = narHashAndSize.second; + newInfo0.ca = FixedOutputHash { + .method = outputHash.method, + .hash = got, + }; + newInfo0.references = refs.second; + if (refs.first) + newInfo0.references.insert(newInfo0.path); + assert(newInfo0.ca); return newInfo0; }; @@ -2495,6 +2412,10 @@ void LocalDerivationGoal::registerOutputs() }, }, output.output); + /* FIXME: set proper permissions in restorePath() so + we don't have to do another traversal. */ + canonicalisePathMetaData(actualPath, {}, inodesSeen); + /* Calculate where we'll move the output files. In the checking case we will leave leave them where they are, for now, rather than move to their usual "final destination" */ @@ -2504,6 +2425,7 @@ void LocalDerivationGoal::registerOutputs() floating CA derivations and hash-mismatching fixed-output derivations. */ PathLocks dynamicOutputLock; + dynamicOutputLock.setDeletion(true); auto optFixedPath = output.path(worker.store, drv->name, outputName); if (!optFixedPath || worker.store.printStorePath(*optFixedPath) != finalDestPath) @@ -2527,6 +2449,7 @@ void LocalDerivationGoal::registerOutputs() assert(newInfo.ca); } else { auto destPath = worker.store.toRealPath(finalDestPath); + deletePath(destPath); movePath(actualPath, destPath); actualPath = destPath; } |