diff options
| -rw-r--r-- | src/libstore/build/derivation-goal.cc | 37 | ||||
| -rw-r--r-- | src/libstore/build/local-derivation-goal.cc | 115 | ||||
| -rw-r--r-- | src/libstore/parsed-derivations.cc | 110 | ||||
| -rw-r--r-- | src/libstore/parsed-derivations.hh | 4 | ||||
| -rw-r--r-- | src/libstore/store-api.cc | 36 | ||||
| -rw-r--r-- | src/libstore/store-api.hh | 2 | ||||
| -rwxr-xr-x | src/nix-build/nix-build.cc | 45 | ||||
| -rw-r--r-- | tests/structured-attrs-shell.nix | 19 | ||||
| -rw-r--r-- | tests/structured-attrs.sh | 6 |
9 files changed, 228 insertions, 146 deletions
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 9100d3333..4c49f3cab 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -143,7 +143,6 @@ void DerivationGoal::work() (this->*state)(); } - void DerivationGoal::addWantedOutputs(const StringSet & outputs) { /* If we already want all outputs, there is nothing to do. */ @@ -1048,42 +1047,6 @@ HookReply DerivationGoal::tryBuildHook() } -StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths) -{ - StorePathSet paths; - - for (auto & storePath : storePaths) { - if (!inputPaths.count(storePath)) - throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", worker.store.printStorePath(storePath)); - - worker.store.computeFSClosure({storePath}, paths); - } - - /* If there are derivations in the graph, then include their - outputs as well. This is useful if you want to do things - like passing all build-time dependencies of some path to a - derivation that builds a NixOS DVD image. */ - auto paths2 = paths; - - for (auto & j : paths2) { - if (j.isDerivation()) { - Derivation drv = worker.store.derivationFromPath(j); - for (auto & k : drv.outputsAndOptPaths(worker.store)) { - if (!k.second.second) - /* FIXME: I am confused why we are calling - `computeFSClosure` on the output path, rather than - derivation itself. That doesn't seem right to me, so I - won't try to implemented this for CA derivations. */ - throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); - worker.store.computeFSClosure(*k.second.second, paths); - } - } - } - - return paths; -} - - void DerivationGoal::registerOutputs() { /* When using a build hook, the build hook can register the output diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 279139020..9bb6f276c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -518,7 +518,7 @@ void LocalDerivationGoal::startBuilder() /* Write closure info to <fileName>. */ writeFile(tmpDir + "/" + fileName, worker.store.makeValidityRegistration( - exportReferences({storePath}), false, false)); + worker.store.exportReferences({storePath}, inputPaths), false, false)); } } @@ -1084,113 +1084,18 @@ void LocalDerivationGoal::initEnv() } -static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); - - void LocalDerivationGoal::writeStructuredAttrs() { - auto structuredAttrs = parsedDrv->getStructuredAttrs(); - if (!structuredAttrs) return; - - auto json = *structuredAttrs; - - /* Add an "outputs" object containing the output paths. */ - nlohmann::json outputs; - for (auto & i : drv->outputs) { - /* The placeholder must have a rewrite, so we use it to cover both the - cases where we know or don't know the output path ahead of time. */ - outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites); - } - json["outputs"] = outputs; - - /* Handle exportReferencesGraph. */ - auto e = json.find("exportReferencesGraph"); - if (e != json.end() && e->is_object()) { - for (auto i = e->begin(); i != e->end(); ++i) { - std::ostringstream str; - { - JSONPlaceholder jsonRoot(str, true); - StorePathSet storePaths; - for (auto & p : *i) - storePaths.insert(worker.store.parseStorePath(p.get<std::string>())); - worker.store.pathInfoToJSON(jsonRoot, - exportReferences(storePaths), false, true); - } - json[i.key()] = nlohmann::json::parse(str.str()); // urgh - } + if (auto structAttrs = parsedDrv->generateStructuredAttrs(inputRewrites, worker.store, inputPaths)) { + auto value = structAttrs.value(); + auto jsonSh = value.first; + auto json = value.second; + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); } - - writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.json"); - - /* As a convenience to bash scripts, write a shell file that - maps all attributes that are representable in bash - - namely, strings, integers, nulls, Booleans, and arrays and - objects consisting entirely of those values. (So nested - arrays or objects are not supported.) */ - - auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> { - if (value.is_string()) - return shellEscape(value); - - if (value.is_number()) { - auto f = value.get<float>(); - if (std::ceil(f) == f) - return std::to_string(value.get<int>()); - } - - if (value.is_null()) - return std::string("''"); - - if (value.is_boolean()) - return value.get<bool>() ? std::string("1") : std::string(""); - - return {}; - }; - - std::string jsonSh; - - for (auto i = json.begin(); i != json.end(); ++i) { - - if (!std::regex_match(i.key(), shVarName)) continue; - - auto & value = i.value(); - - auto s = handleSimpleType(value); - if (s) - jsonSh += fmt("declare %s=%s\n", i.key(), *s); - - else if (value.is_array()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += *s3; s2 += ' '; - } - - if (good) - jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); - } - - else if (value.is_object()) { - std::string s2; - bool good = true; - - for (auto i = value.begin(); i != value.end(); ++i) { - auto s3 = handleSimpleType(i.value()); - if (!s3) { good = false; break; } - s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); - } - - if (good) - jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); - } - } - - writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); - chownToBuilder(tmpDir + "/.attrs.sh"); } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index c5c3ae3dc..029da8bd3 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -1,6 +1,8 @@ #include "parsed-derivations.hh" #include <nlohmann/json.hpp> +#include <regex> +#include "json.hh" namespace nix { @@ -121,4 +123,112 @@ bool ParsedDerivation::substitutesAllowed() const return getBoolAttr("allowSubstitutes", true); } +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); +std::optional<StructuredAttrsWithShellRC> ParsedDerivation::generateStructuredAttrs( + std::optional<StringMap> inputRewrites, Store & store, const StorePathSet & inputPaths) +{ + auto structuredAttrs = getStructuredAttrs(); + if (!structuredAttrs) return std::nullopt; + + auto json = *structuredAttrs; + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv.outputs) { + if (inputRewrites) { + /* The placeholder must have a rewrite, so we use it to cover both the + cases where we know or don't know the output path ahead of time. */ + outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites.value()); + } else { + /* This case is only relevant for the nix-shell */ + outputs[i.first] = hashPlaceholder(i.first); + } + } + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + StorePathSet storePaths; + for (auto & p : *i) + storePaths.insert(store.parseStorePath(p.get<std::string>())); + store.pathInfoToJSON(jsonRoot, + store.exportReferences(storePaths, inputPaths), false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ + + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> { + if (value.is_string()) + return shellEscape(value); + + if (value.is_number()) { + auto f = value.get<float>(); + if (std::ceil(f) == f) + return std::to_string(value.get<int>()); + } + + if (value.is_null()) + return std::string("''"); + + if (value.is_boolean()) + return value.get<bool>() ? std::string("1") : std::string(""); + + return {}; + }; + + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + + if (!std::regex_match(i.key(), shVarName)) continue; + + auto & value = i.value(); + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } + + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } + } + + return std::make_pair(jsonSh, json); +} } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index c9fbe68c4..4b8b8c8ff 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -6,6 +6,8 @@ namespace nix { +typedef std::pair<std::string, nlohmann::json> StructuredAttrsWithShellRC; + class ParsedDerivation { StorePath drvPath; @@ -36,6 +38,8 @@ public: bool willBuildLocally(Store & localStore) const; bool substitutesAllowed() const; + + std::optional<StructuredAttrsWithShellRC> generateStructuredAttrs(std::optional<StringMap> inputRewrites, Store & store, const StorePathSet & inputPaths); }; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 93fcb068f..cfdfb5269 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -627,6 +627,42 @@ string Store::makeValidityRegistration(const StorePathSet & paths, } +StorePathSet Store::exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths) +{ + StorePathSet paths; + + for (auto & storePath : storePaths) { + if (!inputPaths.count(storePath)) + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", printStorePath(storePath)); + + computeFSClosure({storePath}, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + auto paths2 = paths; + + for (auto & j : paths2) { + if (j.isDerivation()) { + Derivation drv = derivationFromPath(j); + for (auto & k : drv.outputsAndOptPaths(*this)) { + if (!k.second.second) + /* FIXME: I am confused why we are calling + `computeFSClosure` on the output path, rather than + derivation itself. That doesn't seem right to me, so I + won't try to implemented this for CA derivations. */ + throw UnimplementedError("exportReferences on CA derivations is not yet implemented"); + computeFSClosure(*k.second.second, paths); + } + } + } + + return paths; +} + + void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, Base hashBase, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f66298991..b742fed6d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -695,6 +695,8 @@ public: const Stats & getStats(); + StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); + /* Return the build log of the specified store path, if available, or null otherwise. */ virtual std::shared_ptr<std::string> getBuildLog(const StorePath & path) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 9acbedda2..f91a56d6a 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -1,10 +1,15 @@ #include <cstring> #include <fstream> #include <iostream> +#include <filesystem> #include <regex> #include <sstream> #include <vector> +#include <map> +#include <nlohmann/json.hpp> + +#include "parsed-derivations.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -422,12 +427,41 @@ static void main_nix_build(int argc, char * * argv) } else env[var.first] = var.second; + std::string structuredAttrsRC; + std::string exitCmd; + + if (env.count("__json")) { + StorePathSet inputs; + for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) { + auto outputs = store->queryPartialDerivationOutputMap(depDrvPath); + for (auto & i : wantedDepOutputs) { + auto o = outputs.at(i); + store->computeFSClosure(*o, inputs); + } + } + + ParsedDerivation parsedDrv( + StorePath(store->parseStorePath(drvInfo.queryDrvPath())), + drv + ); + + if (auto structAttrs = parsedDrv.generateStructuredAttrs(std::nullopt, *store, inputs)) { + auto val = structAttrs.value(); + structuredAttrsRC = val.first; + auto attrsJSON = std::filesystem::current_path().string() + "/.attrs.json"; + writeFile(attrsJSON, val.second.dump()); + exitCmd = "\n_rm_attrs_json() { rm -f " + attrsJSON + "; }" + + "\nexitHooks+=(_rm_attrs_json)" + + "\nfailureHooks+=(_rm_attrs_json)\n"; + } + } + /* Run a shell using the derivation's environment. For convenience, source $stdenv/setup to setup additional environment variables and shell functions. Also don't lose the current $PATH directories. */ auto rcfile = (Path) tmpDir + "/rc"; - writeFile(rcfile, fmt( + std::string rc = fmt( R"(_nix_shell_clean_tmpdir() { rm -rf %1%; }; )"s + (keepTmp ? "trap _nix_shell_clean_tmpdir EXIT; " @@ -436,8 +470,9 @@ static void main_nix_build(int argc, char * * argv) "_nix_shell_clean_tmpdir; ") + (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + "%2%" - "dontAddDisableDepTrack=1; " - "[ -e $stdenv/setup ] && source $stdenv/setup; " + "dontAddDisableDepTrack=1;\n" + + structuredAttrsRC + exitCmd + + "\n[ -e $stdenv/setup ] && source $stdenv/setup; " "%3%" "PATH=%4%:\"$PATH\"; " "SHELL=%5%; " @@ -455,7 +490,9 @@ static void main_nix_build(int argc, char * * argv) shellEscape(dirOf(*shell)), shellEscape(*shell), (getenv("TZ") ? (string("export TZ=") + shellEscape(getenv("TZ")) + "; ") : ""), - envCommand)); + envCommand); + vomit("Sourcing nix-shell with file %s and contents:\n%s", rcfile, rc); + writeFile(rcfile, rc); Strings envStrs; for (auto & i : env) diff --git a/tests/structured-attrs-shell.nix b/tests/structured-attrs-shell.nix new file mode 100644 index 000000000..0c01c2568 --- /dev/null +++ b/tests/structured-attrs-shell.nix @@ -0,0 +1,19 @@ +with import ./config.nix; +let + dep = mkDerivation { + name = "dep"; + buildCommand = '' + mkdir $out; echo bla > $out/bla + ''; + }; +in +mkDerivation { + name = "structured2"; + __structuredAttrs = true; + outputs = [ "out" "dev" ]; + my.list = [ "a" "b" "c" ]; + exportReferencesGraph.refs = [ dep ]; + buildCommand = '' + touch ''${outputs[out]}; touch ''${outputs[dev]} + ''; +} diff --git a/tests/structured-attrs.sh b/tests/structured-attrs.sh index dcfe6d580..c0fcb021a 100644 --- a/tests/structured-attrs.sh +++ b/tests/structured-attrs.sh @@ -8,3 +8,9 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result [[ $(cat $TEST_ROOT/result/foo) = bar ]] [[ $(cat $TEST_ROOT/result-dev/foo) = foo ]] + +export NIX_BUILD_SHELL=$SHELL +[[ ! -e '.attrs.json' ]] +env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ + --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < .attrs.json)"' +[[ ! -e '.attrs.json' ]] |