From 2cf591a134f3ec6f634b47eeb522f422c64a9d33 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 11 Jun 2021 13:31:19 +0200 Subject: Make `nix develop` work with CA derivations Fix #4823 --- src/nix/develop.cc | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) (limited to 'src/nix/develop.cc') diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 10f843651..aeeaf3e13 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -144,17 +144,26 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ drv.name += "-env"; - for (auto & output : drv.outputs) { - output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; - drv.env[output.first] = ""; - } drv.inputSrcs.insert(std::move(getEnvShPath)); - Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto & output : drv.outputs) { + output.second = { + .output = DerivationOutputDeferred{}, + }; + drv.env[output.first] = ""; + } + } else { + for (auto & output : drv.outputs) { + output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; + drv.env[output.first] = ""; + } + Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); - for (auto & output : drv.outputs) { - auto outPath = store->makeOutputPath(output.first, h, drv.name); - output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; - drv.env[output.first] = store->printStorePath(outPath); + for (auto & output : drv.outputs) { + auto outPath = store->makeOutputPath(output.first, h, drv.name); + output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; + drv.env[output.first] = store->printStorePath(outPath); + } } auto shellDrvPath = writeDerivation(*store, drv); @@ -162,8 +171,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) /* Build the derivation. */ store->buildPaths({DerivedPath::Built{shellDrvPath}}); - for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { - auto & [_1, optPath] = outputAndOptPath; + for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); auto & outPath = *optPath; assert(store->isValidPath(outPath)); -- cgit v1.2.3 From ce674cb2cf97fc14ce48acef7f8ef88f4ac771f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 21 Jun 2021 15:52:01 +0200 Subject: Properly set the output env variables Co-authored-by: John Ericson --- src/nix/develop.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/nix/develop.cc') diff --git a/src/nix/develop.cc b/src/nix/develop.cc index aeeaf3e13..350b08648 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -150,7 +150,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) output.second = { .output = DerivationOutputDeferred{}, }; - drv.env[output.first] = ""; + drv.env[output.first] = hashPlaceholder(output.first); } } else { for (auto & output : drv.outputs) { -- cgit v1.2.3 From bf7960a4ede0f5ba7e88b939ed0f33aa6087cbc6 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 29 Jun 2021 10:13:42 +0200 Subject: develop: Discard the input{Overrides,Updates} when getting bash MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `nix develop` is getting bash from an (assumed existing) `nixpkgs` flake. However, when doing so, it reuses the `lockFlags` passed to the current flake, including the `--input-overrides` and `--input-update` which generally don’t make sense anymore at that point (and trigger a warning because of that) Clear these overrides before getting the nixpkgs flake to get rid of the warning. --- src/nix/develop.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/nix/develop.cc') diff --git a/src/nix/develop.cc b/src/nix/develop.cc index d77ff52d7..6c089469d 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -443,13 +443,17 @@ struct CmdDevelop : Common, MixEnvironment try { auto state = getEvalState(); + auto nixpkgsLockFlags = lockFlags; + nixpkgsLockFlags.inputOverrides = {}; + nixpkgsLockFlags.inputUpdates = {}; + auto bashInstallable = std::make_shared( this, state, installable->nixpkgsFlakeRef(), Strings{"bashInteractive"}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, - lockFlags); + nixpkgsLockFlags); shell = state->store->printStorePath( toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; -- cgit v1.2.3 From 087c5f5325c46485a9bc5f8e7f2620af6bf6bf56 Mon Sep 17 00:00:00 2001 From: Michael Fellinger Date: Mon, 5 Jul 2021 12:00:08 +0200 Subject: Fix devShell handling of env values including @ and % --- src/nix/develop.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/nix/develop.cc') diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 6c089469d..699ec0b99 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -54,7 +54,7 @@ BuildEnvironment readEnvironment(const Path & path) R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; static std::string simpleStringRegex = - R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; + R"re((?:[a-zA-Z0-9_/:\.\-\+=@%]*))re"; static std::string dquotedStringRegex = R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re"; -- cgit v1.2.3 From b1f1347ade81d1f04f2d490baceefb3c4de0b4e3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Jul 2021 00:47:57 +0200 Subject: nix develop: Don't parse bash environment with regexes Instead have get-env.sh dump the bash environment as JSON. This should be a lot less error-prone. Fixes #4992. --- src/nix/develop.cc | 170 +++++++++++++++++++++++++---------------------------- 1 file changed, 81 insertions(+), 89 deletions(-) (limited to 'src/nix/develop.cc') diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 699ec0b99..e00f0d575 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,7 +8,7 @@ #include "affinity.hh" #include "progress-bar.hh" -#include +#include using namespace nix; @@ -25,94 +25,98 @@ static DevelopSettings developSettings; static GlobalConfig::Register rDevelopSettings(&developSettings); -struct Var -{ - bool exported = true; - bool associative = false; - std::string quoted; // quoted string or array -}; - struct BuildEnvironment { - std::map env; - std::string bashFunctions; -}; - -BuildEnvironment readEnvironment(const Path & path) -{ - BuildEnvironment res; - - std::set exported; - - debug("reading environment file '%s'", path); - - auto file = readFile(path); - - auto pos = file.cbegin(); - - static std::string varNameRegex = - R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; - - static std::string simpleStringRegex = - R"re((?:[a-zA-Z0-9_/:\.\-\+=@%]*))re"; - - static std::string dquotedStringRegex = - R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re"; + struct String + { + bool exported; + std::string value; + }; - static std::string squotedStringRegex = - R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re"; + using Array = std::vector; - static std::string indexedArrayRegex = - R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; + using Associative = std::map; - static std::regex declareRegex( - "^declare -a?x (" + varNameRegex + ")(=(" + - dquotedStringRegex + "|" + indexedArrayRegex + "))?\n"); + using Value = std::variant; - static std::regex varRegex( - "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n"); + std::map vars; + std::map bashFunctions; - /* Note: we distinguish between an indexed and associative array - using the space before the closing parenthesis. Will - undoubtedly regret this some day. */ - static std::regex assocArrayRegex( - "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n"); + static BuildEnvironment fromJSON(const Path & path) + { + BuildEnvironment res; - static std::regex functionRegex( - "^" + varNameRegex + " \\(\\) *\n"); + std::set exported; - while (pos != file.end()) { + debug("reading environment file '%s'", path); - std::smatch match; + auto json = nlohmann::json::parse(readFile(path)); - if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - exported.insert(match[1]); + for (auto & [name, info] : json["variables"].items()) { + std::string type = info["type"]; + if (type == "var" || type == "exported") + res.vars.insert({name, BuildEnvironment::String { .exported = type == "exported", .value = info["value"] }}); + else if (type == "array") + res.vars.insert({name, (Array) info["value"]}); + else if (type == "associative") + res.vars.insert({name, (Associative) info["value"]}); } - else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }}); + for (auto & [name, def] : json["bashFunctions"].items()) { + res.bashFunctions.insert({name, def}); } - else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }}); - } + return res; + } - else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) { - res.bashFunctions = std::string(pos, file.cend()); - break; + void toBash(std::ostream & out, const std::set & ignoreVars) const + { + for (auto & [name, value] : vars) { + if (!ignoreVars.count(name) && !hasPrefix(name, "BASH_")) { + if (auto str = std::get_if(&value)) { + out << fmt("%s=%s\n", name, shellEscape(str->value)); + if (str->exported) + out << fmt("export %s\n", name); + } + else if (auto arr = std::get_if(&value)) { + out << "declare -a " << name << "=("; + for (auto & s : *arr) + out << shellEscape(s) << " "; + out << ")\n"; + } + else if (auto arr = std::get_if(&value)) { + out << "declare -A " << name << "=("; + for (auto & [n, v] : *arr) + out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " "; + out << ")\n"; + } + } } - else throw Error("shell environment '%s' has unexpected line '%s'", - path, file.substr(pos - file.cbegin(), 60)); + for (auto & [name, def] : bashFunctions) { + out << name << " ()\n{\n" << def << "}\n"; + } } - res.env.erase("__output"); + static std::string getString(const Value & value) + { + if (auto str = std::get_if(&value)) + return str->value; + else + throw Error("bash variable is not a string"); + } - return res; -} + static Array getStrings(const Value & value) + { + if (auto str = std::get_if(&value)) + return tokenizeString(str->value); + else if (auto arr = std::get_if(&value)) { + return *arr; + } + else + throw Error("bash variable is not a string or array"); + } +}; const static std::string getEnvSh = #include "get-env.sh.gen.hh" @@ -185,7 +189,7 @@ StorePath getDerivationEnvironment(ref store, const StorePath & drvPath) struct Common : InstallableCommand, MixProfile { - std::set ignoreVars{ + std::set ignoreVars{ "BASHOPTS", "EUID", "HOME", // FIXME: don't ignore in pure mode? @@ -233,22 +237,10 @@ struct Common : InstallableCommand, MixProfile out << "nix_saved_PATH=\"$PATH\"\n"; - for (auto & i : buildEnvironment.env) { - if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { - if (i.second.associative) - out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted); - else { - out << fmt("%s=%s\n", i.first, i.second.quoted); - if (i.second.exported) - out << fmt("export %s\n", i.first); - } - } - } + buildEnvironment.toBash(out, ignoreVars); out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; - out << buildEnvironment.bashFunctions << "\n"; - out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); @@ -258,16 +250,16 @@ struct Common : InstallableCommand, MixProfile auto script = out.str(); /* Substitute occurrences of output paths. */ - auto outputs = buildEnvironment.env.find("outputs"); - assert(outputs != buildEnvironment.env.end()); + auto outputs = buildEnvironment.vars.find("outputs"); + assert(outputs != buildEnvironment.vars.end()); // FIXME: properly unquote 'outputs'. StringMap rewrites; - for (auto & outputName : tokenizeString>(replaceStrings(outputs->second.quoted, "'", ""))) { - auto from = buildEnvironment.env.find(outputName); - assert(from != buildEnvironment.env.end()); + for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { + auto from = buildEnvironment.vars.find(outputName); + assert(from != buildEnvironment.vars.end()); // FIXME: unquote - rewrites.insert({from->second.quoted, outputsDir + "/" + outputName}); + rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName}); } /* Substitute redirects. */ @@ -321,7 +313,7 @@ struct Common : InstallableCommand, MixProfile updateProfile(shellOutPath); - return {readEnvironment(strPath), strPath}; + return {BuildEnvironment::fromJSON(strPath), strPath}; } }; -- cgit v1.2.3 From 5f6375a816803d17c8ec0f7c2ef960c848be6166 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Jul 2021 01:02:47 +0200 Subject: nix develop: Filter some bash magic variables --- src/nix/develop.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'src/nix/develop.cc') diff --git a/src/nix/develop.cc b/src/nix/develop.cc index e00f0d575..9b2945ba3 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -72,7 +72,7 @@ struct BuildEnvironment void toBash(std::ostream & out, const std::set & ignoreVars) const { for (auto & [name, value] : vars) { - if (!ignoreVars.count(name) && !hasPrefix(name, "BASH_")) { + if (!ignoreVars.count(name)) { if (auto str = std::get_if(&value)) { out << fmt("%s=%s\n", name, shellEscape(str->value)); if (str->exported) @@ -191,17 +191,13 @@ struct Common : InstallableCommand, MixProfile { std::set ignoreVars{ "BASHOPTS", - "EUID", "HOME", // FIXME: don't ignore in pure mode? - "HOSTNAME", "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "NIX_LOG_FD", "NIX_REMOTE", "PPID", - "PWD", "SHELLOPTS", - "SHLVL", "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", "TEMPDIR", -- cgit v1.2.3 From 86fb01c4bee0c4f451504c0f6fd2066d2adc8fc2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Jul 2021 12:10:48 +0200 Subject: nix print-dev-env: Add --json flag --- src/nix/develop.cc | 59 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 52 insertions(+), 7 deletions(-) (limited to 'src/nix/develop.cc') diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 9b2945ba3..ca04f297d 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -31,6 +31,11 @@ struct BuildEnvironment { bool exported; std::string value; + + bool operator == (const String & other) const + { + return exported == other.exported && value == other.value; + } }; using Array = std::vector; @@ -42,15 +47,13 @@ struct BuildEnvironment std::map vars; std::map bashFunctions; - static BuildEnvironment fromJSON(const Path & path) + static BuildEnvironment fromJSON(std::string_view in) { BuildEnvironment res; std::set exported; - debug("reading environment file '%s'", path); - - auto json = nlohmann::json::parse(readFile(path)); + auto json = nlohmann::json::parse(in); for (auto & [name, info] : json["variables"].items()) { std::string type = info["type"]; @@ -69,6 +72,38 @@ struct BuildEnvironment return res; } + std::string toJSON() const + { + auto res = nlohmann::json::object(); + + auto vars2 = nlohmann::json::object(); + for (auto & [name, value] : vars) { + auto info = nlohmann::json::object(); + if (auto str = std::get_if(&value)) { + info["type"] = str->exported ? "exported" : "var"; + info["value"] = str->value; + } + else if (auto arr = std::get_if(&value)) { + info["type"] = "array"; + info["value"] = *arr; + } + else if (auto arr = std::get_if(&value)) { + info["type"] = "associative"; + info["value"] = *arr; + } + vars2[name] = std::move(info); + } + res["variables"] = std::move(vars2); + + res["bashFunctions"] = bashFunctions; + + auto json = res.dump(); + + assert(BuildEnvironment::fromJSON(json) == *this); + + return json; + } + void toBash(std::ostream & out, const std::set & ignoreVars) const { for (auto & [name, value] : vars) { @@ -116,6 +151,11 @@ struct BuildEnvironment else throw Error("bash variable is not a string or array"); } + + bool operator == (const BuildEnvironment & other) const + { + return vars == other.vars && bashFunctions == other.bashFunctions; + } }; const static std::string getEnvSh = @@ -309,7 +349,9 @@ struct Common : InstallableCommand, MixProfile updateProfile(shellOutPath); - return {BuildEnvironment::fromJSON(strPath), strPath}; + debug("reading environment file '%s'", strPath); + + return {BuildEnvironment::fromJSON(readFile(strPath)), strPath}; } }; @@ -462,7 +504,7 @@ struct CmdDevelop : Common, MixEnvironment } }; -struct CmdPrintDevEnv : Common +struct CmdPrintDevEnv : Common, MixJSON { std::string description() override { @@ -484,7 +526,10 @@ struct CmdPrintDevEnv : Common stopProgressBar(); - std::cout << makeRcScript(store, buildEnvironment); + logger->writeToStdout( + json + ? buildEnvironment.toJSON() + : makeRcScript(store, buildEnvironment)); } }; -- cgit v1.2.3