Merge branch 'master' into structured-attrs-shell

 Conflicts:
        src/nix/develop.cc
        src/nix/get-env.sh
        tests/shell.nix

1 files changed, 150 insertions, 109 deletions

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index ee782c4ec..26f53db09 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -8,7 +8,7 @@
 #include "affinity.hh"
 #include "progress-bar.hh"
 
-#include <regex>
+#include <nlohmann/json.hpp>
 
 using namespace nix;
 
@@ -25,94 +25,142 @@ static DevelopSettings developSettings;
 
 static GlobalConfig::Register rDevelopSettings(&developSettings);
 
-struct Var
-{
-    bool exported = true;
-    bool associative = false;
-    std::string quoted; // quoted string or array
-};
-
 struct BuildEnvironment
 {
-    std::map<std::string, Var> env;
-    std::string bashFunctions;
-};
+    struct String
+    {
+        bool exported;
+        std::string value;
 
-BuildEnvironment readEnvironment(const Path & path)
-{
-    BuildEnvironment res;
+        bool operator == (const String & other) const
+        {
+            return exported == other.exported && value == other.value;
+        }
+    };
 
-    std::set<std::string> exported;
+    using Array = std::vector<std::string>;
 
-    debug("reading environment file '%s'", path);
+    using Associative = std::map<std::string, std::string>;
 
-    auto file = readFile(path);
+    using Value = std::variant<String, Array, Associative>;
 
-    auto pos = file.cbegin();
+    std::map<std::string, Value> vars;
+    std::map<std::string, std::string> bashFunctions;
 
-    static std::string varNameRegex =
-        R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";
+    static BuildEnvironment fromJSON(std::string_view in)
+    {
+        BuildEnvironment res;
 
-    static std::string simpleStringRegex =
-        R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re";
+        std::set<std::string> exported;
 
-    static std::string dquotedStringRegex =
-        R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re";
+        auto json = nlohmann::json::parse(in);
 
-    static std::string squotedStringRegex =
-        R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re";
+        for (auto & [name, info] : json["variables"].items()) {
+            std::string type = info["type"];
+            if (type == "var" || type == "exported")
+                res.vars.insert({name, BuildEnvironment::String { .exported = type == "exported", .value = info["value"] }});
+            else if (type == "array")
+                res.vars.insert({name, (Array) info["value"]});
+            else if (type == "associative")
+                res.vars.insert({name, (Associative) info["value"]});
+        }
 
-    static std::string indexedArrayRegex =
-        R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
+        for (auto & [name, def] : json["bashFunctions"].items()) {
+            res.bashFunctions.insert({name, def});
+        }
 
-    static std::regex declareRegex(
-        "^declare -a?x (" + varNameRegex + ")(=(" +
-        dquotedStringRegex + "|" + indexedArrayRegex + "))?\n");
+        return res;
+    }
 
-    static std::regex varRegex(
-        "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n");
+    std::string toJSON() const
+    {
+        auto res = nlohmann::json::object();
+
+        auto vars2 = nlohmann::json::object();
+        for (auto & [name, value] : vars) {
+            auto info = nlohmann::json::object();
+            if (auto str = std::get_if<String>(&value)) {
+                info["type"] = str->exported ? "exported" : "var";
+                info["value"] = str->value;
+            }
+            else if (auto arr = std::get_if<Array>(&value)) {
+                info["type"] = "array";
+                info["value"] = *arr;
+            }
+            else if (auto arr = std::get_if<Associative>(&value)) {
+                info["type"] = "associative";
+                info["value"] = *arr;
+            }
+            vars2[name] = std::move(info);
+        }
+        res["variables"] = std::move(vars2);
 
-    /* Note: we distinguish between an indexed and associative array
-       using the space before the closing parenthesis. Will
-       undoubtedly regret this some day. */
-    static std::regex assocArrayRegex(
-        "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n");
+        res["bashFunctions"] = bashFunctions;
 
-    static std::regex functionRegex(
-        "^" + varNameRegex + " \\(\\) *\n");
+        auto json = res.dump();
 
-    while (pos != file.end()) {
+        assert(BuildEnvironment::fromJSON(json) == *this);
 
-        std::smatch match;
+        return json;
+    }
 
-        if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            exported.insert(match[1]);
+    void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const
+    {
+        for (auto & [name, value] : vars) {
+            if (!ignoreVars.count(name)) {
+                if (auto str = std::get_if<String>(&value)) {
+                    out << fmt("%s=%s\n", name, shellEscape(str->value));
+                    if (str->exported)
+                        out << fmt("export %s\n", name);
+                }
+                else if (auto arr = std::get_if<Array>(&value)) {
+                    out << "declare -a " << name << "=(";
+                    for (auto & s : *arr)
+                        out << shellEscape(s) << " ";
+                    out << ")\n";
+                }
+                else if (auto arr = std::get_if<Associative>(&value)) {
+                    out << "declare -A " << name << "=(";
+                    for (auto & [n, v] : *arr)
+                        out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " ";
+                    out << ")\n";
+                }
+            }
         }
 
-        else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }});
+        for (auto & [name, def] : bashFunctions) {
+            out << name << " ()\n{\n" << def << "}\n";
         }
+    }
 
-        else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }});
-        }
+    static std::string getString(const Value & value)
+    {
+        if (auto str = std::get_if<String>(&value))
+            return str->value;
+        else
+            throw Error("bash variable is not a string");
+    }
 
-        else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
-            res.bashFunctions = std::string(pos, file.cend());
-            break;
+    static Array getStrings(const Value & value)
+    {
+        if (auto str = std::get_if<String>(&value))
+            return tokenizeString<Array>(str->value);
+        else if (auto arr = std::get_if<Array>(&value)) {
+            return *arr;
+        } else if (auto assoc = std::get_if<Associative>(&value)) {
+            Array assocKeys;
+            std::for_each(assoc->begin(), assoc->end(), [&](auto & n) { assocKeys.push_back(n.first); });
+            return assocKeys;
         }
-
-        else throw Error("shell environment '%s' has unexpected line '%s'",
-            path, file.substr(pos - file.cbegin(), 60));
+        else
+            throw Error("bash variable is not a string or array");
     }
 
-    res.env.erase("__output");
-
-    return res;
-}
+    bool operator == (const BuildEnvironment & other) const
+    {
+        return vars == other.vars && bashFunctions == other.bashFunctions;
+    }
+};
 
 const static std::string getEnvSh =
     #include "get-env.sh.gen.hh"
@@ -144,17 +192,26 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
     drv.name += "-env";
-    for (auto & output : drv.outputs) {
-        output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
-        drv.env[output.first] = "";
-    }
     drv.inputSrcs.insert(std::move(getEnvShPath));
-    Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
+    if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+        for (auto & output : drv.outputs) {
+            output.second = {
+                .output = DerivationOutputDeferred{},
+            };
+            drv.env[output.first] = hashPlaceholder(output.first);
+        }
+    } else {
+        for (auto & output : drv.outputs) {
+            output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
+            drv.env[output.first] = "";
+        }
+        Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
 
-    for (auto & output : drv.outputs) {
-        auto outPath = store->makeOutputPath(output.first, h, drv.name);
-        output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
-        drv.env[output.first] = store->printStorePath(outPath);
+        for (auto & output : drv.outputs) {
+            auto outPath = store->makeOutputPath(output.first, h, drv.name);
+            output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
+            drv.env[output.first] = store->printStorePath(outPath);
+        }
     }
 
     auto shellDrvPath = writeDerivation(*store, drv);
@@ -162,8 +219,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Build the derivation. */
     store->buildPaths({DerivedPath::Built{shellDrvPath}});
 
-    for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
-        auto & [_1, optPath] = outputAndOptPath;
+    for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) {
         assert(optPath);
         auto & outPath = *optPath;
         assert(store->isValidPath(outPath));
@@ -177,19 +233,15 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
 
 struct Common : InstallableCommand, MixProfile
 {
-    std::set<string> ignoreVars{
+    std::set<std::string> ignoreVars{
         "BASHOPTS",
-        "EUID",
         "HOME", // FIXME: don't ignore in pure mode?
-        "HOSTNAME",
         "NIX_BUILD_TOP",
         "NIX_ENFORCE_PURITY",
         "NIX_LOG_FD",
         "NIX_REMOTE",
         "PPID",
-        "PWD",
         "SHELLOPTS",
-        "SHLVL",
         "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt
         "TEMP",
         "TEMPDIR",
@@ -225,22 +277,10 @@ struct Common : InstallableCommand, MixProfile
 
         out << "nix_saved_PATH=\"$PATH\"\n";
 
-        for (auto & i : buildEnvironment.env) {
-            if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
-                if (i.second.associative)
-                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted);
-                else {
-                    out << fmt("%s=%s\n", i.first, i.second.quoted);
-                    if (i.second.exported)
-                        out << fmt("export %s\n", i.first);
-                }
-            }
-        }
+        buildEnvironment.toBash(out, ignoreVars);
 
         out << "PATH=\"$PATH:$nix_saved_PATH\"\n";
 
-        out << buildEnvironment.bashFunctions << "\n";
-
         out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n";
         for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
             out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
@@ -250,24 +290,16 @@ struct Common : InstallableCommand, MixProfile
         auto script = out.str();
 
         /* Substitute occurrences of output paths. */
-        auto outputs = buildEnvironment.env.find("outputs");
-        assert(outputs != buildEnvironment.env.end());
+        auto outputs = buildEnvironment.vars.find("outputs");
+        assert(outputs != buildEnvironment.vars.end());
 
         // FIXME: properly unquote 'outputs'.
         StringMap rewrites;
-        for (auto & outputName : tokenizeString<std::vector<std::string>>(replaceStrings(outputs->second.quoted, "'", ""))) {
-            // Hacky way to obtain the key of an associate array. This is needed for strctured attrs where
-            // `outputs` is an associative array. If the regex isn't matched, the non-structured-attrs behavior will
-            // be used.
-            std::regex ptrn(R"re(\[([A-z0-9]+)\]=.*)re");
-            std::smatch match;
-            if (std::regex_match(outputName, match, ptrn)) {
-                outputName = match[1];
-            }
-            auto from = buildEnvironment.env.find(outputName);
-            assert(from != buildEnvironment.env.end());
+        for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) {
+            auto from = buildEnvironment.vars.find(outputName);
+            assert(from != buildEnvironment.vars.end());
             // FIXME: unquote
-            rewrites.insert({from->second.quoted, outputsDir + "/" + outputName});
+            rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName});
         }
 
         /* Substitute redirects. */
@@ -321,7 +353,9 @@ struct Common : InstallableCommand, MixProfile
 
         updateProfile(shellOutPath);
 
-        return {readEnvironment(strPath), strPath};
+        debug("reading environment file '%s'", strPath);
+
+        return {BuildEnvironment::fromJSON(readFile(strPath)), strPath};
     }
 };
 
@@ -443,13 +477,17 @@ struct CmdDevelop : Common, MixEnvironment
         try {
             auto state = getEvalState();
 
+            auto nixpkgsLockFlags = lockFlags;
+            nixpkgsLockFlags.inputOverrides = {};
+            nixpkgsLockFlags.inputUpdates = {};
+
             auto bashInstallable = std::make_shared<InstallableFlake>(
                 this,
                 state,
                 installable->nixpkgsFlakeRef(),
                 Strings{"bashInteractive"},
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
-                lockFlags);
+                nixpkgsLockFlags);
 
             shell = state->store->printStorePath(
                 toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash";
@@ -470,7 +508,7 @@ struct CmdDevelop : Common, MixEnvironment
     }
 };
 
-struct CmdPrintDevEnv : Common
+struct CmdPrintDevEnv : Common, MixJSON
 {
     std::string description() override
     {
@@ -492,7 +530,10 @@ struct CmdPrintDevEnv : Common
 
         stopProgressBar();
 
-        std::cout << makeRcScript(store, buildEnvironment);
+        logger->writeToStdout(
+            json
+            ? buildEnvironment.toJSON()
+            : makeRcScript(store, buildEnvironment));
     }
 };