Merge branch 'master' into debug-merge

1 files changed, 193 insertions, 130 deletions

diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 498a7b45c..5aad53919 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -7,8 +7,10 @@
 #include "derivations.hh"
 #include "affinity.hh"
 #include "progress-bar.hh"
+#include "run.hh"
 
-#include <regex>
+#include <memory>
+#include <nlohmann/json.hpp>
 
 using namespace nix;
 
@@ -25,94 +27,142 @@ static DevelopSettings developSettings;
 
 static GlobalConfig::Register rDevelopSettings(&developSettings);
 
-struct Var
-{
-    bool exported = true;
-    bool associative = false;
-    std::string quoted; // quoted string or array
-};
-
 struct BuildEnvironment
 {
-    std::map<std::string, Var> env;
-    std::string bashFunctions;
-};
+    struct String
+    {
+        bool exported;
+        std::string value;
 
-BuildEnvironment readEnvironment(const Path & path)
-{
-    BuildEnvironment res;
+        bool operator == (const String & other) const
+        {
+            return exported == other.exported && value == other.value;
+        }
+    };
 
-    std::set<std::string> exported;
+    using Array = std::vector<std::string>;
 
-    debug("reading environment file '%s'", path);
+    using Associative = std::map<std::string, std::string>;
 
-    auto file = readFile(path);
+    using Value = std::variant<String, Array, Associative>;
 
-    auto pos = file.cbegin();
+    std::map<std::string, Value> vars;
+    std::map<std::string, std::string> bashFunctions;
 
-    static std::string varNameRegex =
-        R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";
+    static BuildEnvironment fromJSON(std::string_view in)
+    {
+        BuildEnvironment res;
 
-    static std::string simpleStringRegex =
-        R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re";
+        std::set<std::string> exported;
 
-    static std::string dquotedStringRegex =
-        R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re";
+        auto json = nlohmann::json::parse(in);
 
-    static std::string squotedStringRegex =
-        R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re";
+        for (auto & [name, info] : json["variables"].items()) {
+            std::string type = info["type"];
+            if (type == "var" || type == "exported")
+                res.vars.insert({name, BuildEnvironment::String { .exported = type == "exported", .value = info["value"] }});
+            else if (type == "array")
+                res.vars.insert({name, (Array) info["value"]});
+            else if (type == "associative")
+                res.vars.insert({name, (Associative) info["value"]});
+        }
 
-    static std::string indexedArrayRegex =
-        R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
+        for (auto & [name, def] : json["bashFunctions"].items()) {
+            res.bashFunctions.insert({name, def});
+        }
 
-    static std::regex declareRegex(
-        "^declare -a?x (" + varNameRegex + ")(=(" +
-        dquotedStringRegex + "|" + indexedArrayRegex + "))?\n");
+        return res;
+    }
 
-    static std::regex varRegex(
-        "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n");
+    std::string toJSON() const
+    {
+        auto res = nlohmann::json::object();
+
+        auto vars2 = nlohmann::json::object();
+        for (auto & [name, value] : vars) {
+            auto info = nlohmann::json::object();
+            if (auto str = std::get_if<String>(&value)) {
+                info["type"] = str->exported ? "exported" : "var";
+                info["value"] = str->value;
+            }
+            else if (auto arr = std::get_if<Array>(&value)) {
+                info["type"] = "array";
+                info["value"] = *arr;
+            }
+            else if (auto arr = std::get_if<Associative>(&value)) {
+                info["type"] = "associative";
+                info["value"] = *arr;
+            }
+            vars2[name] = std::move(info);
+        }
+        res["variables"] = std::move(vars2);
 
-    /* Note: we distinguish between an indexed and associative array
-       using the space before the closing parenthesis. Will
-       undoubtedly regret this some day. */
-    static std::regex assocArrayRegex(
-        "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n");
+        res["bashFunctions"] = bashFunctions;
 
-    static std::regex functionRegex(
-        "^" + varNameRegex + " \\(\\) *\n");
+        auto json = res.dump();
 
-    while (pos != file.end()) {
+        assert(BuildEnvironment::fromJSON(json) == *this);
 
-        std::smatch match;
+        return json;
+    }
 
-        if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            exported.insert(match[1]);
+    void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const
+    {
+        for (auto & [name, value] : vars) {
+            if (!ignoreVars.count(name)) {
+                if (auto str = std::get_if<String>(&value)) {
+                    out << fmt("%s=%s\n", name, shellEscape(str->value));
+                    if (str->exported)
+                        out << fmt("export %s\n", name);
+                }
+                else if (auto arr = std::get_if<Array>(&value)) {
+                    out << "declare -a " << name << "=(";
+                    for (auto & s : *arr)
+                        out << shellEscape(s) << " ";
+                    out << ")\n";
+                }
+                else if (auto arr = std::get_if<Associative>(&value)) {
+                    out << "declare -A " << name << "=(";
+                    for (auto & [n, v] : *arr)
+                        out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " ";
+                    out << ")\n";
+                }
+            }
         }
 
-        else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }});
+        for (auto & [name, def] : bashFunctions) {
+            out << name << " ()\n{\n" << def << "}\n";
         }
+    }
 
-        else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
-            pos = match[0].second;
-            res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }});
-        }
+    static std::string getString(const Value & value)
+    {
+        if (auto str = std::get_if<String>(&value))
+            return str->value;
+        else
+            throw Error("bash variable is not a string");
+    }
 
-        else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
-            res.bashFunctions = std::string(pos, file.cend());
-            break;
+    static Array getStrings(const Value & value)
+    {
+        if (auto str = std::get_if<String>(&value))
+            return tokenizeString<Array>(str->value);
+        else if (auto arr = std::get_if<Array>(&value)) {
+            return *arr;
+        } else if (auto assoc = std::get_if<Associative>(&value)) {
+            Array assocKeys;
+            std::for_each(assoc->begin(), assoc->end(), [&](auto & n) { assocKeys.push_back(n.first); });
+            return assocKeys;
         }
-
-        else throw Error("shell environment '%s' has unexpected line '%s'",
-            path, file.substr(pos - file.cbegin(), 60));
+        else
+            throw Error("bash variable is not a string or array");
     }
 
-    res.env.erase("__output");
-
-    return res;
-}
+    bool operator == (const BuildEnvironment & other) const
+    {
+        return vars == other.vars && bashFunctions == other.bashFunctions;
+    }
+};
 
 const static std::string getEnvSh =
     #include "get-env.sh.gen.hh"
@@ -123,15 +173,15 @@ const static std::string getEnvSh =
    modified derivation with the same dependencies and nearly the same
    initial environment variables, that just writes the resulting
    environment to a file and exits. */
-StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
+static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore, const StorePath & drvPath)
 {
-    auto drv = store->derivationFromPath(drvPath);
+    auto drv = evalStore->derivationFromPath(drvPath);
 
     auto builder = baseNameOf(drv.builder);
     if (builder != "bash")
         throw Error("'nix develop' only works on derivations that use 'bash' as their builder");
 
-    auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {});
+    auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {});
 
     drv.args = {store->printStorePath(getEnvShPath)};
 
@@ -144,26 +194,34 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
     drv.name += "-env";
-    for (auto & output : drv.outputs) {
-        output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
-        drv.env[output.first] = "";
-    }
     drv.inputSrcs.insert(std::move(getEnvShPath));
-    Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
+    if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
+        for (auto & output : drv.outputs) {
+            output.second = {
+                .output = DerivationOutputDeferred{},
+            };
+            drv.env[output.first] = hashPlaceholder(output.first);
+        }
+    } else {
+        for (auto & output : drv.outputs) {
+            output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
+            drv.env[output.first] = "";
+        }
+        Hash h = std::get<0>(hashDerivationModulo(*evalStore, drv, true));
 
-    for (auto & output : drv.outputs) {
-        auto outPath = store->makeOutputPath(output.first, h, drv.name);
-        output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
-        drv.env[output.first] = store->printStorePath(outPath);
+        for (auto & output : drv.outputs) {
+            auto outPath = store->makeOutputPath(output.first, h, drv.name);
+            output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
+            drv.env[output.first] = store->printStorePath(outPath);
+        }
     }
 
-    auto shellDrvPath = writeDerivation(*store, drv);
+    auto shellDrvPath = writeDerivation(*evalStore, drv);
 
     /* Build the derivation. */
-    store->buildPaths({DerivedPath::Built{shellDrvPath}});
+    store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore);
 
-    for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
-        auto & [_1, optPath] = outputAndOptPath;
+    for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) {
         assert(optPath);
         auto & outPath = *optPath;
         assert(store->isValidPath(outPath));
@@ -177,18 +235,15 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
 
 struct Common : InstallableCommand, MixProfile
 {
-    std::set<string> ignoreVars{
+    std::set<std::string> ignoreVars{
         "BASHOPTS",
-        "EUID",
         "HOME", // FIXME: don't ignore in pure mode?
-        "HOSTNAME",
         "NIX_BUILD_TOP",
         "NIX_ENFORCE_PURITY",
         "NIX_LOG_FD",
+        "NIX_REMOTE",
         "PPID",
-        "PWD",
         "SHELLOPTS",
-        "SHLVL",
         "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt
         "TEMP",
         "TEMPDIR",
@@ -224,22 +279,10 @@ struct Common : InstallableCommand, MixProfile
 
         out << "nix_saved_PATH=\"$PATH\"\n";
 
-        for (auto & i : buildEnvironment.env) {
-            if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
-                if (i.second.associative)
-                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted);
-                else {
-                    out << fmt("%s=%s\n", i.first, i.second.quoted);
-                    if (i.second.exported)
-                        out << fmt("export %s\n", i.first);
-                }
-            }
-        }
+        buildEnvironment.toBash(out, ignoreVars);
 
         out << "PATH=\"$PATH:$nix_saved_PATH\"\n";
 
-        out << buildEnvironment.bashFunctions << "\n";
-
         out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n";
         for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
             out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
@@ -249,25 +292,25 @@ struct Common : InstallableCommand, MixProfile
         auto script = out.str();
 
         /* Substitute occurrences of output paths. */
-        auto outputs = buildEnvironment.env.find("outputs");
-        assert(outputs != buildEnvironment.env.end());
+        auto outputs = buildEnvironment.vars.find("outputs");
+        assert(outputs != buildEnvironment.vars.end());
 
         // FIXME: properly unquote 'outputs'.
         StringMap rewrites;
-        for (auto & outputName : tokenizeString<std::vector<std::string>>(replaceStrings(outputs->second.quoted, "'", ""))) {
-            auto from = buildEnvironment.env.find(outputName);
-            assert(from != buildEnvironment.env.end());
+        for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) {
+            auto from = buildEnvironment.vars.find(outputName);
+            assert(from != buildEnvironment.vars.end());
             // FIXME: unquote
-            rewrites.insert({from->second.quoted, outputsDir + "/" + outputName});
+            rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName});
         }
 
         /* Substitute redirects. */
         for (auto & [installable_, dir_] : redirects) {
             auto dir = absPath(dir_);
             auto installable = parseInstallable(store, installable_);
-            auto buildable = installable->toDerivedPathWithHints();
-            auto doRedirect = [&](const StorePath & path)
-            {
+            auto builtPaths = toStorePaths(
+                getEvalStore(), store, Realise::Nothing, OperateOn::Output, {installable});
+            for (auto & path: builtPaths) {
                 auto from = store->printStorePath(path);
                 if (script.find(from) == std::string::npos)
                     warn("'%s' (path '%s') is not used by this build environment", installable->what(), from);
@@ -275,16 +318,7 @@ struct Common : InstallableCommand, MixProfile
                     printInfo("redirecting '%s' to '%s'", from, dir);
                     rewrites.insert({from, dir});
                 }
-            };
-            std::visit(overloaded {
-                [&](const DerivedPathWithHints::Opaque & bo) {
-                    doRedirect(bo.path);
-                },
-                [&](const DerivedPathWithHints::Built & bfd) {
-                    for (auto & [outputName, path] : bfd.outputs)
-                        if (path) doRedirect(*path);
-                },
-            }, buildable.raw());
+            }
         }
 
         return rewriteStrings(script, rewrites);
@@ -294,6 +328,12 @@ struct Common : InstallableCommand, MixProfile
     {
         return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()};
     }
+    Strings getDefaultFlakeAttrPathPrefixes() override
+    {
+        auto res = SourceExprCommand::getDefaultFlakeAttrPathPrefixes();
+        res.emplace_front("devShells." + settings.thisSystem.get() + ".");
+        return res;
+    }
 
     StorePath getShellOutPath(ref<Store> store)
     {
@@ -309,7 +349,7 @@ struct Common : InstallableCommand, MixProfile
 
             auto & drvPath = *drvs.begin();
 
-            return getDerivationEnvironment(store, drvPath);
+            return getDerivationEnvironment(store, getEvalStore(), drvPath);
         }
     }
 
@@ -321,7 +361,9 @@ struct Common : InstallableCommand, MixProfile
 
         updateProfile(shellOutPath);
 
-        return {readEnvironment(strPath), strPath};
+        debug("reading environment file '%s'", strPath);
+
+        return {BuildEnvironment::fromJSON(readFile(store->toRealPath(shellOutPath))), strPath};
     }
 };
 
@@ -351,6 +393,12 @@ struct CmdDevelop : Common, MixEnvironment
         });
 
         addFlag({
+            .longName = "unpack",
+            .description = "Run the `unpack` phase.",
+            .handler = {&phase, {"unpack"}},
+        });
+
+        addFlag({
             .longName = "configure",
             .description = "Run the `configure` phase.",
             .handler = {&phase, {"configure"}},
@@ -404,7 +452,7 @@ struct CmdDevelop : Common, MixEnvironment
         if (verbosity >= lvlDebug)
             script += "set -x\n";
 
-        script += fmt("rm -f '%s'\n", rcFilePath);
+        script += fmt("command rm -f '%s'\n", rcFilePath);
 
         if (phase) {
             if (!command.empty())
@@ -423,7 +471,7 @@ struct CmdDevelop : Common, MixEnvironment
         }
 
         else {
-            script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n";
+            script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n" + script;
             if (developSettings.bashPrompt != "")
                 script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt));
             if (developSettings.bashPromptSuffix != "")
@@ -432,8 +480,6 @@ struct CmdDevelop : Common, MixEnvironment
 
         writeFull(rcFileFd.get(), script);
 
-        stopProgressBar();
-
         setEnviron();
         // prevent garbage collection until shell exits
         setenv("NIX_GCROOT", gcroot.data(), 1);
@@ -443,16 +489,20 @@ struct CmdDevelop : Common, MixEnvironment
         try {
             auto state = getEvalState();
 
+            auto nixpkgsLockFlags = lockFlags;
+            nixpkgsLockFlags.inputOverrides = {};
+            nixpkgsLockFlags.inputUpdates = {};
+
             auto bashInstallable = std::make_shared<InstallableFlake>(
                 this,
                 state,
                 installable->nixpkgsFlakeRef(),
                 Strings{"bashInteractive"},
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
-                lockFlags);
+                nixpkgsLockFlags);
 
-            shell = state->store->printStorePath(
-                toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash";
+            shell = store->printStorePath(
+                toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash";
         } catch (Error &) {
             ignoreException();
         }
@@ -462,15 +512,25 @@ struct CmdDevelop : Common, MixEnvironment
         auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath}
             : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
 
-        restoreProcessContext();
-
-        execvp(shell.c_str(), stringsToCharPtrs(args).data());
+        // Need to chdir since phases assume in flake directory
+        if (phase) {
+            // chdir if installable is a flake of type git+file or path
+            auto installableFlake = std::dynamic_pointer_cast<InstallableFlake>(installable);
+            if (installableFlake) {
+                auto sourcePath = installableFlake->getLockedFlake()->flake.resolvedRef.input.getSourcePath();
+                if (sourcePath) {
+                    if (chdir(sourcePath->c_str()) == -1) {
+                        throw SysError("chdir to '%s' failed", *sourcePath);
+                    }
+                }
+            }
+        }
 
-        throw SysError("executing shell '%s'", shell);
+        runProgramInStore(store, shell, args);
     }
 };
 
-struct CmdPrintDevEnv : Common
+struct CmdPrintDevEnv : Common, MixJSON
 {
     std::string description() override
     {
@@ -492,7 +552,10 @@ struct CmdPrintDevEnv : Common
 
         stopProgressBar();
 
-        std::cout << makeRcScript(store, buildEnvironment);
+        logger->writeToStdout(
+            json
+            ? buildEnvironment.toJSON()
+            : makeRcScript(store, buildEnvironment));
     }
 };