Merge remote-tracking branch 'upstream/master' into fix-and-ci-static-builds

23 files changed, 329 insertions, 134 deletions

diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index eaec63349..df55d1bc4 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -36,6 +36,14 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         return "add a path to the Nix store";
     }
 
+    std::string doc() override
+    {
+        return R"(
+          Copy the file or directory *path* to the Nix store, and
+          print the resulting store path on standard output.
+        )";
+    }
+
     Examples examples() override
     {
         return {
@@ -60,8 +68,10 @@ struct CmdAddToStore : MixDryRun, StoreCommand
             hash = hsink.finish().first;
         }
 
-        ValidPathInfo info(store->makeFixedOutputPath(ingestionMethod, hash, *namePart));
-        info.narHash = narHash;
+        ValidPathInfo info {
+            store->makeFixedOutputPath(ingestionMethod, hash, *namePart),
+            narHash,
+        };
         info.narSize = sink.s->size();
         info.ca = std::optional { FixedOutputHash {
             .method = ingestionMethod,
diff --git a/src/nix/app.cc b/src/nix/app.cc
index 3935297cf..80acbf658 100644
--- a/src/nix/app.cc
+++ b/src/nix/app.cc
@@ -8,7 +8,7 @@ namespace nix {
 
 App Installable::toApp(EvalState & state)
 {
-    auto [cursor, attrPath] = getCursor(state, true);
+    auto [cursor, attrPath] = getCursor(state);
 
     auto type = cursor->getAttr("type")->getString();
 
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 13d14a7fb..75a42ac55 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -71,14 +71,14 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
                         [&](BuildableOpaque bo) {
                             std::string symlink = outLink;
                             if (i) symlink += fmt("-%d", i);
-                            store2->addPermRoot(bo.path, absPath(symlink), true);
+                            store2->addPermRoot(bo.path, absPath(symlink));
                         },
                         [&](BuildableFromDrv bfd) {
                             for (auto & output : bfd.outputs) {
                                 std::string symlink = outLink;
                                 if (i) symlink += fmt("-%d", i);
                                 if (output.first != "out") symlink += fmt("-%s", output.first);
-                                store2->addPermRoot(output.second, absPath(symlink), true);
+                                store2->addPermRoot(output.second, absPath(symlink));
                             }
                         },
                     }, buildables[i]);
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index eb3339f5d..241c8699b 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -122,7 +122,7 @@ struct CmdBundle : InstallableCommand
         if (!outLink)
             outLink = baseNameOf(app.program);
 
-        store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink), true);
+        store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink));
     }
 };
 
diff --git a/src/nix/command.cc b/src/nix/command.cc
index da32819da..efac230bd 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -4,12 +4,25 @@
 #include "nixexpr.hh"
 #include "profiles.hh"
 
+#include <nlohmann/json.hpp>
+
 extern char * * environ __attribute__((weak));
 
 namespace nix {
 
 Commands * RegisterCommand::commands = nullptr;
 
+void NixMultiCommand::printHelp(const string & programName, std::ostream & out)
+{
+    MultiCommand::printHelp(programName, out);
+}
+
+nlohmann::json NixMultiCommand::toJSON()
+{
+    // FIXME: use Command::toJSON() as well.
+    return MultiCommand::toJSON();
+}
+
 StoreCommand::StoreCommand()
 {
 }
@@ -121,7 +134,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
     switchLink(profile2,
         createGeneration(
             ref<LocalFSStore>(store),
-            profile2, store->printStorePath(storePath)));
+            profile2, storePath));
 }
 
 void MixProfile::updateProfile(const Buildables & buildables)
diff --git a/src/nix/command.hh b/src/nix/command.hh
index bc46a2028..d60c8aeb6 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -21,6 +21,13 @@ static constexpr Command::Category catSecondary = 100;
 static constexpr Command::Category catUtility = 101;
 static constexpr Command::Category catNixInstallation = 102;
 
+struct NixMultiCommand : virtual MultiCommand, virtual Command
+{
+    void printHelp(const string & programName, std::ostream & out) override;
+
+    nlohmann::json toJSON() override;
+};
+
 /* A command that requires a Nix store. */
 struct StoreCommand : virtual Command
 {
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 12658078a..516e7bda9 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -15,7 +15,7 @@ struct Var
 {
     bool exported = true;
     bool associative = false;
-    std::string value; // quoted string or array
+    std::string quoted; // quoted string or array
 };
 
 struct BuildEnvironment
@@ -75,12 +75,12 @@ BuildEnvironment readEnvironment(const Path & path)
 
         else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) {
             pos = match[0].second;
-            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }});
+            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }});
         }
 
         else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) {
             pos = match[0].second;
-            res.env.insert({match[1], Var { .associative = true, .value = match[2] }});
+            res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }});
         }
 
         else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) {
@@ -92,6 +92,8 @@ BuildEnvironment readEnvironment(const Path & path)
             path, file.substr(pos - file.cbegin(), 60));
     }
 
+    res.env.erase("__output");
+
     return res;
 }
 
@@ -124,31 +126,33 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
 
     /* Rehash and write the derivation. FIXME: would be nice to use
        'buildDerivation', but that's privileged. */
-    auto drvName = std::string(drvPath.name());
-    assert(hasSuffix(drvName, ".drv"));
-    drvName.resize(drvName.size() - 4);
-    drvName += "-env";
-    for (auto & output : drv.outputs)
-        drv.env.erase(output.first);
-    drv.outputs = {{"out", DerivationOutput { .output = DerivationOutputInputAddressed { .path = StorePath::dummy }}}};
-    drv.env["out"] = "";
-    drv.env["_outputs_saved"] = drv.env["outputs"];
-    drv.env["outputs"] = "out";
+    drv.name += "-env";
+    for (auto & output : drv.outputs) {
+        output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
+        drv.env[output.first] = "";
+    }
     drv.inputSrcs.insert(std::move(getEnvShPath));
     Hash h = std::get<0>(hashDerivationModulo(*store, drv, true));
-    auto shellOutPath = store->makeOutputPath("out", h, drvName);
-    drv.outputs.insert_or_assign("out", DerivationOutput { .output = DerivationOutputInputAddressed {
-                .path = shellOutPath
-            } });
-    drv.env["out"] = store->printStorePath(shellOutPath);
-    auto shellDrvPath2 = writeDerivation(store, drv, drvName);
+
+    for (auto & output : drv.outputs) {
+        auto outPath = store->makeOutputPath(output.first, h, drv.name);
+        output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
+        drv.env[output.first] = store->printStorePath(outPath);
+    }
+
+    auto shellDrvPath = writeDerivation(*store, drv);
 
     /* Build the derivation. */
-    store->buildPaths({{shellDrvPath2}});
+    store->buildPaths({{shellDrvPath}});
 
-    assert(store->isValidPath(shellOutPath));
+    for (auto & outPath : drv.outputPaths(*store)) {
+        assert(store->isValidPath(outPath));
+        auto outPathS = store->toRealPath(outPath);
+        if (lstat(outPathS).st_size)
+            return outPath;
+    }
 
-    return shellOutPath;
+    throw Error("get-env.sh failed to produce an environment");
 }
 
 struct Common : InstallableCommand, MixProfile
@@ -174,8 +178,12 @@ struct Common : InstallableCommand, MixProfile
         "UID",
     };
 
-    void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out)
+    std::string makeRcScript(
+        const BuildEnvironment & buildEnvironment,
+        const Path & outputsDir = absPath(".") + "/outputs")
     {
+        std::ostringstream out;
+
         out << "unset shellHook\n";
 
         out << "nix_saved_PATH=\"$PATH\"\n";
@@ -183,9 +191,9 @@ struct Common : InstallableCommand, MixProfile
         for (auto & i : buildEnvironment.env) {
             if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
                 if (i.second.associative)
-                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.value);
+                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted);
                 else {
-                    out << fmt("%s=%s\n", i.first, i.second.value);
+                    out << fmt("%s=%s\n", i.first, i.second.quoted);
                     if (i.second.exported)
                         out << fmt("export %s\n", i.first);
                 }
@@ -196,13 +204,26 @@ struct Common : InstallableCommand, MixProfile
 
         out << buildEnvironment.bashFunctions << "\n";
 
-        // FIXME: set outputs
-
         out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n";
         for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
             out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
 
         out << "eval \"$shellHook\"\n";
+
+        /* Substitute occurrences of output paths. */
+        auto outputs = buildEnvironment.env.find("outputs");
+        assert(outputs != buildEnvironment.env.end());
+
+        // FIXME: properly unquote 'outputs'.
+        StringMap rewrites;
+        for (auto & outputName : tokenizeString<std::vector<std::string>>(replaceStrings(outputs->second.quoted, "'", ""))) {
+            auto from = buildEnvironment.env.find(outputName);
+            assert(from != buildEnvironment.env.end());
+            // FIXME: unquote
+            rewrites.insert({from->second.quoted, outputsDir + "/" + outputName});
+        }
+
+        return rewriteStrings(out.str(), rewrites);
     }
 
     Strings getDefaultFlakeAttrPaths() override
@@ -243,19 +264,57 @@ struct Common : InstallableCommand, MixProfile
 struct CmdDevelop : Common, MixEnvironment
 {
     std::vector<std::string> command;
+    std::optional<std::string> phase;
 
     CmdDevelop()
     {
         addFlag({
             .longName = "command",
             .shortName = 'c',
-            .description = "command and arguments to be executed insted of an interactive shell",
+            .description = "command and arguments to be executed instead of an interactive shell",
             .labels = {"command", "args"},
             .handler = {[&](std::vector<std::string> ss) {
                 if (ss.empty()) throw UsageError("--command requires at least one argument");
                 command = ss;
             }}
         });
+
+        addFlag({
+            .longName = "phase",
+            .description = "phase to run (e.g. `build` or `configure`)",
+            .labels = {"phase-name"},
+            .handler = {&phase},
+        });
+
+        addFlag({
+            .longName = "configure",
+            .description = "run the configure phase",
+            .handler = {&phase, {"configure"}},
+        });
+
+        addFlag({
+            .longName = "build",
+            .description = "run the build phase",
+            .handler = {&phase, {"build"}},
+        });
+
+        addFlag({
+            .longName = "check",
+            .description = "run the check phase",
+            .handler = {&phase, {"check"}},
+        });
+
+        addFlag({
+            .longName = "install",
+            .description = "run the install phase",
+            .handler = {&phase, {"install"}},
+        });
+
+        addFlag({
+            .longName = "installcheck",
+            .description = "run the installcheck phase",
+            .handler = {&phase, {"installCheck"}},
+        });
     }
 
     std::string description() override
@@ -291,19 +350,31 @@ struct CmdDevelop : Common, MixEnvironment
 
         auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
 
-        std::ostringstream ss;
-        makeRcScript(buildEnvironment, ss);
+        auto script = makeRcScript(buildEnvironment);
+
+        if (verbosity >= lvlDebug)
+            script += "set -x\n";
 
-        ss << fmt("rm -f '%s'\n", rcFilePath);
+        script += fmt("rm -f '%s'\n", rcFilePath);
+
+        if (phase) {
+            if (!command.empty())
+                throw UsageError("you cannot use both '--command' and '--phase'");
+            // FIXME: foundMakefile is set by buildPhase, need to get
+            // rid of that.
+            script += fmt("foundMakefile=1\n");
+            script += fmt("runHook %1%Phase\n", *phase);
+            script += fmt("exit 0\n", *phase);
+        }
 
-        if (!command.empty()) {
+        else if (!command.empty()) {
             std::vector<std::string> args;
             for (auto s : command)
                 args.push_back(shellEscape(s));
-            ss << fmt("exec %s\n", concatStringsSep(" ", args));
+            script += fmt("exec %s\n", concatStringsSep(" ", args));
         }
 
-        writeFull(rcFileFd.get(), ss.str());
+        writeFull(rcFileFd.get(), script);
 
         stopProgressBar();
 
@@ -365,7 +436,7 @@ struct CmdPrintDevEnv : Common
 
         stopProgressBar();
 
-        makeRcScript(buildEnvironment, std::cout);
+        std::cout << makeRcScript(buildEnvironment);
     }
 };
 
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index a90668307..64fc896d9 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -572,7 +572,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
             Strings{templateName == "" ? "defaultTemplate" : templateName},
             Strings(attrsPathPrefixes), lockFlags);
 
-        auto [cursor, attrPath] = installable.getCursor(*evalState, true);
+        auto [cursor, attrPath] = installable.getCursor(*evalState);
 
         auto templateDir = cursor->getAttr("path")->getString();
 
@@ -782,7 +782,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
 struct CmdFlakeShow : FlakeCommand
 {
     bool showLegacy = false;
-    bool useEvalCache = true;
 
     CmdFlakeShow()
     {
@@ -791,12 +790,6 @@ struct CmdFlakeShow : FlakeCommand
             .description = "show the contents of the 'legacyPackages' output",
             .handler = {&showLegacy, true}
         });
-
-        addFlag({
-            .longName = "no-eval-cache",
-            .description = "do not use the flake evaluation cache",
-            .handler = {[&]() { useEvalCache = false; }}
-        });
     }
 
     std::string description() override
@@ -934,13 +927,13 @@ struct CmdFlakeShow : FlakeCommand
             }
         };
 
-        auto cache = openEvalCache(*state, flake, useEvalCache);
+        auto cache = openEvalCache(*state, flake);
 
         visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), "");
     }
 };
 
-struct CmdFlake : virtual MultiCommand, virtual Command
+struct CmdFlake : NixMultiCommand
 {
     CmdFlake()
         : MultiCommand({
@@ -970,11 +963,6 @@ struct CmdFlake : virtual MultiCommand, virtual Command
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto r1 = registerCommand<CmdFlake>("flake");
diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh
index 2e0e83561..091c0f573 100644
--- a/src/nix/get-env.sh
+++ b/src/nix/get-env.sh
@@ -1,12 +1,6 @@
 set -e
 if [ -e .attrs.sh ]; then source .attrs.sh; fi
 
-outputs=$_outputs_saved
-for __output in $_outputs_saved; do
-    declare "$__output"="$out"
-done
-unset _outputs_saved __output
-
 export IN_NIX_SHELL=impure
 export dontAddDisableDepTrack=1
 
@@ -14,5 +8,12 @@ if [[ -n $stdenv ]]; then
     source $stdenv/setup
 fi
 
-export > $out
-set >> $out
+for __output in $outputs; do
+    if [[ -z $__done ]]; then
+        export > ${!__output}
+        set >> ${!__output}
+        __done=1
+    else
+        echo -n >> ${!__output}
+    fi
+done
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 06e61380b..1f1ed680f 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -76,7 +76,7 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "override-input",
-        .description = "override a specific flake input (e.g. 'dwarffs/nixpkgs')",
+        .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)",
         .labels = {"input-path", "flake-url"},
         .handler = {[&](std::string inputPath, std::string flakeRef) {
             lockFlags.inputOverrides.insert_or_assign(
@@ -116,7 +116,7 @@ SourceExprCommand::SourceExprCommand()
     addFlag({
         .longName = "file",
         .shortName = 'f',
-        .description = "evaluate FILE rather than the default",
+        .description = "evaluate *file* rather than the default",
         .labels = {"file"},
         .handler = {&file},
         .completer = completePath
@@ -124,7 +124,7 @@ SourceExprCommand::SourceExprCommand()
 
     addFlag({
         .longName ="expr",
-        .description = "evaluate attributes from EXPR",
+        .description = "evaluate attributes from *expr*",
         .labels = {"expr"},
         .handler = {&expr}
     });
@@ -183,8 +183,7 @@ void completeFlakeRefWithFragment(
             auto flakeRef = parseFlakeRef(flakeRefS, absPath("."));
 
             auto evalCache = openEvalCache(*evalState,
-                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)),
-                true);
+                std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags)));
 
             auto root = evalCache->getRoot();
 
@@ -273,7 +272,7 @@ Buildable Installable::toBuildable()
 }
 
 std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-Installable::getCursors(EvalState & state, bool useEvalCache)
+Installable::getCursors(EvalState & state)
 {
     auto evalCache =
         std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state,
@@ -282,9 +281,9 @@ Installable::getCursors(EvalState & state, bool useEvalCache)
 }
 
 std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
-Installable::getCursor(EvalState & state, bool useEvalCache)
+Installable::getCursor(EvalState & state)
 {
-    auto cursors = getCursors(state, useEvalCache);
+    auto cursors = getCursors(state);
     if (cursors.empty())
         throw Error("cannot find flake attribute '%s'", what());
     return cursors[0];
@@ -305,8 +304,8 @@ struct InstallableStorePath : Installable
         if (storePath.isDerivation()) {
             std::map<std::string, StorePath> outputs;
             auto drv = store->readDerivation(storePath);
-            for (auto & [name, output] : drv.outputs)
-                outputs.emplace(name, output.path(*store, drv.name));
+            for (auto & i : drv.outputsAndPaths(*store))
+                outputs.emplace(i.first, i.second.second);
             return {
                 BuildableFromDrv {
                     .drvPath = storePath,
@@ -420,12 +419,11 @@ Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::Locked
 
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
-    std::shared_ptr<flake::LockedFlake> lockedFlake,
-    bool useEvalCache)
+    std::shared_ptr<flake::LockedFlake> lockedFlake)
 {
     auto fingerprint = lockedFlake->getFingerprint();
     return make_ref<nix::eval_cache::EvalCache>(
-        useEvalCache && evalSettings.pureEval
+        evalSettings.useEvalCache && evalSettings.pureEval
             ? std::optional { std::cref(fingerprint) }
             : std::nullopt,
         state,
@@ -460,10 +458,9 @@ static std::string showAttrPaths(const std::vector<std::string> & paths)
 
 std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
 {
-
     auto lockedFlake = getLockedFlake();
 
-    auto cache = openEvalCache(*state, lockedFlake, true);
+    auto cache = openEvalCache(*state, lockedFlake);
     auto root = cache->getRoot();
 
     for (auto & attrPath : getActualAttrPaths()) {
@@ -517,11 +514,10 @@ std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state)
 }
 
 std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-InstallableFlake::getCursors(EvalState & state, bool useEvalCache)
+InstallableFlake::getCursors(EvalState & state)
 {
     auto evalCache = openEvalCache(state,
-        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)),
-        useEvalCache);
+        std::make_shared<flake::LockedFlake>(lockFlake(state, flakeRef, lockFlags)));
 
     auto root = evalCache->getRoot();
 
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
index 9edff3331..26e87ee3a 100644
--- a/src/nix/installables.hh
+++ b/src/nix/installables.hh
@@ -62,10 +62,10 @@ struct Installable
     }
 
     virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-    getCursors(EvalState & state, bool useEvalCache);
+    getCursors(EvalState & state);
 
     std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>
-    getCursor(EvalState & state, bool useEvalCache);
+    getCursor(EvalState & state);
 
     virtual FlakeRef nixpkgsFlakeRef() const
     {
@@ -118,7 +118,7 @@ struct InstallableFlake : InstallableValue
     std::pair<Value *, Pos> toValue(EvalState & state) override;
 
     std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>>
-    getCursors(EvalState & state, bool useEvalCache) override;
+    getCursors(EvalState & state) override;
 
     std::shared_ptr<flake::LockedFlake> getLockedFlake() const;
 
@@ -127,7 +127,6 @@ struct InstallableFlake : InstallableValue
 
 ref<eval_cache::EvalCache> openEvalCache(
     EvalState & state,
-    std::shared_ptr<flake::LockedFlake> lockedFlake,
-    bool useEvalCache);
+    std::shared_ptr<flake::LockedFlake> lockedFlake);
 
 }
diff --git a/src/nix/local.mk b/src/nix/local.mk
index b057b7cc6..e96200685 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -19,7 +19,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr
 
 nix_LIBS = libexpr libmain libfetchers libstore libutil
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system -llowdown
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
diff --git a/src/nix/main.cc b/src/nix/main.cc
index e62657e95..e9479f564 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -17,6 +17,8 @@
 #include <netdb.h>
 #include <netinet/in.h>
 
+#include <nlohmann/json.hpp>
+
 extern std::string chrootHelperName;
 
 void chrootHelper(int argc, char * * argv);
@@ -140,6 +142,11 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         printHelp(programName, std::cout);
         throw Exit();
     }
+
+    std::string description() override
+    {
+        return "a tool for reproducible and declarative configuration management";
+    }
 };
 
 void mainWrapped(int argc, char * * argv)
@@ -172,6 +179,29 @@ void mainWrapped(int argc, char * * argv)
 
     NixArgs args;
 
+    if (argc == 2 && std::string(argv[1]) == "__dump-args") {
+        std::cout << args.toJSON().dump() << "\n";
+        return;
+    }
+
+    if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
+        EvalState state({}, openStore("dummy://"));
+        auto res = nlohmann::json::object();
+        auto builtins = state.baseEnv.values[0]->attrs;
+        for (auto & builtin : *builtins) {
+            auto b = nlohmann::json::object();
+            if (builtin.value->type != tPrimOp) continue;
+            auto primOp = builtin.value->primOp;
+            if (!primOp->doc) continue;
+            b["arity"] = primOp->arity;
+            b["args"] = primOp->args;
+            b["doc"] = trim(stripIndentation(primOp->doc));
+            res[(std::string) builtin.name] = std::move(b);
+        }
+        std::cout << res.dump() << "\n";
+        return;
+    }
+
     Finally printCompletions([&]()
     {
         if (completions) {
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 2fe2e2fb2..38b60fc38 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -77,14 +77,16 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
 
             auto narHash = hashModuloSink.finish().first;
 
-            ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference));
+            ValidPathInfo info {
+                store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
+                narHash,
+            };
             info.references = std::move(references);
             if (hasSelfReference) info.references.insert(info.path);
-            info.narHash = narHash;
             info.narSize = sink.s->size();
             info.ca = FixedOutputHash {
                 .method = FileIngestionMethod::Recursive,
-                .hash = *info.narHash,
+                .hash = info.narHash,
             };
 
             if (!json)
diff --git a/src/nix/markdown.cc b/src/nix/markdown.cc
new file mode 100644
index 000000000..40788a42f
--- /dev/null
+++ b/src/nix/markdown.cc
@@ -0,0 +1,50 @@
+#include "markdown.hh"
+#include "util.hh"
+#include "finally.hh"
+
+#include <sys/queue.h>
+extern "C" {
+#include <lowdown.h>
+}
+
+namespace nix {
+
+std::string renderMarkdownToTerminal(std::string_view markdown)
+{
+    struct lowdown_opts opts {
+        .type = LOWDOWN_TERM,
+        .maxdepth = 20,
+        .cols = std::min(getWindowSize().second, (unsigned short) 80),
+        .hmargin = 0,
+        .vmargin = 0,
+        .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
+        .oflags = 0,
+    };
+
+    auto doc = lowdown_doc_new(&opts);
+    if (!doc)
+        throw Error("cannot allocate Markdown document");
+    Finally freeDoc([&]() { lowdown_doc_free(doc); });
+
+    size_t maxn = 0;
+    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size());
+    if (!node)
+        throw Error("cannot parse Markdown document");
+    Finally freeNode([&]() { lowdown_node_free(node); });
+
+    auto renderer = lowdown_term_new(&opts);
+    if (!renderer)
+        throw Error("cannot allocate Markdown renderer");
+    Finally freeRenderer([&]() { lowdown_term_free(renderer); });
+
+    auto buf = lowdown_buf_new(16384);
+    if (!buf)
+        throw Error("cannot allocate Markdown output buffer");
+    Finally freeBuffer([&]() { lowdown_buf_free(buf); });
+
+    lowdown_term_rndr(buf, nullptr, renderer, node);
+
+    return std::string(buf->data, buf->size);
+}
+
+}
diff --git a/src/nix/markdown.hh b/src/nix/markdown.hh
new file mode 100644
index 000000000..78320fcf5
--- /dev/null
+++ b/src/nix/markdown.hh
@@ -0,0 +1,7 @@
+#include "types.hh"
+
+namespace nix {
+
+std::string renderMarkdownToTerminal(std::string_view markdown);
+
+}
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index d42d16f7e..f97df4d9e 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -129,11 +129,13 @@ struct ProfileManifest
 
         auto narHash = hashString(htSHA256, *sink.s);
 
-        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references));
+        ValidPathInfo info {
+            store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references),
+            narHash,
+        };
         info.references = std::move(references);
-        info.narHash = narHash;
         info.narSize = sink.s->size();
-        info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = *info.narHash };
+        info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash };
 
         auto source = StringSource { *sink.s };
         store->addToStore(info, source);
@@ -435,7 +437,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
     }
 };
 
-struct CmdProfile : virtual MultiCommand, virtual Command
+struct CmdProfile : NixMultiCommand
 {
     CmdProfile()
         : MultiCommand({
@@ -459,11 +461,6 @@ struct CmdProfile : virtual MultiCommand, virtual Command
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto r1 = registerCommand<CmdProfile>("profile");
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index 54ee9152b..afa3503b8 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -115,7 +115,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand
     }
 };
 
-struct CmdRegistry : virtual MultiCommand, virtual Command
+struct CmdRegistry : virtual NixMultiCommand
 {
     CmdRegistry()
         : MultiCommand({
@@ -141,11 +141,6 @@ struct CmdRegistry : virtual MultiCommand, virtual Command
         command->second->prepare();
         command->second->run();
     }
-
-    void printHelp(const string & programName, std::ostream & out) override
-    {
-        MultiCommand::printHelp(programName, out);
-    }
 };
 
 static auto r1 = registerCommand<CmdRegistry>("registry");
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index c3c9e54a8..50d9b2568 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -32,6 +32,7 @@ extern "C" {
 #include "globals.hh"
 #include "command.hh"
 #include "finally.hh"
+#include "markdown.hh"
 
 #if HAVE_BOEHMGC
 #define GC_INCLUDE_NEW
@@ -64,7 +65,7 @@ struct NixRepl
     void mainLoop(const std::vector<std::string> & files);
     StringSet completePrefix(string prefix);
     bool getLine(string & input, const std::string &prompt);
-    Path getDerivationPath(Value & v);
+    StorePath getDerivationPath(Value & v);
     bool processLine(string line);
     void loadFile(const Path & path);
     void initEnv();
@@ -375,13 +376,16 @@ bool isVarName(const string & s)
 }
 
 
-Path NixRepl::getDerivationPath(Value & v) {
+StorePath NixRepl::getDerivationPath(Value & v) {
     auto drvInfo = getDerivation(*state, v, false);
     if (!drvInfo)
         throw Error("expression does not evaluate to a derivation, so I can't build it");
-    Path drvPath = drvInfo->queryDrvPath();
-    if (drvPath == "" || !state->store->isValidPath(state->store->parseStorePath(drvPath)))
-        throw Error("expression did not evaluate to a valid derivation");
+    Path drvPathRaw = drvInfo->queryDrvPath();
+    if (drvPathRaw == "")
+        throw Error("expression did not evaluate to a valid derivation (no drv path)");
+    StorePath drvPath = state->store->parseStorePath(drvPathRaw);
+    if (!state->store->isValidPath(drvPath))
+        throw Error("expression did not evaluate to a valid derivation (invalid drv path)");
     return drvPath;
 }
 
@@ -416,7 +420,8 @@ bool NixRepl::processLine(string line)
              << "  :r            Reload all files\n"
              << "  :s <expr>     Build dependencies of derivation, then start nix-shell\n"
              << "  :t <expr>     Describe result of evaluation\n"
-             << "  :u <expr>     Build derivation, then start nix-shell\n";
+             << "  :u <expr>     Build derivation, then start nix-shell\n"
+             << "  :doc <expr>   Show documentation of a builtin function\n";
     }
 
     else if (command == ":a" || command == ":add") {
@@ -474,29 +479,30 @@ bool NixRepl::processLine(string line)
         evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f);
         state->callFunction(f, v, result, Pos());
 
-        Path drvPath = getDerivationPath(result);
-        runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
+        StorePath drvPath = getDerivationPath(result);
+        runProgram(settings.nixBinDir + "/nix-shell", Strings{state->store->printStorePath(drvPath)});
     }
 
     else if (command == ":b" || command == ":i" || command == ":s") {
         Value v;
         evalString(arg, v);
-        Path drvPath = getDerivationPath(v);
+        StorePath drvPath = getDerivationPath(v);
+        Path drvPathRaw = state->store->printStorePath(drvPath);
 
         if (command == ":b") {
             /* We could do the build in this process using buildPaths(),
                but doing it in a child makes it easier to recover from
                problems / SIGINT. */
-            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) {
-                auto drv = readDerivation(*state->store, drvPath, Derivation::nameFromPath(state->store->parseStorePath(drvPath)));
+            if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) {
+                auto drv = state->store->readDerivation(drvPath);
                 std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
-                for (auto & i : drv.outputs)
-                    std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(i.second.path(*state->store, drv.name)));
+                for (auto & i : drv.outputsAndPaths(*state->store))
+                    std::cout << fmt("  %s -> %s\n", i.first, state->store->printStorePath(i.second.second));
             }
         } else if (command == ":i") {
-            runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath});
+            runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw});
         } else {
-            runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath});
+            runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPathRaw});
         }
     }
 
@@ -509,6 +515,29 @@ bool NixRepl::processLine(string line)
     else if (command == ":q" || command == ":quit")
         return false;
 
+    else if (command == ":doc") {
+        Value v;
+        evalString(arg, v);
+        if (auto doc = state->getDoc(v)) {
+            std::string markdown;
+
+            if (!doc->args.empty() && doc->name) {
+                auto args = doc->args;
+                for (auto & arg : args)
+                    arg = "*" + arg + "*";
+
+                markdown +=
+                    "**Synopsis:** `builtins." + (std::string) (*doc->name) + "` "
+                    + concatStringsSep(" ", args) + "\n\n";
+            }
+
+            markdown += trim(stripIndentation(doc->doc));
+
+            std::cout << renderMarkdownToTerminal(markdown);
+        } else
+            throw Error("value does not have documentation");
+    }
+
     else if (command != "")
         throw Error("unknown command '%1%'", command);
 
@@ -782,7 +811,7 @@ struct CmdRepl : StoreCommand, MixEvalArgs
         return {
           Example{
             "Display all special commands within the REPL:",
-              "nix repl\n  nix-repl> :?"
+            "nix repl\nnix-repl> :?"
           }
         };
     }
diff --git a/src/nix/search.cc b/src/nix/search.cc
index f052cc699..88815efdb 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -177,7 +177,7 @@ struct CmdSearch : InstallableCommand, MixJSON
             }
         };
 
-        for (auto & [cursor, prefix] : installable->getCursors(*state, true))
+        for (auto & [cursor, prefix] : installable->getCursors(*state))
             visit(*cursor, parseAttrPath(*state, prefix));
 
         if (!json && !results)
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index a97dc42f9..01a49f107 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -2,7 +2,8 @@
 #include "common-args.hh"
 #include "shared.hh"
 #include "store-api.hh"
-#include "json.hh"
+
+#include <nlohmann/json.hpp>
 
 using namespace nix;
 
@@ -19,8 +20,7 @@ struct CmdShowConfig : Command, MixJSON
     {
         if (json) {
             // FIXME: use appropriate JSON types (bool, ints, etc).
-            JSONObject jsonObj(std::cout);
-            globalConfig.toJSON(jsonObj);
+            logger->stdout_("%s", globalConfig.toJSON().dump());
         } else {
             std::map<std::string, Config::SettingInfo> settings;
             globalConfig.getSettings(settings);
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 1b51d114f..8c4bfb03e 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -67,9 +67,9 @@ struct CmdShowDerivation : InstallablesCommand
 
             {
                 auto outputsObj(drvObj.object("outputs"));
-                for (auto & output : drv.outputs) {
+                for (auto & output : drv.outputsAndPaths(*store)) {
                     auto outputObj(outputsObj.object(output.first));
-                    outputObj.attr("path", store->printStorePath(output.second.path(*store, drv.name)));
+                    outputObj.attr("path", store->printStorePath(output.second.second));
 
                     std::visit(overloaded {
                         [&](DerivationOutputInputAddressed doi) {
@@ -81,7 +81,7 @@ struct CmdShowDerivation : InstallablesCommand
                         [&](DerivationOutputCAFloating dof) {
                             outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
                         },
-                    }, output.second.output);
+                    }, output.second.first.output);
                 }
             }
 
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index fc7a9765c..26f755fd9 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -91,15 +91,15 @@ struct CmdVerify : StorePathsCommand
 
                     std::unique_ptr<AbstractHashSink> hashSink;
                     if (!info->ca)
-                        hashSink = std::make_unique<HashSink>(info->narHash->type);
+                        hashSink = std::make_unique<HashSink>(info->narHash.type);
                     else
-                        hashSink = std::make_unique<HashModuloSink>(info->narHash->type, std::string(info->path.hashPart()));
+                        hashSink = std::make_unique<HashModuloSink>(info->narHash.type, std::string(info->path.hashPart()));
 
                     store->narFromPath(info->path, *hashSink);
 
                     auto hash = hashSink->finish();
 
-                    if (hash.first != *info->narHash) {
+                    if (hash.first != info->narHash) {
                         corrupted++;
                         act2.result(resCorruptedPath, store->printStorePath(info->path));
                         logError({
@@ -107,7 +107,7 @@ struct CmdVerify : StorePathsCommand
                             .hint = hintfmt(
                                 "path '%s' was modified! expected hash '%s', got '%s'",
                                 store->printStorePath(info->path),
-                                info->narHash->to_string(Base32, true),
+                                info->narHash.to_string(Base32, true),
                                 hash.first.to_string(Base32, true))
                         });
                     }