Merge branch 'master' of github.com:NixOS/nix into enum-class

33 files changed, 920 insertions, 365 deletions

diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index ed35616e6..39d49721a 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -14,12 +14,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand
     {
         expectArg("path", &path);
 
-        mkFlag()
-            .longName("name")
-            .shortName('n')
-            .description("name component of the store path")
-            .labels({"name"})
-            .dest(&namePart);
+        addFlag({
+            .longName = "name",
+            .shortName = 'n',
+            .description = "name component of the store path",
+            .labels = {"name"},
+            .handler = {&namePart},
+        });
     }
 
     std::string description() override
@@ -33,6 +34,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         if (!namePart) namePart = baseNameOf(path);
@@ -42,15 +45,15 @@ struct CmdAddToStore : MixDryRun, StoreCommand
 
         auto narHash = hashString(HashType::SHA256, *sink.s);
 
-        ValidPathInfo info(store->makeFixedOutputPath(true, narHash, *namePart));
+        ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart));
         info.narHash = narHash;
         info.narSize = sink.s->size();
-        info.ca = makeFixedOutputCA(true, info.narHash);
+        info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
 
         if (!dryRun)
             store->addToStore(info, sink.s);
 
-        std::cout << fmt("%s\n", store->printStorePath(info.path));
+        logger->stdout("%s", store->printStorePath(info.path));
     }
 };
 
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 3c9d2df39..850e09ce8 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -5,23 +5,25 @@
 
 using namespace nix;
 
-struct CmdBuild : MixDryRun, InstallablesCommand
+struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
 {
     Path outLink = "result";
 
     CmdBuild()
     {
-        mkFlag()
-            .longName("out-link")
-            .shortName('o')
-            .description("path of the symlink to the build result")
-            .labels({"path"})
-            .dest(&outLink);
+        addFlag({
+            .longName = "out-link",
+            .shortName = 'o',
+            .description = "path of the symlink to the build result",
+            .labels = {"path"},
+            .handler = {&outLink},
+        });
 
-        mkFlag()
-            .longName("no-link")
-            .description("do not create a symlink to the build result")
-            .set(&outLink, Path(""));
+        addFlag({
+            .longName = "no-link",
+            .description = "do not create a symlink to the build result",
+            .handler = {&outLink, Path("")},
+        });
     }
 
     std::string description() override
@@ -40,6 +42,10 @@ struct CmdBuild : MixDryRun, InstallablesCommand
                 "To build the build.x86_64-linux attribute from release.nix:",
                 "nix build -f release.nix build.x86_64-linux"
             },
+            Example{
+                "To make a profile point at GNU Hello:",
+                "nix build --profile /tmp/profile nixpkgs.hello"
+            },
         };
     }
 
@@ -49,18 +55,19 @@ struct CmdBuild : MixDryRun, InstallablesCommand
 
         if (dryRun) return;
 
-        for (size_t i = 0; i < buildables.size(); ++i) {
-            auto & b(buildables[i]);
-
-            if (outLink != "")
-                for (auto & output : b.outputs)
+        if (outLink != "") {
+            for (size_t i = 0; i < buildables.size(); ++i) {
+                for (auto & output : buildables[i].outputs)
                     if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
                         std::string symlink = outLink;
                         if (i) symlink += fmt("-%d", i);
                         if (output.first != "out") symlink += fmt("-%s", output.first);
                         store2->addPermRoot(output.second, absPath(symlink), true);
                     }
+            }
         }
+
+        updateProfile(buildables);
     }
 };
 
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index 851f90abd..fd91f2036 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -30,9 +30,11 @@ struct CmdCatStore : StoreCommand, MixCat
 
     std::string description() override
     {
-        return "print the contents of a store file on stdout";
+        return "print the contents of a file in the Nix store on stdout";
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         cat(store->getFSAccessor());
@@ -51,9 +53,11 @@ struct CmdCatNar : StoreCommand, MixCat
 
     std::string description() override
     {
-        return "print the contents of a file inside a NAR file";
+        return "print the contents of a file inside a NAR file on stdout";
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         cat(makeNarAccessor(make_ref<std::string>(readFile(narPath))));
diff --git a/src/nix/command.cc b/src/nix/command.cc
index 442bc6c53..71b027719 100644
--- a/src/nix/command.cc
+++ b/src/nix/command.cc
@@ -2,6 +2,9 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "nixexpr.hh"
+#include "profiles.hh"
+
+extern char * * environ;
 
 namespace nix {
 
@@ -32,16 +35,18 @@ StorePathsCommand::StorePathsCommand(bool recursive)
     : recursive(recursive)
 {
     if (recursive)
-        mkFlag()
-            .longName("no-recursive")
-            .description("apply operation to specified paths only")
-            .set(&this->recursive, false);
+        addFlag({
+            .longName = "no-recursive",
+            .description = "apply operation to specified paths only",
+            .handler = {&this->recursive, false},
+        });
     else
-        mkFlag()
-            .longName("recursive")
-            .shortName('r')
-            .description("apply operation to closure of the specified paths")
-            .set(&this->recursive, true);
+        addFlag({
+            .longName = "recursive",
+            .shortName = 'r',
+            .description = "apply operation to closure of the specified paths",
+            .handler = {&this->recursive, true},
+        });
 
     mkFlag(0, "all", "apply operation to the entire store", &all);
 }
@@ -96,4 +101,98 @@ Strings editorFor(const Pos & pos)
     return args;
 }
 
+MixProfile::MixProfile()
+{
+    addFlag({
+        .longName = "profile",
+        .description = "profile to update",
+        .labels = {"path"},
+        .handler = {&profile},
+    });
+}
+
+void MixProfile::updateProfile(const StorePath & storePath)
+{
+    if (!profile) return;
+    auto store = getStore().dynamic_pointer_cast<LocalFSStore>();
+    if (!store) throw Error("'--profile' is not supported for this Nix store");
+    auto profile2 = absPath(*profile);
+    switchLink(profile2,
+        createGeneration(
+            ref<LocalFSStore>(store),
+            profile2, store->printStorePath(storePath)));
+}
+
+void MixProfile::updateProfile(const Buildables & buildables)
+{
+    if (!profile) return;
+
+    std::optional<StorePath> result;
+
+    for (auto & buildable : buildables) {
+        for (auto & output : buildable.outputs) {
+            if (result)
+                throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple");
+            result = output.second.clone();
+        }
+    }
+
+    if (!result)
+        throw Error("'--profile' requires that the arguments produce a single store path, but there are none");
+
+    updateProfile(*result);
+}
+
+MixDefaultProfile::MixDefaultProfile()
+{
+    profile = getDefaultProfile();
+}
+
+MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
+{
+    addFlag({
+        .longName = "ignore-environment",
+        .shortName = 'i',
+        .description = "clear the entire environment (except those specified with --keep)",
+        .handler = {&ignoreEnvironment, true},
+    });
+
+    addFlag({
+        .longName = "keep",
+        .shortName = 'k',
+        .description = "keep specified environment variable",
+        .labels = {"name"},
+        .handler = {[&](std::string s) { keep.insert(s); }},
+    });
+
+    addFlag({
+        .longName = "unset",
+        .shortName = 'u',
+        .description = "unset specified environment variable",
+        .labels = {"name"},
+        .handler = {[&](std::string s) { unset.insert(s); }},
+    });
+}
+
+void MixEnvironment::setEnviron() {
+    if (ignoreEnvironment) {
+        if (!unset.empty())
+            throw UsageError("--unset does not make sense with --ignore-environment");
+
+        for (const auto & var : keep) {
+            auto val = getenv(var.c_str());
+            if (val) stringsEnv.emplace_back(fmt("%s=%s", var.c_str(), val));
+        }
+
+        vectorEnv = stringsToCharPtrs(stringsEnv);
+        environ = vectorEnv.data();
+    } else {
+        if (!keep.empty())
+            throw UsageError("--keep does not make sense without --ignore-environment");
+
+        for (const auto & var : unset)
+            unsetenv(var.c_str());
+    }
+}
+
 }
diff --git a/src/nix/command.hh b/src/nix/command.hh
index a954a7d04..959d5f19d 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -1,5 +1,6 @@
 #pragma once
 
+#include "installables.hh"
 #include "args.hh"
 #include "common-eval-args.hh"
 #include "path.hh"
@@ -9,6 +10,10 @@ namespace nix {
 
 extern std::string programPath;
 
+static constexpr Command::Category catSecondary = 100;
+static constexpr Command::Category catUtility = 101;
+static constexpr Command::Category catNixInstallation = 102;
+
 /* A command that requires a Nix store. */
 struct StoreCommand : virtual Command
 {
@@ -22,34 +27,7 @@ private:
     std::shared_ptr<Store> _store;
 };
 
-struct Buildable
-{
-    std::optional<StorePath> drvPath;
-    std::map<std::string, StorePath> outputs;
-};
-
-typedef std::vector<Buildable> Buildables;
-
-struct Installable
-{
-    virtual ~Installable() { }
-
-    virtual std::string what() = 0;
-
-    virtual Buildables toBuildables()
-    {
-        throw Error("argument '%s' cannot be built", what());
-    }
-
-    Buildable toBuildable();
-
-    virtual std::pair<Value *, Pos> toValue(EvalState & state)
-    {
-        throw Error("argument '%s' cannot be evaluated", what());
-    }
-};
-
-struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs
+struct SourceExprCommand : virtual StoreCommand, MixEvalArgs
 {
     Path file;
 
@@ -67,7 +45,7 @@ private:
 
     std::shared_ptr<EvalState> evalState;
 
-    Value * vSourceExpr = 0;
+    RootValue vSourceExpr;
 };
 
 enum RealiseMode { Build, NoBuild, DryRun };
@@ -184,4 +162,36 @@ std::set<StorePath> toDerivations(ref<Store> store,
    filename:lineno. */
 Strings editorFor(const Pos & pos);
 
+struct MixProfile : virtual StoreCommand
+{
+    std::optional<Path> profile;
+
+    MixProfile();
+
+    /* If 'profile' is set, make it point at 'storePath'. */
+    void updateProfile(const StorePath & storePath);
+
+    /* If 'profile' is set, make it point at the store path produced
+       by 'buildables'. */
+    void updateProfile(const Buildables & buildables);
+};
+
+struct MixDefaultProfile : MixProfile
+{
+    MixDefaultProfile();
+};
+
+struct MixEnvironment : virtual Args {
+
+    StringSet keep, unset;
+    Strings stringsEnv;
+    std::vector<char*> vectorEnv;
+    bool ignoreEnvironment;
+
+    MixEnvironment();
+
+    /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */
+    void setEnviron();
+};
+
 }
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index 85c777d38..c7c38709d 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -19,27 +19,32 @@ struct CmdCopy : StorePathsCommand
     CmdCopy()
         : StorePathsCommand(true)
     {
-        mkFlag()
-            .longName("from")
-            .labels({"store-uri"})
-            .description("URI of the source Nix store")
-            .dest(&srcUri);
-        mkFlag()
-            .longName("to")
-            .labels({"store-uri"})
-            .description("URI of the destination Nix store")
-            .dest(&dstUri);
-
-        mkFlag()
-            .longName("no-check-sigs")
-            .description("do not require that paths are signed by trusted keys")
-            .set(&checkSigs, NoCheckSigs);
-
-        mkFlag()
-            .longName("substitute-on-destination")
-            .shortName('s')
-            .description("whether to try substitutes on the destination store (only supported by SSH)")
-            .set(&substitute, Substitute);
+        addFlag({
+            .longName = "from",
+            .description = "URI of the source Nix store",
+            .labels = {"store-uri"},
+            .handler = {&srcUri},
+        });
+
+        addFlag({
+            .longName = "to",
+            .description = "URI of the destination Nix store",
+            .labels = {"store-uri"},
+            .handler = {&dstUri},
+        });
+
+        addFlag({
+            .longName = "no-check-sigs",
+            .description = "do not require that paths are signed by trusted keys",
+            .handler = {&checkSigs, NoCheckSigs},
+        });
+
+        addFlag({
+            .longName = "substitute-on-destination",
+            .shortName = 's',
+            .description = "whether to try substitutes on the destination store (only supported by SSH)",
+            .handler = {&substitute, Substitute},
+        });
     }
 
     std::string description() override
@@ -75,6 +80,8 @@ struct CmdCopy : StorePathsCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     ref<Store> createStore() override
     {
         return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri);
diff --git a/src/nix/dev-shell.cc b/src/nix/dev-shell.cc
new file mode 100644
index 000000000..d300f6a23
--- /dev/null
+++ b/src/nix/dev-shell.cc
@@ -0,0 +1,344 @@
+#include "eval.hh"
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "derivations.hh"
+#include "affinity.hh"
+#include "progress-bar.hh"
+
+#include <regex>
+
+using namespace nix;
+
+struct Var
+{
+    bool exported = true;
+    bool associative = false;
+    std::string value; // quoted string or array
+};
+
+struct BuildEnvironment
+{
+    std::map<std::string, Var> env;
+    std::string bashFunctions;
+};
+
+BuildEnvironment readEnvironment(const Path & path)
+{
+    BuildEnvironment res;
+
+    std::set<std::string> exported;
+
+    debug("reading environment file '%s'", path);
+
+    auto file = readFile(path);
+
+    auto pos = file.cbegin();
+
+    static std::string varNameRegex =
+        R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";
+
+    static std::regex declareRegex(
+        "^declare -x (" + varNameRegex + ")" +
+        R"re((?:="((?:[^"\\]|\\.)*)")?\n)re");
+
+    static std::string simpleStringRegex =
+        R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re";
+
+    static std::string quotedStringRegex =
+        R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
+
+    static std::string indexedArrayRegex =
+        R"re((?:\(( *\[[0-9]+]="(?:[^"\\]|\\.)*")**\)))re";
+
+    static std::regex varRegex(
+        "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n");
+
+    /* Note: we distinguish between an indexed and associative array
+       using the space before the closing parenthesis. Will
+       undoubtedly regret this some day. */
+    static std::regex assocArrayRegex(
+        "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n");
+
+    static std::regex functionRegex(
+        "^" + varNameRegex + " \\(\\) *\n");
+
+    while (pos != file.end()) {
+
+        std::smatch match;
+
+        if (std::regex_search(pos, file.cend(), match, declareRegex)) {
+            pos = match[0].second;
+            exported.insert(match[1]);
+        }
+
+        else if (std::regex_search(pos, file.cend(), match, varRegex)) {
+            pos = match[0].second;
+            res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }});
+        }
+
+        else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) {
+            pos = match[0].second;
+            res.env.insert({match[1], Var { .associative = true, .value = match[2] }});
+        }
+
+        else if (std::regex_search(pos, file.cend(), match, functionRegex)) {
+            res.bashFunctions = std::string(pos, file.cend());
+            break;
+        }
+
+        else throw Error("shell environment '%s' has unexpected line '%s'",
+            path, file.substr(pos - file.cbegin(), 60));
+    }
+
+    return res;
+}
+
+const static std::string getEnvSh =
+    #include "get-env.sh.gen.hh"
+    ;
+
+/* Given an existing derivation, return the shell environment as
+   initialised by stdenv's setup script. We do this by building a
+   modified derivation with the same dependencies and nearly the same
+   initial environment variables, that just writes the resulting
+   environment to a file and exits. */
+StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
+{
+    auto drv = store->derivationFromPath(drvPath);
+
+    auto builder = baseNameOf(drv.builder);
+    if (builder != "bash")
+        throw Error("'nix dev-shell' only works on derivations that use 'bash' as their builder");
+
+    auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {});
+
+    drv.args = {store->printStorePath(getEnvShPath)};
+
+    /* Remove derivation checks. */
+    drv.env.erase("allowedReferences");
+    drv.env.erase("allowedRequisites");
+    drv.env.erase("disallowedReferences");
+    drv.env.erase("disallowedRequisites");
+
+    /* Rehash and write the derivation. FIXME: would be nice to use
+       'buildDerivation', but that's privileged. */
+    auto drvName = std::string(drvPath.name());
+    assert(hasSuffix(drvName, ".drv"));
+    drvName.resize(drvName.size() - 4);
+    drvName += "-env";
+    for (auto & output : drv.outputs)
+        drv.env.erase(output.first);
+    drv.env["out"] = "";
+    drv.env["outputs"] = "out";
+    drv.inputSrcs.insert(std::move(getEnvShPath));
+    Hash h = hashDerivationModulo(*store, drv, true);
+    auto shellOutPath = store->makeOutputPath("out", h, drvName);
+    drv.outputs.insert_or_assign("out", DerivationOutput(shellOutPath.clone(), "", ""));
+    drv.env["out"] = store->printStorePath(shellOutPath);
+    auto shellDrvPath2 = writeDerivation(store, drv, drvName);
+
+    /* Build the derivation. */
+    store->buildPaths({shellDrvPath2});
+
+    assert(store->isValidPath(shellOutPath));
+
+    return shellOutPath;
+}
+
+struct Common : InstallableCommand, MixProfile
+{
+    std::set<string> ignoreVars{
+        "BASHOPTS",
+        "EUID",
+        "HOME", // FIXME: don't ignore in pure mode?
+        "NIX_BUILD_TOP",
+        "NIX_ENFORCE_PURITY",
+        "NIX_LOG_FD",
+        "PPID",
+        "PWD",
+        "SHELLOPTS",
+        "SHLVL",
+        "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt
+        "TEMP",
+        "TEMPDIR",
+        "TERM",
+        "TMP",
+        "TMPDIR",
+        "TZ",
+        "UID",
+    };
+
+    void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out)
+    {
+        out << "unset shellHook\n";
+
+        out << "nix_saved_PATH=\"$PATH\"\n";
+
+        for (auto & i : buildEnvironment.env) {
+            if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) {
+                if (i.second.associative)
+                    out << fmt("declare -A %s=(%s)\n", i.first, i.second.value);
+                else {
+                    out << fmt("%s=%s\n", i.first, i.second.value);
+                    if (i.second.exported)
+                        out << fmt("export %s\n", i.first);
+                }
+            }
+        }
+
+        out << "PATH=\"$PATH:$nix_saved_PATH\"\n";
+
+        out << buildEnvironment.bashFunctions << "\n";
+
+        // FIXME: set outputs
+
+        out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n";
+        for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
+            out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
+
+        out << "eval \"$shellHook\"\n";
+    }
+
+    StorePath getShellOutPath(ref<Store> store)
+    {
+        auto path = installable->getStorePath();
+        if (path && hasSuffix(path->to_string(), "-env"))
+            return path->clone();
+        else {
+            auto drvs = toDerivations(store, {installable});
+
+            if (drvs.size() != 1)
+                throw Error("'%s' needs to evaluate to a single derivation, but it evaluated to %d derivations",
+                    installable->what(), drvs.size());
+
+            auto & drvPath = *drvs.begin();
+
+            return getDerivationEnvironment(store, drvPath);
+        }
+    }
+
+    std::pair<BuildEnvironment, std::string> getBuildEnvironment(ref<Store> store)
+    {
+        auto shellOutPath = getShellOutPath(store);
+
+        auto strPath = store->printStorePath(shellOutPath);
+
+        updateProfile(shellOutPath);
+
+        return {readEnvironment(strPath), strPath};
+    }
+};
+
+struct CmdDevShell : Common, MixEnvironment
+{
+    std::vector<std::string> command;
+
+    CmdDevShell()
+    {
+        addFlag({
+            .longName = "command",
+            .shortName = 'c',
+            .description = "command and arguments to be executed insted of an interactive shell",
+            .labels = {"command", "args"},
+            .handler = {[&](std::vector<std::string> ss) {
+                if (ss.empty()) throw UsageError("--command requires at least one argument");
+                command = ss;
+            }}
+        });
+    }
+
+    std::string description() override
+    {
+        return "run a bash shell that provides the build environment of a derivation";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To get the build environment of GNU hello:",
+                "nix dev-shell nixpkgs.hello"
+            },
+            Example{
+                "To store the build environment in a profile:",
+                "nix dev-shell --profile /tmp/my-shell nixpkgs.hello"
+            },
+            Example{
+                "To use a build environment previously recorded in a profile:",
+                "nix dev-shell /tmp/my-shell"
+            },
+        };
+    }
+
+    void run(ref<Store> store) override
+    {
+        auto [buildEnvironment, gcroot] = getBuildEnvironment(store);
+
+        auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
+
+        std::ostringstream ss;
+        makeRcScript(buildEnvironment, ss);
+
+        ss << fmt("rm -f '%s'\n", rcFilePath);
+
+        if (!command.empty()) {
+            std::vector<std::string> args;
+            for (auto s : command)
+                args.push_back(shellEscape(s));
+            ss << fmt("exec %s\n", concatStringsSep(" ", args));
+        }
+
+        writeFull(rcFileFd.get(), ss.str());
+
+        stopProgressBar();
+
+        auto shell = getEnv("SHELL").value_or("bash");
+
+        setEnviron();
+        // prevent garbage collection until shell exits
+        setenv("NIX_GCROOT", gcroot.data(), 1);
+
+        auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
+
+        restoreAffinity();
+        restoreSignals();
+
+        execvp(shell.c_str(), stringsToCharPtrs(args).data());
+
+        throw SysError("executing shell '%s'", shell);
+    }
+};
+
+struct CmdPrintDevEnv : Common
+{
+    std::string description() override
+    {
+        return "print shell code that can be sourced by bash to reproduce the build environment of a derivation";
+    }
+
+    Examples examples() override
+    {
+        return {
+            Example{
+                "To apply the build environment of GNU hello to the current shell:",
+                ". <(nix print-dev-env nixpkgs.hello)"
+            },
+        };
+    }
+
+    Category category() override { return catUtility; }
+
+    void run(ref<Store> store) override
+    {
+        auto buildEnvironment = getBuildEnvironment(store).first;
+
+        stopProgressBar();
+
+        makeRcScript(buildEnvironment, std::cout);
+    }
+};
+
+static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env");
+static auto r2 = registerCommand<CmdDevShell>("dev-shell");
diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc
index 0aa634d6e..82e92cdd0 100644
--- a/src/nix/doctor.cc
+++ b/src/nix/doctor.cc
@@ -40,9 +40,11 @@ struct CmdDoctor : StoreCommand
 
     std::string description() override
     {
-        return "check your system for potential problems and print a PASS or FAIL for each check.";
+        return "check your system for potential problems and print a PASS or FAIL for each check";
     }
 
+    Category category() override { return catNixInstallation; }
+
     void run(ref<Store> store) override
     {
         logger->log("Running checks against store uri: " + store->getUri());
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index bb741b572..e1de71bf8 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -20,6 +20,8 @@ struct CmdDumpPath : StorePathCommand
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store, const StorePath & storePath) override
     {
         FdSink sink(STDOUT_FILENO);
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 1683eada0..067d3a973 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -25,6 +25,8 @@ struct CmdEdit : InstallableCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     void run(ref<Store> store) override
     {
         auto state = getEvalState();
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 6398fc58e..26e98ac2a 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -45,6 +45,8 @@ struct CmdEval : MixJSON, InstallableCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     void run(ref<Store> store) override
     {
         if (raw && json)
@@ -55,16 +57,15 @@ struct CmdEval : MixJSON, InstallableCommand
         auto v = installable->toValue(*state).first;
         PathSet context;
 
-        stopProgressBar();
-
         if (raw) {
+            stopProgressBar();
             std::cout << state->coerceToString(noPos, *v, context);
         } else if (json) {
             JSONPlaceholder jsonOut(std::cout);
             printValueAsJSON(*state, true, *v, jsonOut, context);
         } else {
             state->forceValueDeep(*v);
-            std::cout << *v << "\n";
+            logger->stdout("%s", *v);
         }
     }
 };
diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh
new file mode 100644
index 000000000..a25ec43a9
--- /dev/null
+++ b/src/nix/get-env.sh
@@ -0,0 +1,9 @@
+set -e
+if [ -e .attrs.sh ]; then source .attrs.sh; fi
+export IN_NIX_SHELL=impure
+export dontAddDisableDepTrack=1
+if [[ -n $stdenv ]]; then
+    source $stdenv/setup
+fi
+export > $out
+set >> $out
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index deced3d11..3362ffd0d 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -9,23 +9,20 @@ using namespace nix;
 
 struct CmdHash : Command
 {
-    enum Mode { mFile, mPath };
-    Mode mode;
+    FileIngestionMethod mode;
     Base base = Base::SRI;
     bool truncate = false;
     HashType ht = HashType::SHA256;
     std::vector<std::string> paths;
     std::optional<std::string> modulus;
 
-    CmdHash(Mode mode) : mode(mode)
+    CmdHash(FileIngestionMethod mode) : mode(mode)
     {
         mkFlag(0, "sri", "print hash in Base::SRI format", &base, Base::SRI);
         mkFlag(0, "base64", "print hash in base-64", &base, Base::Base64);
         mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base::Base32);
         mkFlag(0, "base16", "print hash in base-16", &base, Base::Base16);
-        mkFlag()
-            .longName("type")
-            .mkHashTypeFlag(&ht);
+        addFlag(Flag::mkHashTypeFlag("type", &ht));
         #if 0
         mkFlag()
             .longName("modulo")
@@ -38,11 +35,18 @@ struct CmdHash : Command
 
     std::string description() override
     {
-        return mode == mFile
-            ? "print cryptographic hash of a regular file"
-            : "print cryptographic hash of the NAR serialisation of a path";
+        const char* d;
+        switch (mode) {
+        case FileIngestionMethod::Flat:
+            d = "print cryptographic hash of a regular file";
+        case FileIngestionMethod::Recursive:
+            d = "print cryptographic hash of the NAR serialisation of a path";
+        };
+        return d;
     }
 
+    Category category() override { return catUtility; }
+
     void run() override
     {
         for (auto path : paths) {
@@ -53,21 +57,24 @@ struct CmdHash : Command
             else
                 hashSink = std::make_unique<HashSink>(ht);
 
-            if (mode == mFile)
+            switch (mode) {
+            case FileIngestionMethod::Flat:
                 readFile(path, *hashSink);
-            else
+                break;
+            case FileIngestionMethod::Recursive:
                 dumpPath(path, *hashSink);
+                break;
+            }
 
             Hash h = hashSink->finish().first;
             if (truncate && h.hashSize > 20) h = compressHash(h, 20);
-            std::cout << format("%1%\n") %
-                h.to_string(base, base == Base::SRI);
+            logger->stdout(h.to_string(base, base == Base::SRI));
         }
     }
 };
 
-static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(CmdHash::mFile); });
-static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(CmdHash::mPath); });
+static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); });
+static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); });
 
 struct CmdToBase : Command
 {
@@ -77,9 +84,7 @@ struct CmdToBase : Command
 
     CmdToBase(Base base) : base(base)
     {
-        mkFlag()
-            .longName("type")
-            .mkHashTypeFlag(&ht);
+        addFlag(Flag::mkHashTypeFlag("type", &ht));
         expectArgs("strings", &args);
     }
 
@@ -92,10 +97,12 @@ struct CmdToBase : Command
             "Base::SRI");
     }
 
+    Category category() override { return catUtility; }
+
     void run() override
     {
         for (auto s : args)
-            std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == Base::SRI));
+            logger->stdout(Hash(s, ht).to_string(base, base == Base::SRI));
     }
 };
 
@@ -138,7 +145,7 @@ static int compatNixHash(int argc, char * * argv)
     });
 
     if (op == opHash) {
-        CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath);
+        CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
         cmd.ht = ht;
         cmd.base = base32 ? Base::Base32 : Base::Base16;
         cmd.truncate = truncate;
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 03d03e90a..17d15f5ee 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -12,26 +12,28 @@
 
 namespace nix {
 
+
 SourceExprCommand::SourceExprCommand()
 {
-    mkFlag()
-        .shortName('f')
-        .longName("file")
-        .label("file")
-        .description("evaluate FILE rather than the default")
-        .dest(&file);
+    addFlag({
+        .longName = "file",
+        .shortName = 'f',
+        .description = "evaluate FILE rather than the default",
+        .labels = {"file"},
+        .handler = {&file}
+    });
 }
 
 Value * SourceExprCommand::getSourceExpr(EvalState & state)
 {
-    if (vSourceExpr) return vSourceExpr;
+    if (vSourceExpr) return *vSourceExpr;
 
     auto sToplevel = state.symbols.create("_toplevel");
 
-    vSourceExpr = state.allocValue();
+    vSourceExpr = allocRootValue(state.allocValue());
 
     if (file != "")
-        state.evalFile(lookupFileArg(state, file), *vSourceExpr);
+        state.evalFile(lookupFileArg(state, file), **vSourceExpr);
 
     else {
 
@@ -39,9 +41,9 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
 
         auto searchPath = state.getSearchPath();
 
-        state.mkAttrs(*vSourceExpr, 1024);
+        state.mkAttrs(**vSourceExpr, 1024);
 
-        mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true);
+        mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true);
 
         std::unordered_set<std::string> seen;
 
@@ -52,7 +54,7 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
             mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath"));
             Value * v2 = state.allocValue();
             mkApp(*v2, *v1, mkString(*state.allocValue(), name));
-            mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)),
+            mkApp(*state.allocAttr(**vSourceExpr, state.symbols.create(name)),
                 state.getBuiltin("import"), *v2);
         };
 
@@ -66,10 +68,10 @@ Value * SourceExprCommand::getSourceExpr(EvalState & state)
             } else
                 addEntry(i.first);
 
-        vSourceExpr->attrs->sort();
+        (*vSourceExpr)->attrs->sort();
     }
 
-    return vSourceExpr;
+    return *vSourceExpr;
 }
 
 ref<EvalState> SourceExprCommand::getEvalState()
@@ -109,6 +111,11 @@ struct InstallableStorePath : Installable
         bs.push_back(std::move(b));
         return bs;
     }
+
+    std::optional<StorePath> getStorePath() override
+    {
+        return storePath.clone();
+    }
 };
 
 struct InstallableValue : Installable
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
new file mode 100644
index 000000000..503984220
--- /dev/null
+++ b/src/nix/installables.hh
@@ -0,0 +1,45 @@
+#pragma once
+
+#include "util.hh"
+#include "path.hh"
+#include "eval.hh"
+
+#include <optional>
+
+namespace nix {
+
+struct Buildable
+{
+    std::optional<StorePath> drvPath;
+    std::map<std::string, StorePath> outputs;
+};
+
+typedef std::vector<Buildable> Buildables;
+
+struct Installable
+{
+    virtual ~Installable() { }
+
+    virtual std::string what() = 0;
+
+    virtual Buildables toBuildables()
+    {
+        throw Error("argument '%s' cannot be built", what());
+    }
+
+    Buildable toBuildable();
+
+    virtual std::pair<Value *, Pos> toValue(EvalState & state)
+    {
+        throw Error("argument '%s' cannot be evaluated", what());
+    }
+
+    /* Return a value only if this installable is a store path or a
+       symlink to it. */
+    virtual std::optional<StorePath> getStorePath()
+    {
+        return {};
+    }
+};
+
+}
diff --git a/src/nix/local.mk b/src/nix/local.mk
index 51dad101f..8c0eed19e 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -15,7 +15,9 @@ nix_SOURCES := \
   $(wildcard src/nix-prefetch-url/*.cc) \
   $(wildcard src/nix-store/*.cc) \
 
-nix_LIBS = libexpr libmain libstore libutil libnixrust
+nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain
+
+nix_LIBS = libexpr libmain libfetchers libstore libutil libnixrust
 
 nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system
 
@@ -25,3 +27,5 @@ $(foreach name, \
 $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote))
 
 src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh
+
+src/nix/dev-shell.cc: src/nix/get-env.sh.gen.hh
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 795991cb7..3fe22f6c2 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -31,6 +31,8 @@ struct CmdLog : InstallableCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     void run(ref<Store> store) override
     {
         settings.readOnlyMode = true;
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index 3ef1f2750..b9716a6a1 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -34,16 +34,14 @@ struct MixLs : virtual Args, MixJSON
                         (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") :
                     st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" :
                     "dr-xr-xr-x";
-                std::cout <<
-                    (format("%s %20d %s") % tp % st.fileSize % relPath);
+                auto line = fmt("%s %20d %s", tp, st.fileSize, relPath);
                 if (st.type == FSAccessor::Type::tSymlink)
-                    std::cout << " -> " << accessor->readLink(curPath)
-                    ;
-                std::cout << "\n";
+                    line += " -> " + accessor->readLink(curPath);
+                logger->stdout(line);
                 if (recursive && st.type == FSAccessor::Type::tDirectory)
                     doPath(st, curPath, relPath, false);
             } else {
-                std::cout << relPath << "\n";
+                logger->stdout(relPath);
                 if (recursive) {
                     auto st = accessor->stat(curPath);
                     if (st.type == FSAccessor::Type::tDirectory)
@@ -102,9 +100,11 @@ struct CmdLsStore : StoreCommand, MixLs
 
     std::string description() override
     {
-        return "show information about a store path";
+        return "show information about a path in the Nix store";
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         list(store->getFSAccessor());
@@ -133,12 +133,14 @@ struct CmdLsNar : Command, MixLs
 
     std::string description() override
     {
-        return "show information about the contents of a NAR file";
+        return "show information about a path inside a NAR file";
     }
 
+    Category category() override { return catUtility; }
+
     void run() override
     {
-        list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true))));
+        list(makeNarAccessor(make_ref<std::string>(readFile(narPath))));
     }
 };
 
diff --git a/src/nix/main.cc b/src/nix/main.cc
index d0a43ab23..fa0bb5b51 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -8,7 +8,7 @@
 #include "shared.hh"
 #include "store-api.hh"
 #include "progress-bar.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "finally.hh"
 
 #include <sys/types.h>
@@ -59,15 +59,22 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
     NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
     {
-        mkFlag()
-            .longName("help")
-            .description("show usage information")
-            .handler([&]() { showHelpAndExit(); });
-
-        mkFlag()
-            .longName("help-config")
-            .description("show configuration options")
-            .handler([&]() {
+        categories.clear();
+        categories[Command::catDefault] = "Main commands";
+        categories[catSecondary] = "Infrequently used commands";
+        categories[catUtility] = "Utility/scripting commands";
+        categories[catNixInstallation] = "Commands for upgrading or troubleshooting your Nix installation";
+
+        addFlag({
+            .longName = "help",
+            .description = "show usage information",
+            .handler = {[&]() { showHelpAndExit(); }},
+        });
+
+        addFlag({
+            .longName = "help-config",
+            .description = "show configuration options",
+            .handler = {[&]() {
                 std::cout << "The following configuration options are available:\n\n";
                 Table2 tbl;
                 std::map<std::string, Config::SettingInfo> settings;
@@ -76,28 +83,33 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
                     tbl.emplace_back(s.first, s.second.description);
                 printTable(std::cout, tbl);
                 throw Exit();
-            });
-
-        mkFlag()
-            .longName("print-build-logs")
-            .shortName('L')
-            .description("print full build logs on stderr")
-            .set(&printBuildLogs, true);
-
-        mkFlag()
-            .longName("version")
-            .description("show version information")
-            .handler([&]() { printVersion(programName); });
-
-        mkFlag()
-            .longName("no-net")
-            .description("disable substituters and consider all previously downloaded files up-to-date")
-            .handler([&]() { useNet = false; });
-
-        mkFlag()
-            .longName("refresh")
-            .description("consider all previously downloaded files out-of-date")
-            .handler([&]() { refresh = true; });
+            }},
+        });
+
+        addFlag({
+            .longName = "print-build-logs",
+            .shortName = 'L',
+            .description = "print full build logs on stderr",
+            .handler = {&printBuildLogs, true},
+        });
+
+        addFlag({
+            .longName = "version",
+            .description = "show version information",
+            .handler = {[&]() { printVersion(programName); }},
+        });
+
+        addFlag({
+            .longName = "no-net",
+            .description = "disable substituters and consider all previously downloaded files up-to-date",
+            .handler = {[&]() { useNet = false; }},
+        });
+
+        addFlag({
+            .longName = "refresh",
+            .description = "consider all previously downloaded files out-of-date",
+            .handler = {[&]() { refresh = true; }},
+        });
     }
 
     void printFlags(std::ostream & out) override
@@ -105,8 +117,8 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         Args::printFlags(out);
         std::cout <<
             "\n"
-            "In addition, most configuration settings can be overriden using '--<name> <value>'.\n"
-            "Boolean settings can be overriden using '--<name>' or '--no-<name>'. See 'nix\n"
+            "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n"
+            "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n"
             "--help-config' for a list of configuration settings.\n";
     }
 
@@ -115,10 +127,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
         MultiCommand::printHelp(programName, out);
 
 #if 0
-        out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n";
+        out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n";
 #endif
 
-        std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n";
+        std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n";
     }
 
     void showHelpAndExit()
@@ -155,12 +167,15 @@ void mainWrapped(int argc, char * * argv)
 
     args.parseCmdline(argvToStrings(argc, argv));
 
-    settings.requireExperimentalFeature("nix-command");
-
     initPlugins();
 
     if (!args.command) args.showHelpAndExit();
 
+    if (args.command->first != "repl"
+        && args.command->first != "doctor"
+        && args.command->first != "upgrade-nix")
+        settings.requireExperimentalFeature("nix-command");
+
     Finally f([]() { stopProgressBar(); });
 
     startProgressBar(args.printBuildLogs);
@@ -176,17 +191,17 @@ void mainWrapped(int argc, char * * argv)
             settings.useSubstitutes = false;
         if (!settings.tarballTtl.overriden)
             settings.tarballTtl = std::numeric_limits<unsigned int>::max();
-        if (!downloadSettings.tries.overriden)
-            downloadSettings.tries = 0;
-        if (!downloadSettings.connectTimeout.overriden)
-            downloadSettings.connectTimeout = 1;
+        if (!fileTransferSettings.tries.overriden)
+            fileTransferSettings.tries = 0;
+        if (!fileTransferSettings.connectTimeout.overriden)
+            fileTransferSettings.connectTimeout = 1;
     }
 
     if (args.refresh)
         settings.tarballTtl = 0;
 
-    args.command->prepare();
-    args.command->run();
+    args.command->second->prepare();
+    args.command->second->run();
 }
 
 }
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 5c964ec27..bd948a983 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -31,6 +31,9 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
             },
         };
     }
+
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store, StorePaths storePaths) override
     {
         auto paths = store->topoSortPaths(storePathsToSet(storePaths));
@@ -74,12 +77,12 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
 
             auto narHash = hashModuloSink.finish().first;
 
-            ValidPathInfo info(store->makeFixedOutputPath(true, narHash, path.name(), references, hasSelfReference));
+            ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference));
             info.references = std::move(references);
             if (hasSelfReference) info.references.insert(info.path.clone());
             info.narHash = narHash;
             info.narSize = sink.s->size();
-            info.ca = makeFixedOutputCA(true, info.narHash);
+            info.ca = makeFixedOutputCA(FileIngestionMethod::Recursive, info.narHash);
 
             if (!json)
                 printError("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc
index fed012b04..b45951879 100644
--- a/src/nix/optimise-store.cc
+++ b/src/nix/optimise-store.cc
@@ -23,6 +23,8 @@ struct CmdOptimiseStore : StoreCommand
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         store->optimiseStore();
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index a15912ccc..91d62bcec 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -29,6 +29,8 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
         return "query information about store paths";
     }
 
+    Category category() override { return catSecondary; }
+
     Examples examples() override
     {
         return {
diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc
index 3a2e542a3..127397a29 100644
--- a/src/nix/ping-store.cc
+++ b/src/nix/ping-store.cc
@@ -21,6 +21,8 @@ struct CmdPingStore : StoreCommand
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         store->connect();
diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc
index d53e671ed..76c6123f0 100644
--- a/src/nix/progress-bar.cc
+++ b/src/nix/progress-bar.cc
@@ -7,6 +7,7 @@
 #include <atomic>
 #include <map>
 #include <thread>
+#include <iostream>
 
 namespace nix {
 
@@ -442,6 +443,18 @@ public:
 
         return res;
     }
+
+    void writeToStdout(std::string_view s) override
+    {
+        auto state(state_.lock());
+        if (state->active) {
+            std::cerr << "\r\e[K";
+            Logger::writeToStdout(s);
+            draw(*state);
+        } else {
+            Logger::writeToStdout(s);
+        }
+    }
 };
 
 void startProgressBar(bool printBuildLogs)
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 795aa6682..7d66419bd 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -82,40 +82,6 @@ struct NixRepl : gc
 };
 
 
-void printHelp()
-{
-    std::cout
-         << "Usage: nix-repl [--help] [--version] [-I path] paths...\n"
-         << "\n"
-         << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n"
-         << "\n"
-         << "Options:\n"
-         << "    --help\n"
-         << "        Prints out a summary of the command syntax and exits.\n"
-         << "\n"
-         << "    --version\n"
-         << "        Prints out the Nix version number on standard output and exits.\n"
-         << "\n"
-         << "    -I path\n"
-         << "        Add a path to the Nix expression search path. This option may be given\n"
-         << "        multiple times. See the NIX_PATH environment variable for information on\n"
-         << "        the semantics of the Nix search path. Paths added through -I take\n"
-         << "        precedence over NIX_PATH.\n"
-         << "\n"
-         << "    paths...\n"
-         << "        A list of paths to files containing Nix expressions which nix-repl will\n"
-         << "        load and add to its scope.\n"
-         << "\n"
-         << "        A path surrounded in < and > will be looked up in the Nix expression search\n"
-         << "        path, as in the Nix language itself.\n"
-         << "\n"
-         << "        If an element of paths starts with http:// or https://, it is interpreted\n"
-         << "        as the URL of a tarball that will be downloaded and unpacked to a temporary\n"
-         << "        location. The tarball must include a single top-level directory containing\n"
-         << "        at least a file named default.nix.\n";
-}
-
-
 string removeWhitespace(string s)
 {
     s = chomp(s);
@@ -809,6 +775,16 @@ struct CmdRepl : StoreCommand, MixEvalArgs
         return "start an interactive environment for evaluating Nix expressions";
     }
 
+    Examples examples() override
+    {
+        return {
+          Example{
+            "Display all special commands within the REPL:",
+              "nix repl\n  nix-repl> :?"
+          }
+        };
+    }
+
     void run(ref<Store> store) override
     {
         auto repl = std::make_unique<NixRepl>(searchPath, openStore());
diff --git a/src/nix/run.cc b/src/nix/run.cc
index f885c5e49..b888281a5 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -8,6 +8,7 @@
 #include "fs-accessor.hh"
 #include "progress-bar.hh"
 #include "affinity.hh"
+#include "eval.hh"
 
 #if __linux__
 #include <sys/mount.h>
@@ -19,46 +20,59 @@ using namespace nix;
 
 std::string chrootHelperName = "__run_in_chroot";
 
-struct CmdRun : InstallablesCommand
+struct RunCommon : virtual Command
 {
-    std::vector<std::string> command = { "bash" };
-    StringSet keep, unset;
-    bool ignoreEnvironment = false;
+    void runProgram(ref<Store> store,
+        const std::string & program,
+        const Strings & args)
+    {
+        stopProgressBar();
 
-    CmdRun()
+        restoreSignals();
+
+        restoreAffinity();
+
+        /* If this is a diverted store (i.e. its "logical" location
+           (typically /nix/store) differs from its "physical" location
+           (e.g. /home/eelco/nix/store), then run the command in a
+           chroot. For non-root users, this requires running it in new
+           mount and user namespaces. Unfortunately,
+           unshare(CLONE_NEWUSER) doesn't work in a multithreaded
+           program (which "nix" is), so we exec() a single-threaded
+           helper program (chrootHelper() below) to do the work. */
+        auto store2 = store.dynamic_pointer_cast<LocalStore>();
+
+        if (store2 && store->storeDir != store2->realStoreDir) {
+            Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program };
+            for (auto & arg : args) helperArgs.push_back(arg);
+
+            execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());
+
+            throw SysError("could not execute chroot helper");
+        }
+
+        execvp(program.c_str(), stringsToCharPtrs(args).data());
+
+        throw SysError("unable to execute '%s'", program);
+    }
+};
+
+struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
+{
+    std::vector<std::string> command = { getEnv("SHELL").value_or("bash") };
+
+    CmdShell()
     {
-        mkFlag()
-            .longName("command")
-            .shortName('c')
-            .description("command and arguments to be executed; defaults to 'bash'")
-            .labels({"command", "args"})
-            .arity(ArityAny)
-            .handler([&](std::vector<std::string> ss) {
+        addFlag({
+            .longName = "command",
+            .shortName = 'c',
+            .description = "command and arguments to be executed; defaults to '$SHELL'",
+            .labels = {"command", "args"},
+            .handler = {[&](std::vector<std::string> ss) {
                 if (ss.empty()) throw UsageError("--command requires at least one argument");
                 command = ss;
-            });
-
-        mkFlag()
-            .longName("ignore-environment")
-            .shortName('i')
-            .description("clear the entire environment (except those specified with --keep)")
-            .set(&ignoreEnvironment, true);
-
-        mkFlag()
-            .longName("keep")
-            .shortName('k')
-            .description("keep specified environment variable")
-            .arity(1)
-            .labels({"name"})
-            .handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); });
-
-        mkFlag()
-            .longName("unset")
-            .shortName('u')
-            .description("unset specified environment variable")
-            .arity(1)
-            .labels({"name"})
-            .handler([&](std::vector<std::string> ss) { unset.insert(ss.front()); });
+            }}
+        });
     }
 
     std::string description() override
@@ -71,19 +85,19 @@ struct CmdRun : InstallablesCommand
         return {
             Example{
                 "To start a shell providing GNU Hello from NixOS 17.03:",
-                "nix run -f channel:nixos-17.03 hello"
+                "nix shell -f channel:nixos-17.03 hello"
             },
             Example{
                 "To start a shell providing youtube-dl from your 'nixpkgs' channel:",
-                "nix run nixpkgs.youtube-dl"
+                "nix shell nixpkgs.youtube-dl"
             },
             Example{
                 "To run GNU Hello:",
-                "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'"
+                "nix shell nixpkgs.hello -c hello --greeting 'Hi everybody!'"
             },
             Example{
                 "To run GNU Hello in a chroot store:",
-                "nix run --store ~/my-nix nixpkgs.hello -c hello"
+                "nix shell --store ~/my-nix nixpkgs.hello -c hello"
             },
         };
     }
@@ -94,35 +108,13 @@ struct CmdRun : InstallablesCommand
 
         auto accessor = store->getFSAccessor();
 
-        if (ignoreEnvironment) {
-
-            if (!unset.empty())
-                throw UsageError("--unset does not make sense with --ignore-environment");
-
-            std::map<std::string, std::string> kept;
-            for (auto & var : keep) {
-                auto s = getenv(var.c_str());
-                if (s) kept[var] = s;
-            }
-
-            clearEnv();
-
-            for (auto & var : kept)
-                setenv(var.first.c_str(), var.second.c_str(), 1);
-
-        } else {
-
-            if (!keep.empty())
-                throw UsageError("--keep does not make sense without --ignore-environment");
-
-            for (auto & var : unset)
-                unsetenv(var.c_str());
-        }
 
         std::unordered_set<StorePath> done;
         std::queue<StorePath> todo;
         for (auto & path : outPaths) todo.push(path.clone());
 
+        setEnviron();
+
         auto unixPath = tokenizeString<Strings>(getEnv("PATH").value_or(""), ":");
 
         while (!todo.empty()) {
@@ -142,42 +134,14 @@ struct CmdRun : InstallablesCommand
 
         setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1);
 
-        std::string cmd = *command.begin();
         Strings args;
         for (auto & arg : command) args.push_back(arg);
 
-        stopProgressBar();
-
-        restoreSignals();
-
-        restoreAffinity();
-
-        /* If this is a diverted store (i.e. its "logical" location
-           (typically /nix/store) differs from its "physical" location
-           (e.g. /home/eelco/nix/store), then run the command in a
-           chroot. For non-root users, this requires running it in new
-           mount and user namespaces. Unfortunately,
-           unshare(CLONE_NEWUSER) doesn't work in a multithreaded
-           program (which "nix" is), so we exec() a single-threaded
-           helper program (chrootHelper() below) to do the work. */
-        auto store2 = store.dynamic_pointer_cast<LocalStore>();
-
-        if (store2 && store->storeDir != store2->realStoreDir) {
-            Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd };
-            for (auto & arg : args) helperArgs.push_back(arg);
-
-            execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data());
-
-            throw SysError("could not execute chroot helper");
-        }
-
-        execvp(cmd.c_str(), stringsToCharPtrs(args).data());
-
-        throw SysError("unable to exec '%s'", cmd);
+        runProgram(store, *command.begin(), args);
     }
 };
 
-static auto r1 = registerCommand<CmdRun>("run");
+static auto r1 = registerCommand<CmdShell>("shell");
 
 void chrootHelper(int argc, char * * argv)
 {
diff --git a/src/nix/search.cc b/src/nix/search.cc
index 769274543..ba72c1e79 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -40,16 +40,18 @@ struct CmdSearch : SourceExprCommand, MixJSON
     {
         expectArgs("regex", &res);
 
-        mkFlag()
-            .longName("update-cache")
-            .shortName('u')
-            .description("update the package search cache")
-            .handler([&]() { writeCache = true; useCache = false; });
-
-        mkFlag()
-            .longName("no-cache")
-            .description("do not use or update the package search cache")
-            .handler([&]() { writeCache = false; useCache = false; });
+        addFlag({
+            .longName = "update-cache",
+            .shortName = 'u',
+            .description = "update the package search cache",
+            .handler = {[&]() { writeCache = true; useCache = false; }}
+        });
+
+        addFlag({
+            .longName = "no-cache",
+            .description = "do not use or update the package search cache",
+            .handler = {[&]() { writeCache = false; useCache = false; }}
+        });
     }
 
     std::string description() override
@@ -263,7 +265,7 @@ struct CmdSearch : SourceExprCommand, MixJSON
                 throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName);
         }
 
-        if (results.size() == 0)
+        if (!json && results.size() == 0)
             throw Error("no results for the given search term(s)!");
 
         RunPager pager;
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index 87544f937..4fd8886de 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -13,6 +13,8 @@ struct CmdShowConfig : Command, MixJSON
         return "show the Nix configuration";
     }
 
+    Category category() override { return catUtility; }
+
     void run() override
     {
         if (json) {
@@ -23,7 +25,7 @@ struct CmdShowConfig : Command, MixJSON
             std::map<std::string, Config::SettingInfo> settings;
             globalConfig.getSettings(settings);
             for (auto & s : settings)
-                std::cout << s.first + " = " + s.second.value + "\n";
+                logger->stdout("%s = %s", s.first, s.second.value);
         }
     }
 };
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 0ede7b468..22c569f3c 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -15,11 +15,12 @@ struct CmdShowDerivation : InstallablesCommand
 
     CmdShowDerivation()
     {
-        mkFlag()
-            .longName("recursive")
-            .shortName('r')
-            .description("include the dependencies of the specified derivations")
-            .set(&recursive, true);
+        addFlag({
+            .longName = "recursive",
+            .shortName = 'r',
+            .description = "include the dependencies of the specified derivations",
+            .handler = {&recursive, true}
+        });
     }
 
     std::string description() override
@@ -41,6 +42,8 @@ struct CmdShowDerivation : InstallablesCommand
         };
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store) override
     {
         auto drvPaths = toDerivations(store, installables, true);
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 2c3d2a107..311817d1f 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -13,13 +13,13 @@ struct CmdCopySigs : StorePathsCommand
 
     CmdCopySigs()
     {
-        mkFlag()
-            .longName("substituter")
-            .shortName('s')
-            .labels({"store-uri"})
-            .description("use signatures from specified store")
-            .arity(1)
-            .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); });
+        addFlag({
+            .longName = "substituter",
+            .shortName = 's',
+            .description = "use signatures from specified store",
+            .labels = {"store-uri"},
+            .handler = {[&](std::string s) { substituterUris.push_back(s); }},
+        });
     }
 
     std::string description() override
@@ -27,6 +27,8 @@ struct CmdCopySigs : StorePathsCommand
         return "copy path signatures from substituters (like binary caches)";
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store, StorePaths storePaths) override
     {
         if (substituterUris.empty())
@@ -98,12 +100,13 @@ struct CmdSignPaths : StorePathsCommand
 
     CmdSignPaths()
     {
-        mkFlag()
-            .shortName('k')
-            .longName("key-file")
-            .label("file")
-            .description("file containing the secret signing key")
-            .dest(&secretKeyFile);
+        addFlag({
+            .longName = "key-file",
+            .shortName = 'k',
+            .description = "file containing the secret signing key",
+            .labels = {"file"},
+            .handler = {&secretKeyFile}
+        });
     }
 
     std::string description() override
@@ -111,6 +114,8 @@ struct CmdSignPaths : StorePathsCommand
         return "sign the specified paths";
     }
 
+    Category category() override { return catUtility; }
+
     void run(ref<Store> store, StorePaths storePaths) override
     {
         if (secretKeyFile.empty())
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 26de92d32..9018e69b3 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -1,7 +1,7 @@
 #include "command.hh"
 #include "common-args.hh"
 #include "store-api.hh"
-#include "download.hh"
+#include "filetransfer.hh"
 #include "eval.hh"
 #include "attr-path.hh"
 #include "names.hh"
@@ -16,18 +16,20 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
 
     CmdUpgradeNix()
     {
-        mkFlag()
-            .longName("profile")
-            .shortName('p')
-            .labels({"profile-dir"})
-            .description("the Nix profile to upgrade")
-            .dest(&profileDir);
-
-        mkFlag()
-            .longName("nix-store-paths-url")
-            .labels({"url"})
-            .description("URL of the file that contains the store paths of the latest Nix release")
-            .dest(&storePathsUrl);
+        addFlag({
+            .longName = "profile",
+            .shortName = 'p',
+            .description = "the Nix profile to upgrade",
+            .labels = {"profile-dir"},
+            .handler = {&profileDir}
+        });
+
+        addFlag({
+            .longName = "nix-store-paths-url",
+            .description = "URL of the file that contains the store paths of the latest Nix release",
+            .labels = {"url"},
+            .handler = {&storePathsUrl}
+        });
     }
 
     std::string description() override
@@ -49,6 +51,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         };
     }
 
+    Category category() override { return catNixInstallation; }
+
     void run(ref<Store> store) override
     {
         evalSettings.pureEval = true;
@@ -138,8 +142,8 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
         Activity act(*logger, Verbosity::Info, ActivityType::Unknown, "querying latest Nix version");
 
         // FIXME: use nixos.org?
-        auto req = DownloadRequest(storePathsUrl);
-        auto res = getDownloader()->download(req);
+        auto req = FileTransferRequest(storePathsUrl);
+        auto res = getFileTransfer()->download(req);
 
         auto state = std::make_unique<EvalState>(Strings(), store);
         auto v = state->allocValue();
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index 17d4410cf..0c3478ff5 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -20,13 +20,13 @@ struct CmdVerify : StorePathsCommand
     {
         mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents);
         mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust);
-        mkFlag()
-            .longName("substituter")
-            .shortName('s')
-            .labels({"store-uri"})
-            .description("use signatures from specified store")
-            .arity(1)
-            .handler([&](std::vector<std::string> ss) { substituterUris.push_back(ss[0]); });
+        addFlag({
+            .longName = "substituter",
+            .shortName = 's',
+            .description = "use signatures from specified store",
+            .labels = {"store-uri"},
+            .handler = {[&](std::string s) { substituterUris.push_back(s); }}
+        });
         mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded);
     }
 
@@ -49,6 +49,8 @@ struct CmdVerify : StorePathsCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     void run(ref<Store> store, StorePaths storePaths) override
     {
         std::vector<ref<Store>> substituters;
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index d3b7a674a..6057beedb 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -37,11 +37,12 @@ struct CmdWhyDepends : SourceExprCommand
         expectArg("package", &_package);
         expectArg("dependency", &_dependency);
 
-        mkFlag()
-            .longName("all")
-            .shortName('a')
-            .description("show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path")
-            .set(&all, true);
+        addFlag({
+            .longName = "all",
+            .shortName = 'a',
+            .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path",
+            .handler = {&all, true},
+        });
     }
 
     std::string description() override
@@ -67,6 +68,8 @@ struct CmdWhyDepends : SourceExprCommand
         };
     }
 
+    Category category() override { return catSecondary; }
+
     void run(ref<Store> store) override
     {
         auto package = parseInstallable(*this, store, _package, false);
@@ -149,7 +152,7 @@ struct CmdWhyDepends : SourceExprCommand
             auto pathS = store->printStorePath(node.path);
 
             assert(node.dist != inf);
-            std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n",
+            logger->stdout("%s%s%s%s" ANSI_NORMAL,
                 firstPad,
                 node.visited ? "\e[38;5;244m" : "",
                 firstPad != "" ? "→ " : "",