Merge remote-tracking branch 'upstream/master' into trustless-remote-builder-simple

6 files changed, 291 insertions, 232 deletions

diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index d29954f67..fd3edfc46 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -54,7 +54,31 @@ void StoreCommand::run()
     run(getStore());
 }
 
-RealisedPathsCommand::RealisedPathsCommand(bool recursive)
+EvalCommand::EvalCommand()
+{
+}
+
+EvalCommand::~EvalCommand()
+{
+    if (evalState)
+        evalState->printStats();
+}
+
+ref<Store> EvalCommand::getEvalStore()
+{
+    if (!evalStore)
+        evalStore = evalStoreUrl ? openStore(*evalStoreUrl) : getStore();
+    return ref<Store>(evalStore);
+}
+
+ref<EvalState> EvalCommand::getEvalState()
+{
+    if (!evalState)
+        evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore());
+    return ref<EvalState>(evalState);
+}
+
+BuiltPathsCommand::BuiltPathsCommand(bool recursive)
     : recursive(recursive)
 {
     if (recursive)
@@ -81,44 +105,53 @@ RealisedPathsCommand::RealisedPathsCommand(bool recursive)
     });
 }
 
-void RealisedPathsCommand::run(ref<Store> store)
+void BuiltPathsCommand::run(ref<Store> store)
 {
-    std::vector<RealisedPath> paths;
+    BuiltPaths paths;
     if (all) {
         if (installables.size())
             throw UsageError("'--all' does not expect arguments");
         // XXX: Only uses opaque paths, ignores all the realisations
         for (auto & p : store->queryAllValidPaths())
-            paths.push_back(p);
+            paths.push_back(BuiltPath::Opaque{p});
     } else {
-        auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables);
+        paths = toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables);
         if (recursive) {
-            auto roots = std::move(pathSet);
-            pathSet = {};
-            RealisedPath::closure(*store, roots, pathSet);
+            // XXX: This only computes the store path closure, ignoring
+            // intermediate realisations
+            StorePathSet pathsRoots, pathsClosure;
+            for (auto & root : paths) {
+                auto rootFromThis = root.outPaths();
+                pathsRoots.insert(rootFromThis.begin(), rootFromThis.end());
+            }
+            store->computeFSClosure(pathsRoots, pathsClosure);
+            for (auto & path : pathsClosure)
+                paths.push_back(BuiltPath::Opaque{path});
         }
-        for (auto & path : pathSet)
-            paths.push_back(path);
     }
 
     run(store, std::move(paths));
 }
 
 StorePathsCommand::StorePathsCommand(bool recursive)
-    : RealisedPathsCommand(recursive)
+    : BuiltPathsCommand(recursive)
 {
 }
 
-void StorePathsCommand::run(ref<Store> store, std::vector<RealisedPath> paths)
+void StorePathsCommand::run(ref<Store> store, BuiltPaths && paths)
 {
-    StorePaths storePaths;
-    for (auto & p : paths)
-        storePaths.push_back(p.path());
+    StorePathSet storePaths;
+    for (auto & builtPath : paths)
+        for (auto & p : builtPath.outPaths())
+            storePaths.insert(p);
+
+    auto sorted = store->topoSortPaths(storePaths);
+    std::reverse(sorted.begin(), sorted.end());
 
-    run(store, std::move(storePaths));
+    run(store, std::move(sorted));
 }
 
-void StorePathCommand::run(ref<Store> store, std::vector<StorePath> storePaths)
+void StorePathCommand::run(ref<Store> store, std::vector<StorePath> && storePaths)
 {
     if (storePaths.size() != 1)
         throw UsageError("this command requires exactly one store path");
@@ -162,7 +195,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
             profile2, storePath));
 }
 
-void MixProfile::updateProfile(const Buildables & buildables)
+void MixProfile::updateProfile(const BuiltPaths & buildables)
 {
     if (!profile) return;
 
@@ -170,22 +203,19 @@ void MixProfile::updateProfile(const Buildables & buildables)
 
     for (auto & buildable : buildables) {
         std::visit(overloaded {
-            [&](BuildableOpaque bo) {
+            [&](const BuiltPath::Opaque & bo) {
                 result.push_back(bo.path);
             },
-            [&](BuildableFromDrv bfd) {
+            [&](const BuiltPath::Built & bfd) {
                 for (auto & output : bfd.outputs) {
-                    /* Output path should be known because we just tried to
-                       build it. */
-                    assert(output.second);
-                    result.push_back(*output.second);
+                    result.push_back(output.second);
                 }
             },
-        }, buildable);
+        }, buildable.raw());
     }
 
     if (result.size() != 1)
-        throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
+        throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size());
 
     updateProfile(result[0]);
 }
diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh
index c02193924..07f398468 100644
--- a/src/libcmd/command.hh
+++ b/src/libcmd/command.hh
@@ -45,8 +45,17 @@ private:
 
 struct EvalCommand : virtual StoreCommand, MixEvalArgs
 {
+    EvalCommand();
+
+    ~EvalCommand();
+
+    ref<Store> getEvalStore();
+
     ref<EvalState> getEvalState();
 
+private:
+    std::shared_ptr<Store> evalStore;
+
     std::shared_ptr<EvalState> evalState;
 };
 
@@ -99,6 +108,8 @@ enum class Realise {
        exists. */
     Derivation,
     /* Evaluate in dry-run mode. Postcondition: nothing. */
+    // FIXME: currently unused, but could be revived if we can
+    // evaluate derivations in-memory.
     Nothing
 };
 
@@ -141,7 +152,7 @@ private:
 };
 
 /* A command that operates on zero or more store paths. */
-struct RealisedPathsCommand : public InstallablesCommand
+struct BuiltPathsCommand : public InstallablesCommand
 {
 private:
 
@@ -154,26 +165,26 @@ protected:
 
 public:
 
-    RealisedPathsCommand(bool recursive = false);
+    BuiltPathsCommand(bool recursive = false);
 
     using StoreCommand::run;
 
-    virtual void run(ref<Store> store, std::vector<RealisedPath> paths) = 0;
+    virtual void run(ref<Store> store, BuiltPaths && paths) = 0;
 
     void run(ref<Store> store) override;
 
     bool useDefaultInstallables() override { return !all; }
 };
 
-struct StorePathsCommand : public RealisedPathsCommand
+struct StorePathsCommand : public BuiltPathsCommand
 {
     StorePathsCommand(bool recursive = false);
 
-    using RealisedPathsCommand::run;
+    using BuiltPathsCommand::run;
 
-    virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
+    virtual void run(ref<Store> store, std::vector<StorePath> && storePaths) = 0;
 
-    void run(ref<Store> store, std::vector<RealisedPath> paths) override;
+    void run(ref<Store> store, BuiltPaths && paths) override;
 };
 
 /* A command that operates on exactly one store path. */
@@ -183,7 +194,7 @@ struct StorePathCommand : public StorePathsCommand
 
     virtual void run(ref<Store> store, const StorePath & storePath) = 0;
 
-    void run(ref<Store> store, std::vector<StorePath> storePaths) override;
+    void run(ref<Store> store, std::vector<StorePath> && storePaths) override;
 };
 
 /* A helper class for registering commands globally. */
@@ -214,26 +225,37 @@ static RegisterCommand registerCommand2(std::vector<std::string> && name)
     return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
 }
 
-Buildables build(ref<Store> store, Realise mode,
-    std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
+BuiltPaths build(
+    ref<Store> evalStore,
+    ref<Store> store, Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode = bmNormal);
 
-std::set<StorePath> toStorePaths(ref<Store> store,
-    Realise mode, OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables);
+std::set<StorePath> toStorePaths(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    OperateOn operateOn,
+    const std::vector<std::shared_ptr<Installable>> & installables);
 
-StorePath toStorePath(ref<Store> store,
-    Realise mode, OperateOn operateOn,
+StorePath toStorePath(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    OperateOn operateOn,
     std::shared_ptr<Installable> installable);
 
-std::set<StorePath> toDerivations(ref<Store> store,
-    std::vector<std::shared_ptr<Installable>> installables,
+std::set<StorePath> toDerivations(
+    ref<Store> store,
+    const std::vector<std::shared_ptr<Installable>> & installables,
     bool useDeriver = false);
 
-std::set<RealisedPath> toRealisedPaths(
+BuiltPaths toBuiltPaths(
+    ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables);
+    const std::vector<std::shared_ptr<Installable>> & installables);
 
 /* Helper function to generate args that invoke $EDITOR on
    filename:lineno. */
@@ -250,7 +272,7 @@ struct MixProfile : virtual StoreCommand
 
     /* If 'profile' is set, make it point at the store path produced
        by 'buildables'. */
-    void updateProfile(const Buildables & buildables);
+    void updateProfile(const BuiltPaths & buildables);
 };
 
 struct MixDefaultProfile : MixProfile
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 4739dc974..0f0fcf39e 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -20,31 +20,6 @@
 
 namespace nix {
 
-nlohmann::json BuildableOpaque::toJSON(ref<Store> store) const {
-    nlohmann::json res;
-    res["path"] = store->printStorePath(path);
-    return res;
-}
-
-nlohmann::json BuildableFromDrv::toJSON(ref<Store> store) const {
-    nlohmann::json res;
-    res["drvPath"] = store->printStorePath(drvPath);
-    for (const auto& [output, path] : outputs) {
-        res["outputs"][output] = path ? store->printStorePath(*path) : "";
-    }
-    return res;
-}
-
-nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store) {
-    auto res = nlohmann::json::array();
-    for (const Buildable & buildable : buildables) {
-        std::visit([&res, store](const auto & buildable) {
-            res.push_back(buildable.toJSON(store));
-        }, buildable);
-    }
-    return res;
-}
-
 void completeFlakeInputPath(
     ref<EvalState> evalState,
     const FlakeRef & flakeRef,
@@ -83,9 +58,13 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "no-registries",
-        .description = "Don't allow lookups in the flake registries.",
+        .description =
+          "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.",
         .category = category,
-        .handler = {&lockFlags.useRegistries, false}
+        .handler = {[&]() {
+            lockFlags.useRegistries = false;
+            warn("'--no-registries' is deprecated; use '--no-use-registries'");
+        }}
     });
 
     addFlag({
@@ -111,10 +90,11 @@ MixFlakeOptions::MixFlakeOptions()
 
     addFlag({
         .longName = "override-input",
-        .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).",
+        .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
         .category = category,
         .labels = {"input-path", "flake-url"},
         .handler = {[&](std::string inputPath, std::string flakeRef) {
+            lockFlags.writeLockFile = false;
             lockFlags.inputOverrides.insert_or_assign(
                 flake::parseInputPath(inputPath),
                 parseFlakeRef(flakeRef, absPath(".")));
@@ -195,14 +175,50 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
 
 void SourceExprCommand::completeInstallable(std::string_view prefix)
 {
-    if (file) return; // FIXME
+    if (file) {
+        evalSettings.pureEval = false;
+        auto state = getEvalState();
+        Expr *e = state->parseExprFromFile(
+            resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
+        );
+
+        Value root;
+        state->eval(e, root);
 
-    completeFlakeRefWithFragment(
-        getEvalState(),
-        lockFlags,
-        getDefaultFlakeAttrPathPrefixes(),
-        getDefaultFlakeAttrPaths(),
-        prefix);
+        auto autoArgs = getAutoArgs(*state);
+
+        std::string prefix_ = std::string(prefix);
+        auto sep = prefix_.rfind('.');
+        std::string searchWord;
+        if (sep != std::string::npos) {
+            searchWord = prefix_.substr(sep, std::string::npos);
+            prefix_ = prefix_.substr(0, sep);
+        } else {
+            searchWord = prefix_;
+            prefix_ = "";
+        }
+
+        Value &v1(*findAlongAttrPath(*state, prefix_, *autoArgs, root).first);
+        state->forceValue(v1);
+        Value v2;
+        state->autoCallFunction(*autoArgs, v1, v2);
+
+        if (v2.type() == nAttrs) {
+            for (auto & i : *v2.attrs) {
+                std::string name = i.name;
+                if (name.find(searchWord) == 0) {
+                    completions->add(i.name);
+                }
+            }
+        }
+    } else {
+        completeFlakeRefWithFragment(
+            getEvalState(),
+            lockFlags,
+            getDefaultFlakeAttrPathPrefixes(),
+            getDefaultFlakeAttrPaths(),
+            prefix);
+    }
 }
 
 void completeFlakeRefWithFragment(
@@ -273,13 +289,6 @@ void completeFlakeRefWithFragment(
     completeFlakeRef(evalState->store, prefix);
 }
 
-ref<EvalState> EvalCommand::getEvalState()
-{
-    if (!evalState)
-        evalState = std::make_shared<EvalState>(searchPath, getStore());
-    return ref<EvalState>(evalState);
-}
-
 void completeFlakeRef(ref<Store> store, std::string_view prefix)
 {
     if (prefix == "")
@@ -303,9 +312,9 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
     }
 }
 
-Buildable Installable::toBuildable()
+DerivedPath Installable::toDerivedPath()
 {
-    auto buildables = toBuildables();
+    auto buildables = toDerivedPaths();
     if (buildables.size() != 1)
         throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size());
     return std::move(buildables[0]);
@@ -339,22 +348,19 @@ struct InstallableStorePath : Installable
 
     std::string what() override { return store->printStorePath(storePath); }
 
-    Buildables toBuildables() override
+    DerivedPaths toDerivedPaths() override
     {
         if (storePath.isDerivation()) {
-            std::map<std::string, std::optional<StorePath>> outputs;
             auto drv = store->readDerivation(storePath);
-            for (auto & [name, output] : drv.outputsAndOptPaths(*store))
-                outputs.emplace(name, output.second);
             return {
-                BuildableFromDrv {
+                DerivedPath::Built {
                     .drvPath = storePath,
-                    .outputs = std::move(outputs)
+                    .outputs = drv.outputNames(),
                 }
             };
         } else {
             return {
-                BuildableOpaque {
+                DerivedPath::Opaque {
                     .path = storePath,
                 }
             };
@@ -367,22 +373,24 @@ struct InstallableStorePath : Installable
     }
 };
 
-Buildables InstallableValue::toBuildables()
+DerivedPaths InstallableValue::toDerivedPaths()
 {
-    Buildables res;
+    DerivedPaths res;
 
-    std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
+    std::map<StorePath, std::set<std::string>> drvsToOutputs;
+    RealisedPath::Set drvsToCopy;
 
     // Group by derivation, helps with .all in particular
     for (auto & drv : toDerivations()) {
         auto outputName = drv.outputName;
         if (outputName == "")
             throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(drv.drvPath));
-        drvsToOutputs[drv.drvPath].insert_or_assign(outputName, drv.outPath);
+        drvsToOutputs[drv.drvPath].insert(outputName);
+        drvsToCopy.insert(drv.drvPath);
     }
 
     for (auto & i : drvsToOutputs)
-        res.push_back(BuildableFromDrv { i.first, i.second });
+        res.push_back(DerivedPath::Built { i.first, i.second });
 
     return res;
 }
@@ -521,7 +529,11 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto root = cache->getRoot();
 
     for (auto & attrPath : getActualAttrPaths()) {
-        auto attr = root->findAlongAttrPath(parseAttrPath(*state, attrPath));
+        auto attr = root->findAlongAttrPath(
+            parseAttrPath(*state, attrPath),
+            true
+        );
+
         if (!attr) continue;
 
         if (!attr->isDerivation())
@@ -590,10 +602,10 @@ InstallableFlake::getCursors(EvalState & state)
 
 std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
 {
+    flake::LockFlags lockFlagsApplyConfig = lockFlags;
+    lockFlagsApplyConfig.applyNixConfig = true;
     if (!_lockedFlake) {
-        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags));
-        _lockedFlake->flake.config.apply();
-        // FIXME: send new config to the daemon.
+        _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig));
     }
     return _lockedFlake;
 }
@@ -642,6 +654,17 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
         for (auto & s : ss) {
             std::exception_ptr ex;
 
+            if (s.find('/') != std::string::npos) {
+                try {
+                    result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
+                    continue;
+                } catch (BadStorePath &) {
+                } catch (...) {
+                    if (!ex)
+                        ex = std::current_exception();
+                }
+            }
+
             try {
                 auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
                 result.push_back(std::make_shared<InstallableFlake>(
@@ -656,25 +679,7 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
                 ex = std::current_exception();
             }
 
-            if (s.find('/') != std::string::npos) {
-                try {
-                    result.push_back(std::make_shared<InstallableStorePath>(store, store->followLinksToStorePath(s)));
-                    continue;
-                } catch (BadStorePath &) {
-                } catch (...) {
-                    if (!ex)
-                        ex = std::current_exception();
-                }
-            }
-
             std::rethrow_exception(ex);
-
-            /*
-            throw Error(
-                pathExists(s)
-                ? "path '%s' is not a flake or a store path"
-                : "don't know how to handle argument '%s'", s);
-            */
         }
     }
 
@@ -689,108 +694,121 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-Buildables build(ref<Store> store, Realise mode,
-    std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
+BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths)
+{
+    BuiltPaths res;
+    for (const auto & b : hopefullyBuiltPaths)
+        std::visit(
+            overloaded{
+                [&](const DerivedPath::Opaque & bo) {
+                    res.push_back(BuiltPath::Opaque{bo.path});
+                },
+                [&](const DerivedPath::Built & bfd) {
+                    OutputPathMap outputs;
+                    auto drv = evalStore->readDerivation(bfd.drvPath);
+                    auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
+                    auto drvOutputs = drv.outputsAndOptPaths(*store);
+                    for (auto & output : bfd.outputs) {
+                        if (!outputHashes.count(output))
+                            throw Error(
+                                "the derivation '%s' doesn't have an output named '%s'",
+                                store->printStorePath(bfd.drvPath), output);
+                        if (settings.isExperimentalFeatureEnabled(
+                                "ca-derivations")) {
+                            auto outputId =
+                                DrvOutput{outputHashes.at(output), output};
+                            auto realisation =
+                                store->queryRealisation(outputId);
+                            if (!realisation)
+                                throw Error(
+                                    "cannot operate on an output of unbuilt "
+                                    "content-addressed derivation '%s'",
+                                    outputId.to_string());
+                            outputs.insert_or_assign(
+                                output, realisation->outPath);
+                        } else {
+                            // If ca-derivations isn't enabled, assume that
+                            // the output path is statically known.
+                            assert(drvOutputs.count(output));
+                            assert(drvOutputs.at(output).second);
+                            outputs.insert_or_assign(
+                                output, *drvOutputs.at(output).second);
+                        }
+                    }
+                    res.push_back(BuiltPath::Built{bfd.drvPath, outputs});
+                },
+            },
+            b.raw());
+
+    return res;
+}
+
+BuiltPaths build(
+    ref<Store> evalStore,
+    ref<Store> store,
+    Realise mode,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    BuildMode bMode)
 {
     if (mode == Realise::Nothing)
         settings.readOnlyMode = true;
 
-    Buildables buildables;
-
-    std::vector<StorePathWithOutputs> pathsToBuild;
+    std::vector<DerivedPath> pathsToBuild;
 
     for (auto & i : installables) {
-        for (auto & b : i->toBuildables()) {
-            std::visit(overloaded {
-                [&](BuildableOpaque bo) {
-                    pathsToBuild.push_back({bo.path});
-                },
-                [&](BuildableFromDrv bfd) {
-                    StringSet outputNames;
-                    for (auto & output : bfd.outputs)
-                        outputNames.insert(output.first);
-                    pathsToBuild.push_back({bfd.drvPath, outputNames});
-                },
-            }, b);
-            buildables.push_back(std::move(b));
-        }
+        auto b = i->toDerivedPaths();
+        pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
     }
 
-    if (mode == Realise::Nothing)
+    if (mode == Realise::Nothing || mode == Realise::Derivation)
         printMissing(store, pathsToBuild, lvlError);
     else if (mode == Realise::Outputs)
-        store->buildPaths(pathsToBuild, bMode);
+        store->buildPaths(pathsToBuild, bMode, evalStore);
 
-    return buildables;
+    return getBuiltPaths(evalStore, store, pathsToBuild);
 }
 
-std::set<RealisedPath> toRealisedPaths(
+BuiltPaths toBuiltPaths(
+    ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables)
+    const std::vector<std::shared_ptr<Installable>> & installables)
 {
-    std::set<RealisedPath> res;
-    if (operateOn == OperateOn::Output) {
-        for (auto & b : build(store, mode, installables))
-            std::visit(overloaded {
-                [&](BuildableOpaque bo) {
-                    res.insert(bo.path);
-                },
-                [&](BuildableFromDrv bfd) {
-                    auto drv = store->readDerivation(bfd.drvPath);
-                    auto outputHashes = staticOutputHashes(*store, drv);
-                    for (auto & output : bfd.outputs) {
-                        if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
-                            if (!outputHashes.count(output.first))
-                                throw Error(
-                                    "the derivation '%s' doesn't have an output named '%s'",
-                                    store->printStorePath(bfd.drvPath),
-                                    output.first);
-                            auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
-                            auto realisation = store->queryRealisation(outputId);
-                            if (!realisation)
-                                throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
-                            res.insert(RealisedPath{*realisation});
-                        }
-                        else {
-                            // If ca-derivations isn't enabled, behave as if
-                            // all the paths are opaque to keep the default
-                            // behavior
-                            assert(output.second);
-                            res.insert(*output.second);
-                        }
-                    }
-                },
-            }, b);
-    } else {
+    if (operateOn == OperateOn::Output)
+        return build(evalStore, store, mode, installables);
+    else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
-        for (auto & i : installables)
-            for (auto & b : i->toBuildables())
-                if (auto bfd = std::get_if<BuildableFromDrv>(&b))
-                    res.insert(bfd->drvPath);
+        BuiltPaths res;
+        for (auto & drvPath : toDerivations(store, installables, true))
+            res.push_back(BuiltPath::Opaque{drvPath});
+        return res;
     }
-
-    return res;
 }
 
-StorePathSet toStorePaths(ref<Store> store,
+StorePathSet toStorePaths(
+    ref<Store> evalStore,
+    ref<Store> store,
     Realise mode, OperateOn operateOn,
-    std::vector<std::shared_ptr<Installable>> installables)
+    const std::vector<std::shared_ptr<Installable>> & installables)
 {
     StorePathSet outPaths;
-    for (auto & path : toRealisedPaths(store, mode, operateOn, installables))
-            outPaths.insert(path.path());
+    for (auto & path : toBuiltPaths(evalStore, store, mode, operateOn, installables)) {
+        auto thisOutPaths = path.outPaths();
+        outPaths.insert(thisOutPaths.begin(), thisOutPaths.end());
+    }
     return outPaths;
 }
 
-StorePath toStorePath(ref<Store> store,
+StorePath toStorePath(
+    ref<Store> evalStore,
+    ref<Store> store,
     Realise mode, OperateOn operateOn,
     std::shared_ptr<Installable> installable)
 {
-    auto paths = toStorePaths(store, mode, operateOn, {installable});
+    auto paths = toStorePaths(evalStore, store, mode, operateOn, {installable});
 
     if (paths.size() != 1)
         throw Error("argument '%s' should evaluate to one store path", installable->what());
@@ -798,15 +816,17 @@ StorePath toStorePath(ref<Store> store,
     return *paths.begin();
 }
 
-StorePathSet toDerivations(ref<Store> store,
-    std::vector<std::shared_ptr<Installable>> installables, bool useDeriver)
+StorePathSet toDerivations(
+    ref<Store> store,
+    const std::vector<std::shared_ptr<Installable>> & installables,
+    bool useDeriver)
 {
     StorePathSet drvPaths;
 
-    for (auto & i : installables)
-        for (auto & b : i->toBuildables())
+    for (const auto & i : installables)
+        for (const auto & b : i->toDerivedPaths())
             std::visit(overloaded {
-                [&](BuildableOpaque bo) {
+                [&](const DerivedPath::Opaque & bo) {
                     if (!useDeriver)
                         throw Error("argument '%s' did not evaluate to a derivation", i->what());
                     auto derivers = store->queryValidDerivers(bo.path);
@@ -815,10 +835,10 @@ StorePathSet toDerivations(ref<Store> store,
                     // FIXME: use all derivers?
                     drvPaths.insert(*derivers.begin());
                 },
-                [&](BuildableFromDrv bfd) {
+                [&](const DerivedPath::Built & bfd) {
                     drvPaths.insert(bfd.drvPath);
                 },
-            }, b);
+            }, b.raw());
 
     return drvPaths;
 }
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index b714f097b..79931ad3e 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -2,13 +2,13 @@
 
 #include "util.hh"
 #include "path.hh"
+#include "path-with-outputs.hh"
+#include "derived-path.hh"
 #include "eval.hh"
 #include "flake/flake.hh"
 
 #include <optional>
 
-#include <nlohmann/json_fwd.hpp>
-
 namespace nix {
 
 struct DrvInfo;
@@ -16,25 +16,6 @@ struct SourceExprCommand;
 
 namespace eval_cache { class EvalCache; class AttrCursor; }
 
-struct BuildableOpaque {
-    StorePath path;
-    nlohmann::json toJSON(ref<Store> store) const;
-};
-
-struct BuildableFromDrv {
-    StorePath drvPath;
-    std::map<std::string, std::optional<StorePath>> outputs;
-    nlohmann::json toJSON(ref<Store> store) const;
-};
-
-typedef std::variant<
-    BuildableOpaque,
-    BuildableFromDrv
-> Buildable;
-
-typedef std::vector<Buildable> Buildables;
-nlohmann::json buildablesToJSON(const Buildables & buildables, ref<Store> store);
-
 struct App
 {
     std::vector<StorePathWithOutputs> context;
@@ -42,17 +23,23 @@ struct App
     // FIXME: add args, sandbox settings, metadata, ...
 };
 
+struct UnresolvedApp
+{
+    App unresolved;
+    App resolve(ref<Store> evalStore, ref<Store> store);
+};
+
 struct Installable
 {
     virtual ~Installable() { }
 
     virtual std::string what() = 0;
 
-    virtual Buildables toBuildables() = 0;
+    virtual DerivedPaths toDerivedPaths() = 0;
 
-    Buildable toBuildable();
+    DerivedPath toDerivedPath();
 
-    App toApp(EvalState & state);
+    UnresolvedApp toApp(EvalState & state);
 
     virtual std::pair<Value *, Pos> toValue(EvalState & state)
     {
@@ -93,7 +80,7 @@ struct InstallableValue : Installable
 
     virtual std::vector<DerivationInfo> toDerivations() = 0;
 
-    Buildables toBuildables() override;
+    DerivedPaths toDerivedPaths() override;
 };
 
 struct InstallableFlake : InstallableValue
diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk
index ab0e0e43d..8b0662753 100644
--- a/src/libcmd/local.mk
+++ b/src/libcmd/local.mk
@@ -8,8 +8,8 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
 libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
 
-libcmd_LDFLAGS = -llowdown
+libcmd_LDFLAGS += -llowdown -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
-$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(prefix)/lib/pkgconfig, 0644))
+$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644))
diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc
index d25113d93..29bb4d31e 100644
--- a/src/libcmd/markdown.cc
+++ b/src/libcmd/markdown.cc
@@ -12,7 +12,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     struct lowdown_opts opts {
         .type = LOWDOWN_TERM,
         .maxdepth = 20,
-        .cols = std::min(getWindowSize().second, (unsigned short) 80),
+        .cols = std::max(getWindowSize().second, (unsigned short) 80),
         .hmargin = 0,
         .vmargin = 0,
         .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES,
@@ -25,7 +25,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     Finally freeDoc([&]() { lowdown_doc_free(doc); });
 
     size_t maxn = 0;
-    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size());
+    auto node = lowdown_doc_parse(doc, &maxn, markdown.data(), markdown.size(), nullptr);
     if (!node)
         throw Error("cannot parse Markdown document");
     Finally freeNode([&]() { lowdown_node_free(node); });
@@ -40,11 +40,11 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
         throw Error("cannot allocate Markdown output buffer");
     Finally freeBuffer([&]() { lowdown_buf_free(buf); });
 
-    int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node);
+    int rndr_res = lowdown_term_rndr(buf, renderer, node);
     if (!rndr_res)
         throw Error("allocation error while rendering Markdown");
 
-    return std::string(buf->data, buf->size);
+    return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
 }
 
 }