Merge branch 'master' into indexed-store-path-outputs

24 files changed, 274 insertions, 138 deletions

diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index ff8ba2724..6b81ecc49 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -186,12 +186,12 @@ static int main_build_remote(int argc, char * * argv)
                         // build the hint template.
                         std::string errorText =
                             "Failed to find a machine for remote build!\n"
-                            "derivation: %s\nrequired (system, features): (%s, %s)";
+                            "derivation: %s\nrequired (system, features): (%s, [%s])";
                         errorText += "\n%s available machines:";
                         errorText += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
 
                         for (unsigned int i = 0; i < machines.size(); ++i)
-                            errorText += "\n(%s, %s, %s, %s)";
+                            errorText += "\n([%s], %s, [%s], [%s])";
 
                         // add the template values.
                         std::string drvstr;
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index 76f88db9f..176d655be 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -207,55 +207,59 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
 
 void SourceExprCommand::completeInstallable(std::string_view prefix)
 {
-    if (file) {
-        completionType = ctAttrs;
+    try {
+        if (file) {
+            completionType = ctAttrs;
 
-        evalSettings.pureEval = false;
-        auto state = getEvalState();
-        Expr *e = state->parseExprFromFile(
-            resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
-        );
+            evalSettings.pureEval = false;
+            auto state = getEvalState();
+            Expr *e = state->parseExprFromFile(
+                resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file)))
+                );
 
-        Value root;
-        state->eval(e, root);
+            Value root;
+            state->eval(e, root);
 
-        auto autoArgs = getAutoArgs(*state);
+            auto autoArgs = getAutoArgs(*state);
 
-        std::string prefix_ = std::string(prefix);
-        auto sep = prefix_.rfind('.');
-        std::string searchWord;
-        if (sep != std::string::npos) {
-            searchWord = prefix_.substr(sep + 1, std::string::npos);
-            prefix_ = prefix_.substr(0, sep);
-        } else {
-            searchWord = prefix_;
-            prefix_ = "";
-        }
+            std::string prefix_ = std::string(prefix);
+            auto sep = prefix_.rfind('.');
+            std::string searchWord;
+            if (sep != std::string::npos) {
+                searchWord = prefix_.substr(sep + 1, std::string::npos);
+                prefix_ = prefix_.substr(0, sep);
+            } else {
+                searchWord = prefix_;
+                prefix_ = "";
+            }
 
-        auto [v, pos] = findAlongAttrPath(*state, prefix_, *autoArgs, root);
-        Value &v1(*v);
-        state->forceValue(v1, pos);
-        Value v2;
-        state->autoCallFunction(*autoArgs, v1, v2);
-
-        if (v2.type() == nAttrs) {
-            for (auto & i : *v2.attrs) {
-                std::string name = state->symbols[i.name];
-                if (name.find(searchWord) == 0) {
-                    if (prefix_ == "")
-                        completions->add(name);
-                    else
-                        completions->add(prefix_ + "." + name);
+            auto [v, pos] = findAlongAttrPath(*state, prefix_, *autoArgs, root);
+            Value &v1(*v);
+            state->forceValue(v1, pos);
+            Value v2;
+            state->autoCallFunction(*autoArgs, v1, v2);
+
+            if (v2.type() == nAttrs) {
+                for (auto & i : *v2.attrs) {
+                    std::string name = state->symbols[i.name];
+                    if (name.find(searchWord) == 0) {
+                        if (prefix_ == "")
+                            completions->add(name);
+                        else
+                            completions->add(prefix_ + "." + name);
+                    }
                 }
             }
+        } else {
+            completeFlakeRefWithFragment(
+                getEvalState(),
+                lockFlags,
+                getDefaultFlakeAttrPathPrefixes(),
+                getDefaultFlakeAttrPaths(),
+                prefix);
         }
-    } else {
-        completeFlakeRefWithFragment(
-            getEvalState(),
-            lockFlags,
-            getDefaultFlakeAttrPathPrefixes(),
-            getDefaultFlakeAttrPaths(),
-            prefix);
+    } catch (EvalError&) {
+        // Don't want eval errors to mess-up with the completion engine, so let's just swallow them
     }
 }
 
@@ -867,20 +871,20 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
     return installables.front();
 }
 
-BuiltPaths Installable::build(
+std::vector<BuiltPathWithResult> Installable::build(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
     const std::vector<std::shared_ptr<Installable>> & installables,
     BuildMode bMode)
 {
-    BuiltPaths res;
-    for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode))
-        res.push_back(builtPath);
+    std::vector<BuiltPathWithResult> res;
+    for (auto & [_, builtPathWithResult] : build2(evalStore, store, mode, installables, bMode))
+        res.push_back(builtPathWithResult);
     return res;
 }
 
-std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2(
+std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> Installable::build2(
     ref<Store> evalStore,
     ref<Store> store,
     Realise mode,
@@ -900,7 +904,7 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
         }
     }
 
-    std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res;
+    std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> res;
 
     switch (mode) {
 
@@ -941,10 +945,10 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
                                     output, *drvOutput->second);
                             }
                         }
-                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                        res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }}});
                     },
                     [&](const DerivedPath::Opaque & bo) {
-                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                        res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }}});
                     },
                 }, path.raw());
             }
@@ -966,10 +970,10 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui
                         std::map<std::string, StorePath> outputs;
                         for (auto & path : buildResult.builtOutputs)
                             outputs.emplace(path.first.outputName, path.second.outPath);
-                        res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }});
+                        res.push_back({installable, {.path = BuiltPath::Built { bfd.drvPath, outputs }, .result = buildResult}});
                     },
                     [&](const DerivedPath::Opaque & bo) {
-                        res.push_back({installable, BuiltPath::Opaque { bo.path }});
+                        res.push_back({installable, {.path = BuiltPath::Opaque { bo.path }, .result = buildResult}});
                     },
                 }, buildResult.path.raw());
             }
@@ -992,9 +996,12 @@ BuiltPaths Installable::toBuiltPaths(
     OperateOn operateOn,
     const std::vector<std::shared_ptr<Installable>> & installables)
 {
-    if (operateOn == OperateOn::Output)
-        return Installable::build(evalStore, store, mode, installables);
-    else {
+    if (operateOn == OperateOn::Output) {
+        BuiltPaths res;
+        for (auto & p : Installable::build(evalStore, store, mode, installables))
+            res.push_back(p.path);
+        return res;
+    } else {
         if (mode == Realise::Nothing)
             settings.readOnlyMode = true;
 
diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh
index 948f78919..02ea351d3 100644
--- a/src/libcmd/installables.hh
+++ b/src/libcmd/installables.hh
@@ -7,6 +7,7 @@
 #include "eval.hh"
 #include "store-api.hh"
 #include "flake/flake.hh"
+#include "build-result.hh"
 
 #include <optional>
 
@@ -51,6 +52,12 @@ enum class OperateOn {
     Derivation
 };
 
+struct BuiltPathWithResult
+{
+    BuiltPath path;
+    std::optional<BuildResult> result;
+};
+
 struct Installable
 {
     virtual ~Installable() { }
@@ -91,14 +98,14 @@ struct Installable
         return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}});
     }
 
-    static BuiltPaths build(
+    static std::vector<BuiltPathWithResult> build(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
         const std::vector<std::shared_ptr<Installable>> & installables,
         BuildMode bMode = bmNormal);
 
-    static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2(
+    static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> build2(
         ref<Store> evalStore,
         ref<Store> store,
         Realise mode,
diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk
index 3a4de6bcb..152bc388d 100644
--- a/src/libcmd/local.mk
+++ b/src/libcmd/local.mk
@@ -8,7 +8,7 @@ libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
 libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers -I src/nix
 
-libcmd_LDFLAGS = $(EDITLINE_LIBS) -llowdown -pthread
+libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread
 
 libcmd_LIBS = libstore libutil libexpr libmain libfetchers
 
diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc
index df8932087..bb254ff8d 100644
--- a/src/libcmd/repl.cc
+++ b/src/libcmd/repl.cc
@@ -384,6 +384,10 @@ StringSet NixRepl::completePrefix(const std::string & prefix)
             i++;
         }
     } else {
+        /* Temporarily disable the debugger, to avoid re-entering readline. */
+        auto debug_repl = state->debugRepl;
+        state->debugRepl = nullptr;
+        Finally restoreDebug([&]() { state->debugRepl = debug_repl; });
         try {
             /* This is an expression that should evaluate to an
                attribute set.  Evaluate it to get the names of the
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index e3716f217..563f24e48 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -904,7 +904,7 @@ void EvalState::throwEvalError(const char * s, const std::string & s2,
     const std::string & s3)
 {
     debugThrowLastTrace(EvalError({
-        .msg = hintfmt(s, s2),
+        .msg = hintfmt(s, s2, s3),
         .errPos = positions[noPos]
     }));
 }
@@ -913,7 +913,7 @@ void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::stri
     const std::string & s3)
 {
     debugThrowLastTrace(EvalError({
-        .msg = hintfmt(s, s2),
+        .msg = hintfmt(s, s2, s3),
         .errPos = positions[pos]
     }));
 }
@@ -922,7 +922,7 @@ void EvalState::throwEvalError(const PosIdx pos, const char * s, const std::stri
     const std::string & s3, Env & env, Expr & expr)
 {
     debugThrow(EvalError({
-        .msg = hintfmt(s, s2),
+        .msg = hintfmt(s, s2, s3),
         .errPos = positions[pos]
     }), env, expr);
 }
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 840bfecef..22f6ad3cc 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -5,6 +5,7 @@
 #include "globals.hh"
 #include "json-to-value.hh"
 #include "names.hh"
+#include "references.hh"
 #include "store-api.hh"
 #include "util.hh"
 #include "json.hh"
@@ -1542,6 +1543,10 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V
             refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references;
         } catch (Error &) { // FIXME: should be InvalidPathError
         }
+        // Re-scan references to filter down to just the ones that actually occur in the file.
+        auto refsSink = PathRefScanSink::fromPaths(refs);
+        refsSink << s;
+        refs = refsSink.getResultPaths();
     }
     auto context = state.store->printStorePathSet(refs);
     v.mkString(s, context);
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index a491d82a6..2115ce2f5 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -262,17 +262,20 @@ struct GitHubInputScheme : GitArchiveInputScheme
 
     DownloadUrl getDownloadUrl(const Input & input) const override
     {
-        // FIXME: use regular /archive URLs instead? api.github.com
-        // might have stricter rate limits.
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        auto url = fmt(
-            host == "github.com"
-            ? "https://api.%s/repos/%s/%s/tarball/%s"
-            : "https://%s/api/v3/repos/%s/%s/tarball/%s",
-            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+        Headers headers = makeHeadersWithAuthTokens(host);
+        // If we have no auth headers then we default to the public archive
+        // urls so we do not run into rate limits.
+        const auto urlFmt =
+            host != "github.com"
+            ? "https://%s/api/v3/repos/%s/%s/tarball/%s"
+            : headers.empty()
+            ? "https://%s/%s/%s/archive/%s.tar.gz"
+            : "https://api.%s/repos/%s/%s/tarball/%s";
+
+        const auto url = fmt(urlFmt, host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        Headers headers = makeHeadersWithAuthTokens(host);
         return DownloadUrl { url, headers };
     }
 
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index c1cf38565..a58428762 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -33,6 +33,7 @@
 
 namespace nix {
 
+char * * savedArgv;
 
 static bool gcWarning = true;
 
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 41d2e2a1c..00e375fe9 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -528,13 +528,31 @@ void DerivationGoal::inputsRealised()
             /* Add the relevant output closures of the input derivation
                `i' as input paths.  Only add the closures of output paths
                that are specified as inputs. */
-            for (auto & j : wantedDepOutputs)
-                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j }))
+            for (auto & j : wantedDepOutputs) {
+                /* TODO (impure derivations-induced tech debt):
+                   Tracking input derivation outputs statefully through the
+                   goals is error prone and has led to bugs.
+                   For a robust nix, we need to move towards the `else` branch,
+                   which does not rely on goal state to match up with the
+                   reality of the store, which is our real source of truth.
+                   However, the impure derivations feature still relies on this
+                   fragile way of doing things, because its builds do not have
+                   a representation in the store, which is a usability problem
+                   in itself */
+                if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) {
                     worker.store.computeFSClosure(*outPath, inputPaths);
-                else
-                    throw Error(
-                        "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
-                        worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+                }
+                else {
+                    auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath);
+                    auto outMapPath = outMap.find(j);
+                    if (outMapPath == outMap.end()) {
+                        throw Error(
+                            "derivation '%s' requires non-existent output '%s' from input derivation '%s'",
+                            worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
+                    }
+                    worker.store.computeFSClosure(outMapPath->second, inputPaths);
+                }
+            }
         }
     }
 
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 5cea3b590..3439dd2cc 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -1552,6 +1552,22 @@ void setupSeccomp()
         seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
         printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
 
+    if (nativeSystem == "mips64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPS) != 0)
+        printError("unable to add mips seccomp architecture");
+
+    if (nativeSystem == "mips64-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPS64N32) != 0)
+        printError("unable to add mips64-*abin32 seccomp architecture");
+
+    if (nativeSystem == "mips64el-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL) != 0)
+        printError("unable to add mipsel seccomp architecture");
+
+    if (nativeSystem == "mips64el-linux" &&
+        seccomp_arch_add(ctx, SCMP_ARCH_MIPSEL64N32) != 0)
+        printError("unable to add mips64el-*abin32 seccomp architecture");
+
     /* Prevent builders from creating setuid/setgid binaries. */
     for (int perm : { S_ISUID, S_ISGID }) {
         if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index f6a0c01df..7fe797aa1 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -53,28 +53,13 @@ StorePathSet BuiltPath::outPaths() const
     );
 }
 
-template<typename T>
-nlohmann::json stuffToJSON(const std::vector<T> & ts, ref<Store> store) {
-    auto res = nlohmann::json::array();
-    for (const T & t : ts) {
-        std::visit([&res, store](const auto & t) {
-            res.push_back(t.toJSON(store));
-        }, t.raw());
-    }
-    return res;
-}
-
-nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store)
-{ return stuffToJSON<BuiltPath>(buildables, store); }
-nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
-{ return stuffToJSON<DerivedPath>(paths, store); }
-
-
-std::string DerivedPath::Opaque::to_string(const Store & store) const {
+std::string DerivedPath::Opaque::to_string(const Store & store) const
+{
     return store.printStorePath(path);
 }
 
-std::string DerivedPath::Built::to_string(const Store & store) const {
+std::string DerivedPath::Built::to_string(const Store & store) const
+{
     return store.printStorePath(drvPath)
         + "!"
         + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index fab1292a7..706e5dcb4 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -125,7 +125,4 @@ struct BuiltPath : _BuiltPathRaw {
 typedef std::vector<DerivedPath> DerivedPaths;
 typedef std::vector<BuiltPath> BuiltPaths;
 
-nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store);
-nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref<Store> store);
-
 }
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index 1d26ac918..8f28bec6c 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -20,7 +20,7 @@ endif
 $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox)))
 
 ifeq ($(ENABLE_S3), 1)
-	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core
+	libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core -laws-crt-cpp
 endif
 
 ifdef HOST_SOLARIS
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index 34dce092c..3bb297fc8 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -67,20 +67,12 @@ void RefScanSink::operator () (std::string_view data)
 }
 
 
-std::pair<StorePathSet, HashResult> scanForReferences(
-    const std::string & path,
-    const StorePathSet & refs)
-{
-    HashSink hashSink { htSHA256 };
-    auto found = scanForReferences(hashSink, path, refs);
-    auto hash = hashSink.finish();
-    return std::pair<StorePathSet, HashResult>(found, hash);
-}
+PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap)
+    : RefScanSink(std::move(hashes))
+    , backMap(std::move(backMap))
+{ }
 
-StorePathSet scanForReferences(
-    Sink & toTee,
-    const Path & path,
-    const StorePathSet & refs)
+PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs)
 {
     StringSet hashes;
     std::map<std::string, StorePath> backMap;
@@ -92,14 +84,14 @@ StorePathSet scanForReferences(
         hashes.insert(hashPart);
     }
 
-    /* Look for the hashes in the NAR dump of the path. */
-    RefScanSink refsSink(std::move(hashes));
-    TeeSink sink { refsSink, toTee };
-    dumpPath(path, sink);
+    return PathRefScanSink(std::move(hashes), std::move(backMap));
+}
 
+StorePathSet PathRefScanSink::getResultPaths()
+{
     /* Map the hashes found back to their store paths. */
     StorePathSet found;
-    for (auto & i : refsSink.getResult()) {
+    for (auto & i : getResult()) {
         auto j = backMap.find(i);
         assert(j != backMap.end());
         found.insert(j->second);
@@ -109,6 +101,31 @@ StorePathSet scanForReferences(
 }
 
 
+std::pair<StorePathSet, HashResult> scanForReferences(
+    const std::string & path,
+    const StorePathSet & refs)
+{
+    HashSink hashSink { htSHA256 };
+    auto found = scanForReferences(hashSink, path, refs);
+    auto hash = hashSink.finish();
+    return std::pair<StorePathSet, HashResult>(found, hash);
+}
+
+StorePathSet scanForReferences(
+    Sink & toTee,
+    const Path & path,
+    const StorePathSet & refs)
+{
+    PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs);
+    TeeSink sink { refsSink, toTee };
+
+    /* Look for the hashes in the NAR dump of the path. */
+    dumpPath(path, sink);
+
+    return refsSink.getResultPaths();
+}
+
+
 RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink)
     : from(from), to(to), nextSink(nextSink)
 {
diff --git a/src/libstore/references.hh b/src/libstore/references.hh
index a6119c861..6f381f96c 100644
--- a/src/libstore/references.hh
+++ b/src/libstore/references.hh
@@ -27,6 +27,19 @@ public:
     void operator () (std::string_view data) override;
 };
 
+class PathRefScanSink : public RefScanSink
+{
+    std::map<std::string, StorePath> backMap;
+
+    PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap);
+
+public:
+
+    static PathRefScanSink fromPaths(const StorePathSet & refs);
+
+    StorePathSet getResultPaths();
+};
+
 struct RewritingSink : Sink
 {
     std::string from, to, prev;
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index a7db58559..238d0a7a6 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -77,9 +77,7 @@ TarArchive::~TarArchive()
 
 static void extract_archive(TarArchive & archive, const Path & destDir)
 {
-    int flags = ARCHIVE_EXTRACT_FFLAGS
-        | ARCHIVE_EXTRACT_PERM
-        | ARCHIVE_EXTRACT_TIME
+    int flags = ARCHIVE_EXTRACT_TIME
         | ARCHIVE_EXTRACT_SECURE_SYMLINKS
         | ARCHIVE_EXTRACT_SECURE_NODOTDOT;
 
@@ -98,6 +96,10 @@ static void extract_archive(TarArchive & archive, const Path & destDir)
         archive_entry_copy_pathname(entry,
             (destDir + "/" + name).c_str());
 
+        // sources can and do contain dirs with no rx bits
+        if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500)
+            archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500);
+
         // Patch hardlink path
         const char *original_hardlink = archive_entry_hardlink(entry);
         if (original_hardlink) {
diff --git a/src/nix/app.cc b/src/nix/app.cc
index 48de8fb82..5658f2a52 100644
--- a/src/nix/app.cc
+++ b/src/nix/app.cc
@@ -37,11 +37,13 @@ struct InstallableDerivedPath : Installable
  * Return the rewrites that are needed to resolve a string whose context is
  * included in `dependencies`.
  */
-StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies)
+StringPairs resolveRewrites(
+    Store & store,
+    const std::vector<BuiltPathWithResult> & dependencies)
 {
     StringPairs res;
     for (auto & dep : dependencies)
-        if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep))
+        if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path))
             for (auto & [ outputName, outputPath ] : drvDep->outputs)
                 res.emplace(
                     downstreamPlaceholder(store, drvDep->drvPath, outputName),
@@ -53,7 +55,10 @@ StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies)
 /**
  * Resolve the given string assuming the given context.
  */
-std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies)
+std::string resolveString(
+    Store & store,
+    const std::string & toResolve,
+    const std::vector<BuiltPathWithResult> & dependencies)
 {
     auto rewrites = resolveRewrites(store, dependencies);
     return rewriteStrings(toResolve, rewrites);
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 9c648d28e..85b1efc33 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -10,6 +10,33 @@
 
 using namespace nix;
 
+nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
+{
+    auto res = nlohmann::json::array();
+    for (auto & t : paths) {
+        std::visit([&res, store](const auto & t) {
+            res.push_back(t.toJSON(store));
+        }, t.raw());
+    }
+    return res;
+}
+
+nlohmann::json builtPathsWithResultToJSON(const std::vector<BuiltPathWithResult> & buildables, ref<Store> store)
+{
+    auto res = nlohmann::json::array();
+    for (auto & b : buildables) {
+        std::visit([&](const auto & t) {
+            auto j = t.toJSON(store);
+            if (b.result) {
+                j["startTime"] = b.result->startTime;
+                j["stopTime"] = b.result->stopTime;
+            }
+            res.push_back(j);
+        }, b.path.raw());
+    }
+    return res;
+}
+
 struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
 {
     Path outLink = "result";
@@ -78,7 +105,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
             Realise::Outputs,
             installables, buildMode);
 
-        if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump());
+        if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump());
 
         if (outLink != "")
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
@@ -98,7 +125,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                                 store2->addPermRoot(output.second, absPath(symlink));
                             }
                         },
-                    }, buildable.raw());
+                    }, buildable.path.raw());
                 }
 
         if (printOutputPaths) {
@@ -113,11 +140,14 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
                             std::cout << store->printStorePath(output.second) << std::endl;
                         }
                     },
-                }, buildable.raw());
+                }, buildable.path.raw());
             }
         }
 
-        updateProfile(buildables);
+        BuiltPaths buildables2;
+        for (auto & b : buildables)
+            buildables2.push_back(b.path);
+        updateProfile(buildables2);
     }
 };
 
diff --git a/src/nix/daemon.md b/src/nix/daemon.md
index e97016a94..d5cdadf08 100644
--- a/src/nix/daemon.md
+++ b/src/nix/daemon.md
@@ -11,7 +11,7 @@ R""(
 # Description
 
 This command runs the Nix daemon, which is a required component in
-multi-user Nix installations. It performs build actions and other
+multi-user Nix installations. It runs build tasks and other
 operations on the Nix store on behalf of non-root users. Usually you
 don't run the daemon directly; instead it's managed by a service
 management framework such as `systemd`.
diff --git a/src/nix/local.mk b/src/nix/local.mk
index e4ec7634d..0f2f016ec 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -18,7 +18,7 @@ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr
 
 nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd
 
-nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown
+nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS)
 
 $(foreach name, \
   nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \
diff --git a/src/nix/main.cc b/src/nix/main.cc
index f8e93e367..2c6309c81 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -53,7 +53,6 @@ static bool haveInternet()
 }
 
 std::string programPath;
-char * * savedArgv;
 
 struct HelpRequested { };
 
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 3814e7d5a..11910523d 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -253,11 +253,11 @@ struct ProfileManifest
 
 static std::map<Installable *, BuiltPaths>
 builtPathsPerInstallable(
-    const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> & builtPaths)
+    const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPathWithResult>> & builtPaths)
 {
     std::map<Installable *, BuiltPaths> res;
     for (auto & [installable, builtPath] : builtPaths)
-        res[installable.get()].push_back(builtPath);
+        res[installable.get()].push_back(builtPath.path);
     return res;
 }
 
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 1d9ab28ba..723017497 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -83,20 +83,47 @@ struct CmdWhyDepends : SourceExprCommand
     {
         auto package = parseInstallable(store, _package);
         auto packagePath = Installable::toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, package);
+
+        /* We don't need to build `dependency`. We try to get the store
+         * path if it's already known, and if not, then it's not a dependency.
+         *
+         * Why? If `package` does depends on `dependency`, then getting the
+         * store path of `package` above necessitated having the store path
+         * of `dependency`. The contrapositive is, if the store path of
+         * `dependency` is not already known at this point (i.e. it's a CA
+         * derivation which hasn't been built), then `package` did not need it
+         * to build.
+         */
         auto dependency = parseInstallable(store, _dependency);
-        auto dependencyPath = Installable::toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency);
-        auto dependencyPathHash = dependencyPath.hashPart();
+        auto derivedDependency = dependency->toDerivedPath();
+        auto optDependencyPath = std::visit(overloaded {
+            [](const DerivedPath::Opaque & nodrv) -> std::optional<StorePath> {
+                return { nodrv.path };
+            },
+            [&](const DerivedPath::Built & hasdrv) -> std::optional<StorePath> {
+                if (hasdrv.outputs.size() != 1) {
+                    throw Error("argument '%s' should evaluate to one store path", dependency->what());
+                }
+                auto outputMap = store->queryPartialDerivationOutputMap(hasdrv.drvPath);
+                auto maybePath = outputMap.find(*hasdrv.outputs.begin());
+                if (maybePath == outputMap.end()) {
+                    throw Error("unexpected end of iterator");
+                }
+                return maybePath->second;
+            },
+        }, derivedDependency.raw());
 
         StorePathSet closure;
         store->computeFSClosure({packagePath}, closure, false, false);
 
-        if (!closure.count(dependencyPath)) {
-            printError("'%s' does not depend on '%s'",
-                store->printStorePath(packagePath),
-                store->printStorePath(dependencyPath));
+        if (!optDependencyPath.has_value() || !closure.count(*optDependencyPath)) {
+            printError("'%s' does not depend on '%s'", package->what(), dependency->what());
             return;
         }
 
+        auto dependencyPath = *optDependencyPath;
+        auto dependencyPathHash = dependencyPath.hashPart();
+
         stopProgressBar(); // FIXME
 
         auto accessor = store->getFSAccessor();