fetchClosure: Allow a path to be rewritten to CA on the fly

The advantage is that the resulting closure doesn't need to be signed,
so you don't need to configure any binary cache keys on the client.

4 files changed, 151 insertions, 73 deletions

diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
index 8e60b2ccc..56fd53ed5 100644
--- a/src/libexpr/primops/fetchClosure.cc
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -1,5 +1,6 @@
 #include "primops.hh"
 #include "store-api.hh"
+#include "make-content-addressed.hh"
 
 namespace nix {
 
@@ -9,6 +10,8 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args
 
     std::optional<std::string> fromStoreUrl;
     std::optional<StorePath> fromPath;
+    bool toCA = false;
+    std::optional<StorePath> toPath;
 
     for (auto & attr : *args[0]->attrs) {
         if (attr.name == "fromPath") {
@@ -16,6 +19,15 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args
             fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
         }
 
+        else if (attr.name == "toPath") {
+            state.forceValue(*attr.value, *attr.pos);
+            toCA = true;
+            if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
+                PathSet context;
+                toPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+            }
+        }
+
         else if (attr.name == "fromStore")
             fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos);
 
@@ -41,21 +53,45 @@ static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args
     // FIXME: only allow some "trusted" store types (like BinaryCacheStore).
     auto fromStore = openStore(*fromStoreUrl);
 
-    copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
+    if (toCA) {
+        auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
+        auto i = remappings.find(*fromPath);
+        assert(i != remappings.end());
+        if (toPath && *toPath != i->second)
+            throw Error({
+                .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
+                    state.store->printStorePath(*fromPath),
+                    state.store->printStorePath(i->second),
+                    state.store->printStorePath(*toPath)),
+                .errPos = pos
+            });
+        if (!toPath)
+            throw Error({
+                .msg = hintfmt(
+                    "rewriting '%s' to content-addressed form yielded '%s'; "
+                    "please set this in the 'toPath' attribute passed to 'fetchClosure'",
+                    state.store->printStorePath(*fromPath),
+                    state.store->printStorePath(i->second)),
+                .errPos = pos
+            });
+    } else {
+        copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
+        toPath = fromPath;
+    }
 
     /* In pure mode, require a CA path. */
     if (evalSettings.pureEval) {
-        auto info = state.store->queryPathInfo(*fromPath);
+        auto info = state.store->queryPathInfo(*toPath);
         if (!info->isContentAddressed(*state.store))
             throw Error({
                 .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
-                    state.store->printStorePath(*fromPath)),
+                    state.store->printStorePath(*toPath)),
                 .errPos = pos
             });
     }
 
-    auto fromPathS = state.store->printStorePath(*fromPath);
-    v.mkString(fromPathS, {fromPathS});
+    auto toPathS = state.store->printStorePath(*toPath);
+    v.mkString(toPathS, {toPathS});
 }
 
 static RegisterPrimOp primop_fetchClosure({
diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc
new file mode 100644
index 000000000..0b95ff37c
--- /dev/null
+++ b/src/libstore/make-content-addressed.cc
@@ -0,0 +1,79 @@
+#include "make-content-addressed.hh"
+#include "references.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+    Store & srcStore,
+    Store & dstStore,
+    const StorePathSet & storePaths)
+{
+    // FIXME: use closure of storePaths.
+
+    auto paths = srcStore.topoSortPaths(storePaths);
+
+    std::reverse(paths.begin(), paths.end());
+
+    std::map<StorePath, StorePath> remappings;
+
+    for (auto & path : paths) {
+        auto pathS = srcStore.printStorePath(path);
+        auto oldInfo = srcStore.queryPathInfo(path);
+        std::string oldHashPart(path.hashPart());
+
+        StringSink sink;
+        srcStore.narFromPath(path, sink);
+
+        StringMap rewrites;
+
+        StorePathSet references;
+        bool hasSelfReference = false;
+        for (auto & ref : oldInfo->references) {
+            if (ref == path)
+                hasSelfReference = true;
+            else {
+                auto i = remappings.find(ref);
+                auto replacement = i != remappings.end() ? i->second : ref;
+                // FIXME: warn about unremapped paths?
+                if (replacement != ref)
+                    rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
+                references.insert(std::move(replacement));
+            }
+        }
+
+        sink.s = rewriteStrings(sink.s, rewrites);
+
+        HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+        hashModuloSink(sink.s);
+
+        auto narHash = hashModuloSink.finish().first;
+
+        ValidPathInfo info {
+            dstStore.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
+            narHash,
+        };
+        info.references = std::move(references);
+        if (hasSelfReference) info.references.insert(info.path);
+        info.narSize = sink.s.size();
+        info.ca = FixedOutputHash {
+            .method = FileIngestionMethod::Recursive,
+            .hash = info.narHash,
+        };
+
+        printInfo("rewrote '%s' to '%s'", pathS, srcStore.printStorePath(info.path));
+
+        auto source = sinkToSource([&](Sink & nextSink) {
+            RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
+            rsink2(sink.s);
+            rsink2.flush();
+        });
+
+        dstStore.addToStore(info, *source);
+
+        remappings.insert_or_assign(std::move(path), std::move(info.path));
+    }
+
+    return remappings;
+}
+
+}
diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh
new file mode 100644
index 000000000..c4a66ed41
--- /dev/null
+++ b/src/libstore/make-content-addressed.hh
@@ -0,0 +1,12 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+    Store & srcStore,
+    Store & dstStore,
+    const StorePathSet & storePaths);
+
+}
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index 2e75a3b61..a8579ea7c 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -1,6 +1,6 @@
 #include "command.hh"
 #include "store-api.hh"
-#include "references.hh"
+#include "make-content-addressed.hh"
 #include "common-args.hh"
 #include "json.hh"
 
@@ -27,74 +27,25 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
 
     void run(ref<Store> store, StorePaths && storePaths) override
     {
-        auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
-
-        std::reverse(paths.begin(), paths.end());
-
-        std::map<StorePath, StorePath> remappings;
-
-        auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
-        auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr;
-
-        for (auto & path : paths) {
-            auto pathS = store->printStorePath(path);
-            auto oldInfo = store->queryPathInfo(path);
-            std::string oldHashPart(path.hashPart());
-
-            StringSink sink;
-            store->narFromPath(path, sink);
-
-            StringMap rewrites;
-
-            StorePathSet references;
-            bool hasSelfReference = false;
-            for (auto & ref : oldInfo->references) {
-                if (ref == path)
-                    hasSelfReference = true;
-                else {
-                    auto i = remappings.find(ref);
-                    auto replacement = i != remappings.end() ? i->second : ref;
-                    // FIXME: warn about unremapped paths?
-                    if (replacement != ref)
-                        rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
-                    references.insert(std::move(replacement));
-                }
+        auto remappings = makeContentAddressed(*store, *store,
+            StorePathSet(storePaths.begin(), storePaths.end()));
+
+        if (json) {
+            JSONObject jsonRoot(std::cout);
+            JSONObject jsonRewrites(jsonRoot.object("rewrites"));
+            for (auto & path : storePaths) {
+                auto i = remappings.find(path);
+                assert(i != remappings.end());
+                jsonRewrites.attr(store->printStorePath(path), store->printStorePath(i->second));
+            }
+        } else {
+            for (auto & path : storePaths) {
+                auto i = remappings.find(path);
+                assert(i != remappings.end());
+                notice("rewrote '%s' to '%s'",
+                    store->printStorePath(path),
+                    store->printStorePath(i->second));
             }
-
-            sink.s = rewriteStrings(sink.s, rewrites);
-
-            HashModuloSink hashModuloSink(htSHA256, oldHashPart);
-            hashModuloSink(sink.s);
-
-            auto narHash = hashModuloSink.finish().first;
-
-            ValidPathInfo info {
-                store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
-                narHash,
-            };
-            info.references = std::move(references);
-            if (hasSelfReference) info.references.insert(info.path);
-            info.narSize = sink.s.size();
-            info.ca = FixedOutputHash {
-                .method = FileIngestionMethod::Recursive,
-                .hash = info.narHash,
-            };
-
-            if (!json)
-                notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
-
-            auto source = sinkToSource([&](Sink & nextSink) {
-                RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
-                rsink2(sink.s);
-                rsink2.flush();
-            });
-
-            store->addToStore(info, *source);
-
-            if (json)
-                jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path));
-
-            remappings.insert_or_assign(std::move(path), std::move(info.path));
         }
     }
 };