aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libexpr/primops/fetchClosure.cc152
-rw-r--r--src/libexpr/primops/fetchTree.cc4
-rw-r--r--src/libstore/make-content-addressed.cc80
-rw-r--r--src/libstore/make-content-addressed.hh12
-rw-r--r--src/libutil/experimental-features.cc1
-rw-r--r--src/libutil/experimental-features.hh3
-rw-r--r--src/nix/main.cc2
-rw-r--r--src/nix/make-content-addressable.cc102
-rw-r--r--src/nix/make-content-addressed.cc55
-rw-r--r--src/nix/make-content-addressed.md (renamed from src/nix/make-content-addressable.md)4
10 files changed, 307 insertions, 108 deletions
diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
new file mode 100644
index 000000000..47f40ef33
--- /dev/null
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -0,0 +1,152 @@
+#include "primops.hh"
+#include "store-api.hh"
+#include "make-content-addressed.hh"
+#include "url.hh"
+
+namespace nix {
+
+static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+ settings.requireExperimentalFeature(Xp::FetchClosure);
+
+ state.forceAttrs(*args[0], pos);
+
+ std::optional<std::string> fromStoreUrl;
+ std::optional<StorePath> fromPath;
+ bool toCA = false;
+ std::optional<StorePath> toPath;
+
+ for (auto & attr : *args[0]->attrs) {
+ if (attr.name == "fromPath") {
+ PathSet context;
+ fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+ }
+
+ else if (attr.name == "toPath") {
+ state.forceValue(*attr.value, *attr.pos);
+ toCA = true;
+ if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
+ PathSet context;
+ toPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+ }
+ }
+
+ else if (attr.name == "fromStore")
+ fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos);
+
+ else
+ throw Error({
+ .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name),
+ .errPos = pos
+ });
+ }
+
+ if (!fromPath)
+ throw Error({
+ .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"),
+ .errPos = pos
+ });
+
+ if (!fromStoreUrl)
+ throw Error({
+ .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
+ .errPos = pos
+ });
+
+ auto parsedURL = parseURL(*fromStoreUrl);
+
+ if (parsedURL.scheme != "http" &&
+ parsedURL.scheme != "https" &&
+ !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
+ throw Error({
+ .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"),
+ .errPos = pos
+ });
+
+ auto fromStore = openStore(parsedURL.to_string());
+
+ if (toCA) {
+ if (!toPath || !state.store->isValidPath(*toPath)) {
+ auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
+ auto i = remappings.find(*fromPath);
+ assert(i != remappings.end());
+ if (toPath && *toPath != i->second)
+ throw Error({
+ .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
+ state.store->printStorePath(*fromPath),
+ state.store->printStorePath(i->second),
+ state.store->printStorePath(*toPath)),
+ .errPos = pos
+ });
+ if (!toPath)
+ throw Error({
+ .msg = hintfmt(
+ "rewriting '%s' to content-addressed form yielded '%s'; "
+ "please set this in the 'toPath' attribute passed to 'fetchClosure'",
+ state.store->printStorePath(*fromPath),
+ state.store->printStorePath(i->second)),
+ .errPos = pos
+ });
+ }
+ } else {
+ copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
+ toPath = fromPath;
+ }
+
+ /* In pure mode, require a CA path. */
+ if (evalSettings.pureEval) {
+ auto info = state.store->queryPathInfo(*toPath);
+ if (!info->isContentAddressed(*state.store))
+ throw Error({
+ .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
+ state.store->printStorePath(*toPath)),
+ .errPos = pos
+ });
+ }
+
+ auto toPathS = state.store->printStorePath(*toPath);
+ v.mkString(toPathS, {toPathS});
+}
+
+static RegisterPrimOp primop_fetchClosure({
+ .name = "__fetchClosure",
+ .args = {"args"},
+ .doc = R"(
+ Fetch a Nix store closure from a binary cache, rewriting it into
+ content-addressed form. For example,
+
+ ```nix
+ builtins.fetchClosure {
+ fromStore = "https://cache.nixos.org";
+ fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
+ toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
+ }
+ ```
+
+ fetches `/nix/store/r2jd...` from the specified binary cache,
+ and rewrites it into the content-addressed store path
+ `/nix/store/ldbh...`.
+
+ If `fromPath` is already content-addressed, or if you are
+ allowing impure evaluation (`--impure`), then `toPath` may be
+ omitted.
+
+ To find out the correct value for `toPath` given a `fromPath`,
+ you can use `nix store make-content-addressed`:
+
+ ```console
+ # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
+ rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
+ ```
+
+ This function is similar to `builtins.storePath` in that it
+ allows you to use a previously built store path in a Nix
+ expression. However, it is more reproducible because it requires
+ specifying a binary cache from which the path can be fetched.
+ Also, requiring a content-addressed final store path avoids the
+ need for users to configure binary cache public keys.
+ )",
+ .fun = prim_fetchClosure,
+});
+
+}
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 9c2da2178..42c98e312 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -145,7 +145,7 @@ static void fetchTree(
if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
throw Error({
- .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
+ .msg = hintfmt("attribute 'name' isn't supported in call to 'fetchTree'"),
.errPos = pos
});
@@ -329,7 +329,7 @@ static RegisterPrimOp primop_fetchTarball({
.fun = prim_fetchTarball,
});
-static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
+static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
}
diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc
new file mode 100644
index 000000000..64d172918
--- /dev/null
+++ b/src/libstore/make-content-addressed.cc
@@ -0,0 +1,80 @@
+#include "make-content-addressed.hh"
+#include "references.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+ Store & srcStore,
+ Store & dstStore,
+ const StorePathSet & storePaths)
+{
+ StorePathSet closure;
+ srcStore.computeFSClosure(storePaths, closure);
+
+ auto paths = srcStore.topoSortPaths(closure);
+
+ std::reverse(paths.begin(), paths.end());
+
+ std::map<StorePath, StorePath> remappings;
+
+ for (auto & path : paths) {
+ auto pathS = srcStore.printStorePath(path);
+ auto oldInfo = srcStore.queryPathInfo(path);
+ std::string oldHashPart(path.hashPart());
+
+ StringSink sink;
+ srcStore.narFromPath(path, sink);
+
+ StringMap rewrites;
+
+ StorePathSet references;
+ bool hasSelfReference = false;
+ for (auto & ref : oldInfo->references) {
+ if (ref == path)
+ hasSelfReference = true;
+ else {
+ auto i = remappings.find(ref);
+ auto replacement = i != remappings.end() ? i->second : ref;
+ // FIXME: warn about unremapped paths?
+ if (replacement != ref)
+ rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
+ references.insert(std::move(replacement));
+ }
+ }
+
+ sink.s = rewriteStrings(sink.s, rewrites);
+
+ HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+ hashModuloSink(sink.s);
+
+ auto narModuloHash = hashModuloSink.finish().first;
+
+ auto dstPath = dstStore.makeFixedOutputPath(
+ FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference);
+
+ printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath));
+
+ StringSink sink2;
+ RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2);
+ rsink2(sink.s);
+ rsink2.flush();
+
+ ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) };
+ info.references = std::move(references);
+ if (hasSelfReference) info.references.insert(info.path);
+ info.narSize = sink.s.size();
+ info.ca = FixedOutputHash {
+ .method = FileIngestionMethod::Recursive,
+ .hash = narModuloHash,
+ };
+
+ StringSource source(sink2.s);
+ dstStore.addToStore(info, source);
+
+ remappings.insert_or_assign(std::move(path), std::move(info.path));
+ }
+
+ return remappings;
+}
+
+}
diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh
new file mode 100644
index 000000000..c4a66ed41
--- /dev/null
+++ b/src/libstore/make-content-addressed.hh
@@ -0,0 +1,12 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+ Store & srcStore,
+ Store & dstStore,
+ const StorePathSet & storePaths);
+
+}
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index b49f47e1d..01f318fa3 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -11,6 +11,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::NixCommand, "nix-command" },
{ Xp::RecursiveNix, "recursive-nix" },
{ Xp::NoUrlLiterals, "no-url-literals" },
+ { Xp::FetchClosure, "fetch-closure" },
};
const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index 291a58e32..b5140dcfe 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -19,7 +19,8 @@ enum struct ExperimentalFeature
Flakes,
NixCommand,
RecursiveNix,
- NoUrlLiterals
+ NoUrlLiterals,
+ FetchClosure,
};
/**
diff --git a/src/nix/main.cc b/src/nix/main.cc
index b923f2535..9bc6c15fa 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -117,7 +117,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
{"hash-path", {"hash", "path"}},
{"ls-nar", {"nar", "ls"}},
{"ls-store", {"store", "ls"}},
- {"make-content-addressable", {"store", "make-content-addressable"}},
+ {"make-content-addressable", {"store", "make-content-addressed"}},
{"optimise-store", {"store", "optimise"}},
{"ping-store", {"store", "ping"}},
{"sign-paths", {"store", "sign"}},
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
deleted file mode 100644
index 2e75a3b61..000000000
--- a/src/nix/make-content-addressable.cc
+++ /dev/null
@@ -1,102 +0,0 @@
-#include "command.hh"
-#include "store-api.hh"
-#include "references.hh"
-#include "common-args.hh"
-#include "json.hh"
-
-using namespace nix;
-
-struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
-{
- CmdMakeContentAddressable()
- {
- realiseMode = Realise::Outputs;
- }
-
- std::string description() override
- {
- return "rewrite a path or closure to content-addressed form";
- }
-
- std::string doc() override
- {
- return
- #include "make-content-addressable.md"
- ;
- }
-
- void run(ref<Store> store, StorePaths && storePaths) override
- {
- auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
-
- std::reverse(paths.begin(), paths.end());
-
- std::map<StorePath, StorePath> remappings;
-
- auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
- auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr;
-
- for (auto & path : paths) {
- auto pathS = store->printStorePath(path);
- auto oldInfo = store->queryPathInfo(path);
- std::string oldHashPart(path.hashPart());
-
- StringSink sink;
- store->narFromPath(path, sink);
-
- StringMap rewrites;
-
- StorePathSet references;
- bool hasSelfReference = false;
- for (auto & ref : oldInfo->references) {
- if (ref == path)
- hasSelfReference = true;
- else {
- auto i = remappings.find(ref);
- auto replacement = i != remappings.end() ? i->second : ref;
- // FIXME: warn about unremapped paths?
- if (replacement != ref)
- rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
- references.insert(std::move(replacement));
- }
- }
-
- sink.s = rewriteStrings(sink.s, rewrites);
-
- HashModuloSink hashModuloSink(htSHA256, oldHashPart);
- hashModuloSink(sink.s);
-
- auto narHash = hashModuloSink.finish().first;
-
- ValidPathInfo info {
- store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference),
- narHash,
- };
- info.references = std::move(references);
- if (hasSelfReference) info.references.insert(info.path);
- info.narSize = sink.s.size();
- info.ca = FixedOutputHash {
- .method = FileIngestionMethod::Recursive,
- .hash = info.narHash,
- };
-
- if (!json)
- notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
-
- auto source = sinkToSource([&](Sink & nextSink) {
- RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
- rsink2(sink.s);
- rsink2.flush();
- });
-
- store->addToStore(info, *source);
-
- if (json)
- jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path));
-
- remappings.insert_or_assign(std::move(path), std::move(info.path));
- }
- }
-};
-
-static auto rCmdMakeContentAddressable = registerCommand2<CmdMakeContentAddressable>({"store", "make-content-addressable"});
diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc
new file mode 100644
index 000000000..34860c38f
--- /dev/null
+++ b/src/nix/make-content-addressed.cc
@@ -0,0 +1,55 @@
+#include "command.hh"
+#include "store-api.hh"
+#include "make-content-addressed.hh"
+#include "common-args.hh"
+#include "json.hh"
+
+using namespace nix;
+
+struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON
+{
+ CmdMakeContentAddressed()
+ {
+ realiseMode = Realise::Outputs;
+ }
+
+ std::string description() override
+ {
+ return "rewrite a path or closure to content-addressed form";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "make-content-addressed.md"
+ ;
+ }
+
+ void run(ref<Store> srcStore, StorePaths && storePaths) override
+ {
+ auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
+
+ auto remappings = makeContentAddressed(*srcStore, *dstStore,
+ StorePathSet(storePaths.begin(), storePaths.end()));
+
+ if (json) {
+ JSONObject jsonRoot(std::cout);
+ JSONObject jsonRewrites(jsonRoot.object("rewrites"));
+ for (auto & path : storePaths) {
+ auto i = remappings.find(path);
+ assert(i != remappings.end());
+ jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second));
+ }
+ } else {
+ for (auto & path : storePaths) {
+ auto i = remappings.find(path);
+ assert(i != remappings.end());
+ notice("rewrote '%s' to '%s'",
+ srcStore->printStorePath(path),
+ srcStore->printStorePath(i->second));
+ }
+ }
+ }
+};
+
+static auto rCmdMakeContentAddressed = registerCommand2<CmdMakeContentAddressed>({"store", "make-content-addressed"});
diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressed.md
index 3dd847edc..215683e6d 100644
--- a/src/nix/make-content-addressable.md
+++ b/src/nix/make-content-addressed.md
@@ -5,7 +5,7 @@ R""(
* Create a content-addressed representation of the closure of GNU Hello:
```console
- # nix store make-content-addressable -r nixpkgs#hello
+ # nix store make-content-addressed nixpkgs#hello
…
rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10'
```
@@ -29,7 +29,7 @@ R""(
system closure:
```console
- # nix store make-content-addressable -r /run/current-system
+ # nix store make-content-addressed /run/current-system
```
# Description
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 20:48:09 +0000