From 462421d34588c227eb736b07f7b40a38aa0972a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 16:04:18 +0200 Subject: Backport libfetchers from the flakes branch This provides a pluggable mechanism for defining new fetchers. It adds a builtin function 'fetchTree' that generalizes existing fetchers like 'fetchGit', 'fetchMercurial' and 'fetchTarball'. 'fetchTree' takes a set of attributes, e.g. fetchTree { type = "git"; url = "https://example.org/repo.git"; ref = "some-branch"; rev = "abcdef..."; } The existing fetchers are just wrappers around this. Note that the input attributes to fetchTree are the same as flake input specifications and flake lock file entries. All fetchers share a common cache stored in ~/.cache/nix/fetcher-cache-v1.sqlite. This replaces the ad hoc caching mechanisms in fetchGit and download.cc (e.g. ~/.cache/nix/{tarballs,git-revs*}). This also adds support for Git worktrees (c169ea59049f861aaba429f48b828d0820b74d1d). --- src/libexpr/primops/fetchTree.cc | 165 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 src/libexpr/primops/fetchTree.cc (limited to 'src/libexpr/primops/fetchTree.cc') diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc new file mode 100644 index 000000000..9586f71ed --- /dev/null +++ b/src/libexpr/primops/fetchTree.cc @@ -0,0 +1,165 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "download.hh" + +#include +#include + +namespace nix { + +void emitTreeAttrs( + EvalState & state, + const fetchers::Tree & tree, + std::shared_ptr input, + Value & v) +{ + state.mkAttrs(v, 8); + + auto storePath = state.store->printStorePath(tree.storePath); + + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + + assert(tree.info.narHash); + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + tree.info.narHash.to_string(SRI)); + + if (input->getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + } + + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + + if (tree.info.lastModified) + mkString(*state.allocAttr(v, state.symbols.create("lastModified")), + fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + + v.attrs->sort(); +} + +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + + std::shared_ptr input; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + fetchers::Attrs attrs; + + for (auto & attr : *args[0]->attrs) { + state.forceValue(*attr.value); + if (attr.value->type == tString) + attrs.emplace(attr.name, attr.value->string.s); + else if (attr.value->type == tBool) + attrs.emplace(attr.name, attr.value->boolean); + else + throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", + attr.name, showType(*attr.value)); + } + + if (!attrs.count("type")) + throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos); + + input = fetchers::inputFromAttrs(attrs); + } else + input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + + if (evalSettings.pureEval && !input->isImmutable()) + throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); + + // FIXME: use fetchOrSubstituteTree + auto [tree, input2] = input->fetchTree(state.store); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + emitTreeAttrs(state, tree, input2, v); +} + +static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); + +static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, + const string & who, bool unpack, std::string name) +{ + std::optional url; + std::optional expectedHash; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError("unsupported argument '%s' to '%s', at %s", + attr.name, who, attr.pos); + } + + if (!url) + throw EvalError("'url' argument required, at %s", pos); + + } else + url = state.forceStringNoCtx(*args[0], pos); + + url = resolveUri(*url); + + state.checkURI(*url); + + if (name == "") + name = baseNameOf(*url); + + if (evalSettings.pureEval && !expectedHash) + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + + auto storePath = + unpack + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + + auto path = state.store->toRealPath(storePath); + + if (expectedHash) { + auto hash = unpack + ? state.store->queryPathInfo(storePath)->narHash + : hashFile(htSHA256, path); + if (hash != *expectedHash) + throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + *url, expectedHash->to_string(), hash.to_string()); + } + + if (state.allowedPaths) + state.allowedPaths->insert(path); + + mkString(v, path, PathSet({path})); +} + +static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchurl", false, ""); +} + +static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchTarball", true, "source"); +} + +static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); +static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); + +} -- cgit v1.2.3 From 7867685dcdf1ba3c9290b777e0975ff0b775f667 Mon Sep 17 00:00:00 2001 From: Nikola Knezevic Date: Wed, 8 Apr 2020 14:12:22 +0200 Subject: after flake rebase --- src/libexpr/primops/fetchTree.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libexpr/primops/fetchTree.cc') diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9586f71ed..43c58485a 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -2,7 +2,7 @@ #include "eval-inline.hh" #include "store-api.hh" #include "fetchers.hh" -#include "download.hh" +#include "filetransfer.hh" #include #include -- cgit v1.2.3 From d1229859c2612587bfd04111bae6584c0c9df051 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 29 Apr 2020 22:48:29 +0200 Subject: Fix displaying error-position in `builtins.fetch{Tree,Tarball}` Without dereferencing this pointer, you'd get an error like this: ``` error: unsupported argument 'abc' to 'fetchTarball', at 0x13627e8 ``` --- src/libexpr/primops/fetchTree.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libexpr/primops/fetchTree.cc') diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 43c58485a..c5a0d9886 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -108,7 +108,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to '%s', at %s", - attr.name, who, attr.pos); + attr.name, who, *attr.pos); } if (!url) -- cgit v1.2.3