From f58604ac32d0ab0718a0e68b3c1b050e4ab121cb Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Fri, 25 Oct 2019 17:45:14 +0200 Subject: Add fetchSubmodules to builtins.fetchGit There are some downsides to this features: - Submodules are not cached (unlike the root repo), - Full checkouts are created in a temporary directory. --- src/libexpr/primops/fetchGit.cc | 45 ++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 10 deletions(-) (limited to 'src/libexpr') diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 4aee1073e..b08e48404 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -22,13 +22,14 @@ struct GitInfo std::string rev; std::string shortRev; uint64_t revCount = 0; + bool submodules = false; }; std::regex revRegex("^[0-9a-fA-F]{40}$"); GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, - const std::string & name) + const std::string & name, bool fetchSubmodules) { if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); @@ -145,7 +146,8 @@ GitInfo exportGit(ref store, const std::string & uri, printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); + std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev + + (fetchSubmodules ? "submodules" : "")).to_string(Base32, false); Path storeLink = cacheDir + "/" + storeLinkName + ".link"; PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken @@ -165,16 +167,35 @@ GitInfo exportGit(ref store, const std::string & uri, if (e.errNo != ENOENT) throw; } - auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); - gitOptions.standardOut = &sink; - runProgram2(gitOptions); - }); - Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - unpackTarfile(*source, tmpDir); + // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root + // repo is cached. + if (fetchSubmodules) { + Path tmpGitDir = createTempDir(); + AutoDelete delTmpGitDir(tmpGitDir, true); + + runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", + "--", cacheDir, fmt("%s:%s", *ref, *ref) }); + + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); + runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init" }); + + deletePath(tmpDir + "/.git"); + + gitInfo.submodules = true; + } else { + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + unpackTarfile(*source, tmpDir); + } gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); @@ -198,6 +219,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va std::optional ref; std::string rev; std::string name = "source"; + bool fetchSubmodules = false; PathSet context; state.forceValue(*args[0]); @@ -216,6 +238,8 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va rev = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "fetchSubmodules") + fetchSubmodules = state.forceBool(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); } @@ -230,13 +254,14 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); - auto gitInfo = exportGit(state.store, url, ref, rev, name); + auto gitInfo = exportGit(state.store, url, ref, rev, name, fetchSubmodules); state.mkAttrs(v, 8); mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), gitInfo.submodules); v.attrs->sort(); if (state.allowedPaths) -- cgit v1.2.3 From c8d33de77781fe7850d04a7374c6c22a2e995b70 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:42:19 +0100 Subject: Add git submodule fixes from @bjornfor This fixes fetching repositories with no submodules and also cleans up .git files in checkouts. --- src/libexpr/local.mk | 1 + src/libexpr/primops/fetchGit.cc | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'src/libexpr') diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 8a9b3c2ea..9b5fcc561 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -9,6 +9,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust libexpr_LDFLAGS = + ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index b08e48404..c58d28bbc 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -6,6 +6,7 @@ #include "hash.hh" #include "tarfile.hh" +#include #include #include @@ -182,9 +183,13 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init" }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); - deletePath(tmpDir + "/.git"); + for (const auto& p : std::filesystem::recursive_directory_iterator(tmpDir)) { + if (p.path().filename() == ".git") { + std::filesystem::remove_all(p.path()); + } + } gitInfo.submodules = true; } else { -- cgit v1.2.3 From 435366ed3c292756bb74fd12f47ff96e93cd48ac Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 2 Mar 2020 14:50:13 +0100 Subject: Rename fetchGit fetchSubmodules to just submodules --- src/libexpr/primops/fetchGit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libexpr') diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index c58d28bbc..e8905b548 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -243,7 +243,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va rev = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "fetchSubmodules") + else if (n == "submodules") fetchSubmodules = state.forceBool(*attr.value, *attr.pos); else throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); -- cgit v1.2.3 From cc4fe977e5eedf00d8e3d267ccbc3676b3fac1a0 Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Mon, 9 Mar 2020 14:07:26 +0100 Subject: Link to stdc++fs Some platforms seem to still require linking with stdc++fs to enable STL std::filesystem support. --- src/libexpr/local.mk | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/libexpr') diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 9b5fcc561..b6ee82424 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,7 +8,11 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust +ifeq ($(CXX), g++) +libexpr_LDFLAGS = -lstdc++fs +else libexpr_LDFLAGS = +endif ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl -- cgit v1.2.3 From 002a3a95dcb7391a8372ba7a74f4e7e9bd48b59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Mon, 16 Mar 2020 14:47:21 +0100 Subject: fetchGit: fix "fatal: couldn't find remote ref refs/heads/master" issue with submodules --- src/libexpr/primops/fetchGit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libexpr') diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index e8905b548..66259d778 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -179,7 +179,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--", cacheDir, fmt("%s:%s", *ref, *ref) }); + "--", cacheDir, fmt("%s", *ref) }); runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); -- cgit v1.2.3 From 6864ad7cf510d96f0cfc0cdbeffc1188a4eab44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 14:27:14 +0100 Subject: fetchGit: fix submodule output attribute Before this change it would be false for all evaluations but the first. Now it follows the input argument (as it should). --- src/libexpr/primops/fetchGit.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'src/libexpr') diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 66259d778..653d99b53 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -32,6 +32,9 @@ GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, const std::string & name, bool fetchSubmodules) { + GitInfo gitInfo; + gitInfo.submodules = fetchSubmodules; + if (evalSettings.pureEval && rev == "") throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); @@ -49,7 +52,6 @@ GitInfo exportGit(ref store, const std::string & uri, if (!clean) { /* This is an unclean working tree. So copy all tracked files. */ - GitInfo gitInfo; gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); @@ -141,7 +143,6 @@ GitInfo exportGit(ref store, const std::string & uri, } // FIXME: check whether rev is an ancestor of ref. - GitInfo gitInfo; gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); @@ -190,8 +191,6 @@ GitInfo exportGit(ref store, const std::string & uri, std::filesystem::remove_all(p.path()); } } - - gitInfo.submodules = true; } else { auto source = sinkToSource([&](Sink & sink) { RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); -- cgit v1.2.3 From 369fffd6f1f7d939e528b3591dca49989c51ae35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Tue, 17 Mar 2020 14:27:58 +0100 Subject: fetchGit: add submodules attribute to the .link file The .link file is used as a lock, so I think we should put the "submodule" attribute in there since turning on submodules creates a new .link file path. --- src/libexpr/primops/fetchGit.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'src/libexpr') diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 653d99b53..969b1a4ed 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -211,6 +211,7 @@ GitInfo exportGit(ref store, const std::string & uri, json["name"] = name; json["rev"] = gitInfo.rev; json["revCount"] = gitInfo.revCount; + json["submodules"] = gitInfo.submodules; writeFile(storeLink, json.dump()); -- cgit v1.2.3 From b306b7039ef3f2b76dbcc8020dcf67ed27442e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 18 Mar 2020 08:44:37 +0100 Subject: fetchGit: checkout rev instead of latest ref Major bugfix for the submodules = true code path. TODO: Add tests. --- src/libexpr/primops/fetchGit.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libexpr') diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 969b1a4ed..0c6bc6dc0 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -182,7 +182,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--", cacheDir, fmt("%s", *ref) }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", "FETCH_HEAD" }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); -- cgit v1.2.3 From cc522d0d23b71faeac95998d264abe14ddceeed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Wed, 18 Mar 2020 08:45:31 +0100 Subject: fetchGit: fix submodules = true for dirty trees --- src/libexpr/primops/fetchGit.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src/libexpr') diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 0c6bc6dc0..66e1d7b98 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -55,8 +55,12 @@ GitInfo exportGit(ref store, const std::string & uri, gitInfo.rev = "0000000000000000000000000000000000000000"; gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + auto gitOpts = Strings({ "-C", uri, "ls-files", "-z" }); + if (fetchSubmodules) { + gitOpts.emplace_back("--recurse-submodules"); + } auto files = tokenizeString>( - runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); + runProgram("git", true, gitOpts), "\0"s); PathFilter filter = [&](const Path & p) -> bool { assert(hasPrefix(p, uri)); -- cgit v1.2.3 From f686efeed4d6305101c56136855e2d6b87e649e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Forsman?= Date: Fri, 20 Mar 2020 14:36:10 +0100 Subject: fetchGit: fix submodule corner case by fetching all refs from cacheDir Due to fetchGit not checking if rev is an ancestor of ref (there is even a FIXME comment about it in the code), the cache repo might not have the ref even though it has the rev. This doesn't matter when submodule = false, but the submodule = true code blows up because it tries to fetch the (missing) ref from the cache repo. Fix this in the simplest way possible: fetch all refs from the local cache repo when submodules = true. TODO: Add tests. --- src/libexpr/primops/fetchGit.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src/libexpr') diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 66e1d7b98..f138b755c 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -183,8 +183,11 @@ GitInfo exportGit(ref store, const std::string & uri, AutoDelete delTmpGitDir(tmpGitDir, true); runProgram("git", true, { "init", tmpDir, "--separate-git-dir", tmpGitDir }); + // TODO: the cacheDir repo might lack the ref (it only checks if rev + // exists, see FIXME above) so use a big hammer and fetch everything to + // ensure we get the rev. runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--", cacheDir, fmt("%s", *ref) }); + "--update-head-ok", "--", cacheDir, "refs/*:refs/*" }); runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", gitInfo.rev }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); -- cgit v1.2.3 From 40c023ecfe49fea6e66db34c5f841fcf7001cbeb Mon Sep 17 00:00:00 2001 From: Julian Stecklina Date: Sun, 29 Mar 2020 23:47:48 +0200 Subject: fetchGit: don't use std::filesystem to filter git repos Using std::filesystem means also having to link with -lstdc++fs on some platforms and it's hard to discover for what platforms this is needed. As all the functionality is already implemented as utilities, use those instead. --- src/libexpr/local.mk | 4 ---- src/libexpr/primops/fetchGit.cc | 17 ++++++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) (limited to 'src/libexpr') diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index b6ee82424..9b5fcc561 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,11 +8,7 @@ libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexe libexpr_LIBS = libutil libstore libnixrust -ifeq ($(CXX), g++) -libexpr_LDFLAGS = -lstdc++fs -else libexpr_LDFLAGS = -endif ifneq ($(OS), FreeBSD) libexpr_LDFLAGS += -ldl diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index f138b755c..d60e6b803 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -6,7 +6,6 @@ #include "hash.hh" #include "tarfile.hh" -#include #include #include @@ -28,6 +27,13 @@ struct GitInfo std::regex revRegex("^[0-9a-fA-F]{40}$"); +static bool isNotDotGitDirectory(const Path & path) +{ + static const std::regex gitDirRegex("^(?:.*/)?\\.git$"); + + return not std::regex_match(path, gitDirRegex); +} + GitInfo exportGit(ref store, const std::string & uri, std::optional ref, std::string rev, const std::string & name, bool fetchSubmodules) @@ -175,6 +181,7 @@ GitInfo exportGit(ref store, const std::string & uri, Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); + PathFilter filter = defaultPathFilter; // Submodule support can be improved by adding caching to the submodules themselves. At the moment, only the root // repo is cached. @@ -193,11 +200,7 @@ GitInfo exportGit(ref store, const std::string & uri, runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", uri }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); - for (const auto& p : std::filesystem::recursive_directory_iterator(tmpDir)) { - if (p.path().filename() == ".git") { - std::filesystem::remove_all(p.path()); - } - } + filter = isNotDotGitDirectory; } else { auto source = sinkToSource([&](Sink & sink) { RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); @@ -208,7 +211,7 @@ GitInfo exportGit(ref store, const std::string & uri, unpackTarfile(*source, tmpDir); } - gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); + gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir, true, htSHA256, filter)); gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); -- cgit v1.2.3 From 4fc4eb6c93511c61a278ba0e5555ffb24435ab59 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Thu, 2 Apr 2020 17:04:00 +0200 Subject: libexpr: remove unused attrError The attrError variable is no longer used but still allocated on every call to the findAlongAttrPath function. --- src/libexpr/attr-path.cc | 3 --- 1 file changed, 3 deletions(-) (limited to 'src/libexpr') diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 4545bfd72..76d101b98 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -37,9 +37,6 @@ std::pair findAlongAttrPath(EvalState & state, const string & attr { Strings tokens = parseAttrPath(attrPath); - Error attrError = - Error(format("attribute selection path '%1%' does not match expression") % attrPath); - Value * v = &vIn; Pos pos = noPos; -- cgit v1.2.3 From 462421d34588c227eb736b07f7b40a38aa0972a6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 16:04:18 +0200 Subject: Backport libfetchers from the flakes branch This provides a pluggable mechanism for defining new fetchers. It adds a builtin function 'fetchTree' that generalizes existing fetchers like 'fetchGit', 'fetchMercurial' and 'fetchTarball'. 'fetchTree' takes a set of attributes, e.g. fetchTree { type = "git"; url = "https://example.org/repo.git"; ref = "some-branch"; rev = "abcdef..."; } The existing fetchers are just wrappers around this. Note that the input attributes to fetchTree are the same as flake input specifications and flake lock file entries. All fetchers share a common cache stored in ~/.cache/nix/fetcher-cache-v1.sqlite. This replaces the ad hoc caching mechanisms in fetchGit and download.cc (e.g. ~/.cache/nix/{tarballs,git-revs*}). This also adds support for Git worktrees (c169ea59049f861aaba429f48b828d0820b74d1d). --- src/libexpr/common-eval-args.cc | 8 +- src/libexpr/local.mk | 4 +- src/libexpr/parser.y | 6 +- src/libexpr/primops.cc | 67 ---------- src/libexpr/primops.hh | 1 + src/libexpr/primops/fetchGit.cc | 223 +++++----------------------------- src/libexpr/primops/fetchMercurial.cc | 207 ++++++------------------------- src/libexpr/primops/fetchTree.cc | 165 +++++++++++++++++++++++++ 8 files changed, 242 insertions(+), 439 deletions(-) create mode 100644 src/libexpr/primops/fetchTree.cc (limited to 'src/libexpr') diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 13950ab8d..82bfeac36 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -3,6 +3,8 @@ #include "download.hh" #include "util.hh" #include "eval.hh" +#include "fetchers.hh" +#include "store-api.hh" namespace nix { @@ -46,9 +48,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) Path lookupFileArg(EvalState & state, string s) { if (isUri(s)) { - CachedDownloadRequest request(s); - request.unpack = true; - return getDownloader()->downloadCached(state.store, request).path; + return state.store->toRealPath( + fetchers::downloadTarball( + state.store, resolveUri(s), "source", false).storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index a4ccab376..917e8a1c7 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -6,9 +6,9 @@ libexpr_DIR := $(d) libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc -libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain -I src/libexpr +libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr -libexpr_LIBS = libutil libstore libnixrust +libexpr_LIBS = libutil libstore libfetchers libnixrust libexpr_LDFLAGS = ifneq ($(OS), FreeBSD) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9c769e803..a30fb44b5 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -545,6 +545,7 @@ formal #include "eval.hh" #include "download.hh" +#include "fetchers.hh" #include "store-api.hh" @@ -687,9 +688,8 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { - CachedDownloadRequest request(elem.second); - request.unpack = true; - res = { true, getDownloader()->downloadCached(store, request).path }; + res = { true, store->toRealPath(fetchers::downloadTarball( + store, resolveUri(elem.second), "source", false).storePath) }; } catch (DownloadError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8de234951..629f3da15 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,6 +1,5 @@ #include "archive.hh" #include "derivations.hh" -#include "download.hh" #include "eval-inline.hh" #include "eval.hh" #include "globals.hh" @@ -2045,68 +2044,6 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args } -/************************************************************* - * Networking - *************************************************************/ - - -void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, - const string & who, bool unpack, const std::string & defaultName) -{ - CachedDownloadRequest request(""); - request.unpack = unpack; - request.name = defaultName; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "sha256") - request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); - else if (n == "name") - request.name = state.forceStringNoCtx(*attr.value, *attr.pos); - else - throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); - } - - if (request.uri.empty()) - throw EvalError(format("'url' argument required, at %1%") % pos); - - } else - request.uri = state.forceStringNoCtx(*args[0], pos); - - state.checkURI(request.uri); - - if (evalSettings.pureEval && !request.expectedHash) - throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); - - auto res = getDownloader()->downloadCached(state.store, request); - - if (state.allowedPaths) - state.allowedPaths->insert(res.path); - - mkString(v, res.storePath, PathSet({res.storePath})); -} - - -static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchurl", false, ""); -} - - -static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - fetch(state, pos, args, v, "fetchTarball", true, "source"); -} - - /************************************************************* * Primop registration *************************************************************/ @@ -2289,10 +2226,6 @@ void EvalState::createBaseEnv() addPrimOp("derivationStrict", 1, prim_derivationStrict); addPrimOp("placeholder", 1, prim_placeholder); - // Networking - addPrimOp("__fetchurl", 1, prim_fetchurl); - addPrimOp("fetchTarball", 1, prim_fetchTarball); - /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index c790b30f6..05d0792ef 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -20,6 +20,7 @@ struct RegisterPrimOp them. */ /* Load a ValueInitializer from a DSO and return whatever it initializes */ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); + /* Execute a program and parse its output */ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 4aee1073e..812de9d91 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -1,202 +1,17 @@ #include "primops.hh" #include "eval-inline.hh" -#include "download.hh" #include "store-api.hh" -#include "pathlocks.hh" #include "hash.hh" -#include "tarfile.hh" - -#include - -#include - -#include - -using namespace std::string_literals; +#include "fetchers.hh" +#include "url.hh" namespace nix { -struct GitInfo -{ - Path storePath; - std::string rev; - std::string shortRev; - uint64_t revCount = 0; -}; - -std::regex revRegex("^[0-9a-fA-F]{40}$"); - -GitInfo exportGit(ref store, const std::string & uri, - std::optional ref, std::string rev, - const std::string & name) -{ - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - - if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { - - bool clean = true; - - try { - runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - clean = false; - } - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked files. */ - GitInfo gitInfo; - gitInfo.rev = "0000000000000000000000000000000000000000"; - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - - auto files = tokenizeString>( - runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, uri)); - std::string file(p, uri.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - gitInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); - - return gitInfo; - } - - // clean working tree, but no ref or rev specified. Use 'HEAD'. - rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })); - ref = "HEAD"s; - } - - if (!ref) ref = "HEAD"s; - - if (rev != "" && !std::regex_match(rev, revRegex)) - throw Error("invalid Git revision '%s'", rev); - - deletePath(getCacheDir() + "/nix/git"); - - Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); - - if (!pathExists(cacheDir)) { - createDirs(dirOf(cacheDir)); - runProgram("git", true, { "init", "--bare", cacheDir }); - } - - Path localRefFile; - if (ref->compare(0, 5, "refs/") == 0) - localRefFile = cacheDir + "/" + *ref; - else - localRefFile = cacheDir + "/refs/heads/" + *ref; - - bool doFetch; - time_t now = time(0); - /* If a rev was specified, we need to fetch if it's not in the - repo. */ - if (rev != "") { - try { - runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } - } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; - } - if (doFetch) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); - - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); - - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; - - utimes(localRefFile.c_str(), times); - } - - // FIXME: check whether rev is an ancestor of ref. - GitInfo gitInfo; - gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); - gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); - - printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); - - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); - Path storeLink = cacheDir + "/" + storeLinkName + ".link"; - PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && json["rev"] == gitInfo.rev); - - gitInfo.storePath = json["storePath"]; - - if (store->isValidPath(store->parseStorePath(gitInfo.storePath))) { - gitInfo.revCount = json["revCount"]; - return gitInfo; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - - auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", cacheDir, "archive", gitInfo.rev }); - gitOptions.standardOut = &sink; - runProgram2(gitOptions); - }); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - unpackTarfile(*source, tmpDir); - - gitInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); - - gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); - - nlohmann::json json; - json["storePath"] = gitInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["rev"] = gitInfo.rev; - json["revCount"] = gitInfo.revCount; - - writeFile(storeLink, json.dump()); - - return gitInfo; -} - static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; std::optional ref; - std::string rev; + std::optional rev; std::string name = "source"; PathSet context; @@ -213,7 +28,7 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va else if (n == "ref") ref = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); + rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else @@ -230,17 +45,35 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va // whitelist. Ah well. state.checkURI(url); - auto gitInfo = exportGit(state.store, url, ref, rev, name); + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + + auto parsedUrl = parseURL( + url.find("://") != std::string::npos + ? "git+" + url + : "git+file://" + url); + if (ref) parsedUrl.query.insert_or_assign("ref", *ref); + if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); + // FIXME: use name + auto input = fetchers::inputFromURL(parsedUrl); + + auto [tree, input2] = input->fetchTree(state.store); state.mkAttrs(v, 8); - mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + auto storePath = state.store->printStorePath(tree.storePath); + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + // Backward compatibility: set 'rev' to + // 0000000000000000000000000000000000000000 for a dirty tree. + auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); + // Backward compatibility: set 'revCount' to 0 for a dirty tree. + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), + tree.info.revCount.value_or(0)); v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); + state.allowedPaths->insert(tree.actualPath); } static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index db274fa4f..f18351646 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,174 +1,18 @@ #include "primops.hh" #include "eval-inline.hh" -#include "download.hh" #include "store-api.hh" -#include "pathlocks.hh" - -#include +#include "fetchers.hh" +#include "url.hh" #include -#include - -using namespace std::string_literals; - namespace nix { -struct HgInfo -{ - Path storePath; - std::string branch; - std::string rev; - uint64_t revCount = 0; -}; - -std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); - -HgInfo exportMercurial(ref store, const std::string & uri, - std::string rev, const std::string & name) -{ - if (evalSettings.pureEval && rev == "") - throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); - - if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { - - bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked - files. */ - - printTalkative("copying unclean Mercurial working tree '%s'", uri); - - HgInfo hgInfo; - hgInfo.rev = "0000000000000000000000000000000000000000"; - hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri })); - - auto files = tokenizeString>( - runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, uri)); - std::string file(p, uri.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - hgInfo.storePath = store->printStorePath(store->addToStore("source", uri, true, htSHA256, filter)); - - return hgInfo; - } - } - - if (rev == "") rev = "default"; - - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false)); - - Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false)); - - /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, - do so now. */ - time_t now = time(0); - struct stat st; - if (stat(stampFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) - { - /* Except that if this is a commit hash that we already have, - we don't have to pull again. */ - if (!(std::regex_match(rev, commitHashRegex) - && pathExists(cacheDir) - && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" }) - .killStderr(true)).second == "1")) - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); - - if (pathExists(cacheDir)) { - try { - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } - catch (ExecError & e) { - string transJournal = cacheDir + "/.hg/store/journal"; - /* hg throws "abandoned transaction" error only if this file exists */ - if (pathExists(transJournal)) { - runProgram("hg", true, { "recover", "-R", cacheDir }); - runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); - } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); - } - } - } else { - createDirs(dirOf(cacheDir)); - runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir }); - } - } - - writeFile(stampFile, ""); - } - - auto tokens = tokenizeString>( - runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" })); - assert(tokens.size() == 3); - - HgInfo hgInfo; - hgInfo.rev = tokens[0]; - hgInfo.revCount = std::stoull(tokens[1]); - hgInfo.branch = tokens[2]; - - std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false); - Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); - - try { - auto json = nlohmann::json::parse(readFile(storeLink)); - - assert(json["name"] == name && json["rev"] == hgInfo.rev); - - hgInfo.storePath = json["storePath"]; - - if (store->isValidPath(store->parseStorePath(hgInfo.storePath))) { - printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath); - return hgInfo; - } - - } catch (SysError & e) { - if (e.errNo != ENOENT) throw; - } - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir }); - - deletePath(tmpDir + "/.hg_archival.txt"); - - hgInfo.storePath = store->printStorePath(store->addToStore(name, tmpDir)); - - nlohmann::json json; - json["storePath"] = hgInfo.storePath; - json["uri"] = uri; - json["name"] = name; - json["branch"] = hgInfo.branch; - json["rev"] = hgInfo.rev; - json["revCount"] = hgInfo.revCount; - - writeFile(storeLink, json.dump()); - - return hgInfo; -} - static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) { std::string url; - std::string rev; + std::optional rev; + std::optional ref; std::string name = "source"; PathSet context; @@ -182,8 +26,15 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar string n(attr.name); if (n == "url") url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "rev") - rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "rev") { + // Ugly: unlike fetchGit, here the "rev" attribute can + // be both a revision or a branch/tag name. + auto value = state.forceStringNoCtx(*attr.value, *attr.pos); + if (std::regex_match(value, revRegex)) + rev = Hash(value, htSHA1); + else + ref = value; + } else if (n == "name") name = state.forceStringNoCtx(*attr.value, *attr.pos); else @@ -200,18 +51,36 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // whitelist. Ah well. state.checkURI(url); - auto hgInfo = exportMercurial(state.store, url, rev, name); + if (evalSettings.pureEval && !rev) + throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); + + auto parsedUrl = parseURL( + url.find("://") != std::string::npos + ? "hg+" + url + : "hg+file://" + url); + if (rev) parsedUrl.query.insert_or_assign("rev", rev->gitRev()); + if (ref) parsedUrl.query.insert_or_assign("ref", *ref); + // FIXME: use name + auto input = fetchers::inputFromURL(parsedUrl); + + auto [tree, input2] = input->fetchTree(state.store); state.mkAttrs(v, 8); - mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath})); - mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch); - mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12)); - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); + auto storePath = state.store->printStorePath(tree.storePath); + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + if (input2->getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + // Backward compatibility: set 'rev' to + // 0000000000000000000000000000000000000000 for a dirty tree. + auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); v.attrs->sort(); if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); + state.allowedPaths->insert(tree.actualPath); } static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc new file mode 100644 index 000000000..9586f71ed --- /dev/null +++ b/src/libexpr/primops/fetchTree.cc @@ -0,0 +1,165 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "download.hh" + +#include +#include + +namespace nix { + +void emitTreeAttrs( + EvalState & state, + const fetchers::Tree & tree, + std::shared_ptr input, + Value & v) +{ + state.mkAttrs(v, 8); + + auto storePath = state.store->printStorePath(tree.storePath); + + mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + + assert(tree.info.narHash); + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + tree.info.narHash.to_string(SRI)); + + if (input->getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + } + + if (tree.info.revCount) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + + if (tree.info.lastModified) + mkString(*state.allocAttr(v, state.symbols.create("lastModified")), + fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + + v.attrs->sort(); +} + +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + + std::shared_ptr input; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + fetchers::Attrs attrs; + + for (auto & attr : *args[0]->attrs) { + state.forceValue(*attr.value); + if (attr.value->type == tString) + attrs.emplace(attr.name, attr.value->string.s); + else if (attr.value->type == tBool) + attrs.emplace(attr.name, attr.value->boolean); + else + throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", + attr.name, showType(*attr.value)); + } + + if (!attrs.count("type")) + throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos); + + input = fetchers::inputFromAttrs(attrs); + } else + input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + + if (evalSettings.pureEval && !input->isImmutable()) + throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); + + // FIXME: use fetchOrSubstituteTree + auto [tree, input2] = input->fetchTree(state.store); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + emitTreeAttrs(state, tree, input2, v); +} + +static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); + +static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, + const string & who, bool unpack, std::string name) +{ + std::optional url; + std::optional expectedHash; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError("unsupported argument '%s' to '%s', at %s", + attr.name, who, attr.pos); + } + + if (!url) + throw EvalError("'url' argument required, at %s", pos); + + } else + url = state.forceStringNoCtx(*args[0], pos); + + url = resolveUri(*url); + + state.checkURI(*url); + + if (name == "") + name = baseNameOf(*url); + + if (evalSettings.pureEval && !expectedHash) + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + + auto storePath = + unpack + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + + auto path = state.store->toRealPath(storePath); + + if (expectedHash) { + auto hash = unpack + ? state.store->queryPathInfo(storePath)->narHash + : hashFile(htSHA256, path); + if (hash != *expectedHash) + throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + *url, expectedHash->to_string(), hash.to_string()); + } + + if (state.allowedPaths) + state.allowedPaths->insert(path); + + mkString(v, path, PathSet({path})); +} + +static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchurl", false, ""); +} + +static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchTarball", true, "source"); +} + +static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); +static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); + +} -- cgit v1.2.3