From 75b62e52600a44b42693944b50638bf580a2c86e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jun 2020 17:54:16 +0000 Subject: Avoid `fmt` when constructor already does it There is a correctnes issue here, but #3724 will fix that. This is just a cleanup for brevity's sake. --- src/libfetchers/mercurial.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 1beb8b944..5c5671681 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -36,7 +36,7 @@ static std::string runHg(const Strings & args, const std::optional auto res = runProgram(std::move(opts)); if (!statusOk(res.first)) - throw ExecError(res.first, fmt("hg %1%", statusToString(res.first))); + throw ExecError(res.first, "hg %1%", statusToString(res.first)); return res.second; } @@ -273,7 +273,7 @@ struct MercurialInputScheme : InputScheme runHg({ "recover", "-R", cacheDir }); runHg({ "pull", "-R", cacheDir, "--", actualUrl }); } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + throw ExecError(e.status, "'hg pull' %s", statusToString(e.status)); } } } else { -- cgit v1.2.3 From 0256e5578e97a11db66207e1f8e231db115c91f8 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 23 Apr 2022 23:04:35 +0200 Subject: libfetchers/git: hardcode `--git-dir` To demonstrate the problem: * You need a `git` at 2.33.3 in your $PATH * An expression like this in a git repository: ``` nix { outputs = { self, nixpkgs }: { packages.foo.x86_64-linux = with nixpkgs.legacyPackages.x86_64-linux; runCommand "snens" { } '' echo ${(builtins.fetchGit ./.).lastModifiedDate} > $out ''; }; } ``` Now, when instantiating the package via `builtins.getFlake`, it fails on Nix 2.7 like this: $ nix-instantiate -E '(builtins.getFlake "'"$(pwd)"'").packages.foo.x86_64-linux' fatal: unsafe repository ('/nix/store/a7j3125km4h8l0p71q6ssfkxamfh5d61-source' is owned by someone else) To add an exception for this directory, call: git config --global --add safe.directory /nix/store/a7j3125km4h8l0p71q6ssfkxamfh5d61-source error: program 'git' failed with exit code 128 (use '--show-trace' to show detailed location information) This breaks e.g. `nixops`-deployments using flakes with similar expressions as shown above. The cause for this is that `git(1)` tries to find the highest `.git`-directory in the directory tree and if it finds a such a directory, but with another owning user (root vs. the user who evaluates the expression), it fails as above. This was changed recently to fix CVE-2022-24765[1]. By explicitly specifying `--git-dir`, Git assumes to be in the top-level directory and doesn't attempt to look for a `.git`-directory in the parent directories and thus the code-path leading to said error is never reached. [1] https://lore.kernel.org/git/xmqqv8veb5i6.fsf@gitster.g/ --- src/libfetchers/git.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 34b1342a0..219a5ca7a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -23,7 +23,7 @@ const std::string gitInitialBranch = "__nix_dummy_branch"; static std::string readHead(const Path & path) { - return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); + return chomp(runProgram("git", true, { "-C", path, "--git-dir", ".git", "rev-parse", "--abbrev-ref", "HEAD" })); } static bool isNotDotGitDirectory(const Path & path) @@ -152,11 +152,11 @@ struct GitInputScheme : InputScheme assert(sourcePath); runProgram("git", true, - { "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) }); + { "-C", *sourcePath, "--git-dir", ".git", "add", "--force", "--intent-to-add", "--", std::string(file) }); if (commitMsg) runProgram("git", true, - { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); + { "-C", *sourcePath, "--git-dir", ".git", "commit", std::string(file), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const @@ -259,7 +259,7 @@ struct GitInputScheme : InputScheme if (hasHead) { // Using git diff is preferrable over lower-level operations here, // because its conceptually simpler and we only need the exit code anyways. - auto gitDiffOpts = Strings({ "-C", actualUrl, "diff", "HEAD", "--quiet"}); + auto gitDiffOpts = Strings({ "-C", actualUrl, "--git-dir", ".git", "diff", "HEAD", "--quiet"}); if (!submodules) { // Changes in submodules should only make the tree dirty // when those submodules will be copied as well. @@ -284,7 +284,7 @@ struct GitInputScheme : InputScheme if (fetchSettings.warnDirty) warn("Git tree '%s' is dirty", actualUrl); - auto gitOpts = Strings({ "-C", actualUrl, "ls-files", "-z" }); + auto gitOpts = Strings({ "-C", actualUrl, "--git-dir", ".git", "ls-files", "-z" }); if (submodules) gitOpts.emplace_back("--recurse-submodules"); @@ -314,7 +314,7 @@ struct GitInputScheme : InputScheme // modified dirty file? input.attrs.insert_or_assign( "lastModified", - hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", ".git", "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return {std::move(storePath), input}; } @@ -335,7 +335,7 @@ struct GitInputScheme : InputScheme if (!input.getRev()) input.attrs.insert_or_assign("rev", - Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); + Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", ".git", "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; -- cgit v1.2.3 From d1f5356311bff3cb7840a82e6b35aeb66a1f6416 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 24 Apr 2022 18:06:36 +0200 Subject: libfetchers/git: fix for nixos-rebuild The `--git-dir=` must be `.` in some cases (for cached repos that are "bare" repos in `~/.cache/nix/gitv3`). With this fix we can add `--git-dir` to each `git`-invokation needed for `nixos-rebuild`. --- src/libfetchers/git.cc | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 219a5ca7a..af40990e5 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -21,6 +21,15 @@ namespace nix::fetchers { // old version of git, which will ignore unrecognized `-c` options. const std::string gitInitialBranch = "__nix_dummy_branch"; +static std::string getGitDir() +{ + auto gitDir = getEnv("GIT_DIR"); + if (!gitDir) { + return ".git"; + } + return *gitDir; +} + static std::string readHead(const Path & path) { return chomp(runProgram("git", true, { "-C", path, "--git-dir", ".git", "rev-parse", "--abbrev-ref", "HEAD" })); @@ -150,13 +159,14 @@ struct GitInputScheme : InputScheme { auto sourcePath = getSourcePath(input); assert(sourcePath); + auto gitDir = getGitDir(); runProgram("git", true, - { "-C", *sourcePath, "--git-dir", ".git", "add", "--force", "--intent-to-add", "--", std::string(file) }); + { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) }); if (commitMsg) runProgram("git", true, - { "-C", *sourcePath, "--git-dir", ".git", "commit", std::string(file), "-m", *commitMsg }); + { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const @@ -175,6 +185,7 @@ struct GitInputScheme : InputScheme std::pair fetch(ref store, const Input & _input) override { Input input(_input); + auto gitDir = getGitDir(); std::string name = input.getName(); @@ -237,7 +248,7 @@ struct GitInputScheme : InputScheme since that is the refrence we want to use later on. */ auto result = runProgram(RunOptions { .program = "git", - .args = { "-C", actualUrl, "--git-dir=.git", "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, + .args = { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, .environment = env, .mergeStderrToStdout = true }); @@ -259,7 +270,7 @@ struct GitInputScheme : InputScheme if (hasHead) { // Using git diff is preferrable over lower-level operations here, // because its conceptually simpler and we only need the exit code anyways. - auto gitDiffOpts = Strings({ "-C", actualUrl, "--git-dir", ".git", "diff", "HEAD", "--quiet"}); + auto gitDiffOpts = Strings({ "-C", actualUrl, "--git-dir", gitDir, "diff", "HEAD", "--quiet"}); if (!submodules) { // Changes in submodules should only make the tree dirty // when those submodules will be copied as well. @@ -284,7 +295,7 @@ struct GitInputScheme : InputScheme if (fetchSettings.warnDirty) warn("Git tree '%s' is dirty", actualUrl); - auto gitOpts = Strings({ "-C", actualUrl, "--git-dir", ".git", "ls-files", "-z" }); + auto gitOpts = Strings({ "-C", actualUrl, "--git-dir", gitDir, "ls-files", "-z" }); if (submodules) gitOpts.emplace_back("--recurse-submodules"); @@ -314,7 +325,7 @@ struct GitInputScheme : InputScheme // modified dirty file? input.attrs.insert_or_assign( "lastModified", - hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", ".git", "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return {std::move(storePath), input}; } @@ -335,7 +346,7 @@ struct GitInputScheme : InputScheme if (!input.getRev()) input.attrs.insert_or_assign("rev", - Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", ".git", "rev-parse", *input.getRef() })), htSHA1).gitRev()); + Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; @@ -351,6 +362,7 @@ struct GitInputScheme : InputScheme Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); repoDir = cacheDir; + gitDir = "."; createDirs(dirOf(cacheDir)); PathLocks cacheDirLock({cacheDir + ".lock"}); @@ -427,7 +439,7 @@ struct GitInputScheme : InputScheme // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } - bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true"; if (isShallow && !shallow) throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); -- cgit v1.2.3 From 401e60f2893d689772f716cbd800ee2ee0309362 Mon Sep 17 00:00:00 2001 From: Kjetil Orbekk Date: Tue, 2 Mar 2021 20:13:20 -0500 Subject: Resolve reference for remote repository Resolves the HEAD reference from the remote repository instead of assuming "master". --- src/libfetchers/git.cc | 104 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 98 insertions(+), 6 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index af40990e5..6b83638f6 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -5,9 +5,11 @@ #include "store-api.hh" #include "url-parts.hh" #include "pathlocks.hh" +#include "util.hh" #include "fetch-settings.hh" +#include #include #include @@ -19,7 +21,7 @@ namespace nix::fetchers { // The value itself does not matter, since we always fetch a specific revision or branch. // It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with // old version of git, which will ignore unrecognized `-c` options. -const std::string gitInitialBranch = "__nix_dummy_branch"; +static const std::string gitInitialBranch = "__nix_dummy_branch"; static std::string getGitDir() { @@ -30,9 +32,92 @@ static std::string getGitDir() return *gitDir; } -static std::string readHead(const Path & path) +static bool isCacheFileWithinTtl(const time_t now, const struct stat& st) { + return st.st_mtime + settings.tarballTtl > now; +} + +static Path getCachePath(std::string key) { + return getCacheDir() + "/nix/gitv3/" + + hashString(htSHA256, key).to_string(Base32, false); +} + +// Returns the name of the HEAD branch. +// +// Returns the head branch name as reported by git ls-remote --symref, e.g., if +// ls-remote returns the output below, "main" is returned based on the ref line. +// +// ref: refs/heads/main HEAD +// ... +static std::optional readHead(const Path & path) +{ + auto [exit_code, output] = runProgram(RunOptions { + .program = "git", + .args = {"ls-remote", "--symref", path}, + }); + if (exit_code != 0) { + return std::nullopt; + } + + // Matches the common case when HEAD points to a branch, e.g.: + // "ref: refs/heads/main HEAD". + const static std::regex head_ref_regex("^ref:\\s*([^\\s]+)\\s*HEAD$"); + // Matches when HEAD points directly at a commit, e.g.: + // "71abcd... HEAD". + const static std::regex head_rev_regex("^([^\\s]+)\\s*HEAD$"); + + for (const auto & line : tokenizeString>(output, "\n")) { + std::smatch match; + if (std::regex_match(line, match, head_ref_regex)) { + debug("resolved HEAD ref '%s' for repo '%s'", match[1], path); + return match[1]; + } else if (std::regex_match(line, match, head_rev_regex)) { + debug("resolved HEAD ref '%s' for repo '%s'", match[1], path); + return match[1]; + } + } + + return std::nullopt; +} + +static std::optional readHeadCached(std::string actualUrl) { - return chomp(runProgram("git", true, { "-C", path, "--git-dir", ".git", "rev-parse", "--abbrev-ref", "HEAD" })); + // Create a cache path to store the branch of the HEAD ref. Append something + // in front of the URL to prevent collision with the repository itself. + Path cachePath = getCachePath("|" + actualUrl); + time_t now = time(0); + struct stat st; + std::optional cachedRef; + if (stat(cachePath.c_str(), &st) == 0 && + // The file may be empty, because writeFile() is not atomic. + st.st_size > 0) { + + // The cached ref is persisted unconditionally (see below). + cachedRef = readFile(cachePath); + if (isCacheFileWithinTtl(now, st)) { + debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl); + return cachedRef; + } + } + + auto ref = readHead(actualUrl); + + if (ref) { + debug("storing cached HEAD ref '%s' for repo '%s' at '%s'", *ref, actualUrl, cachePath); + createDirs(dirOf(cachePath)); + writeFile(cachePath, *ref); + return ref; + } + + if (cachedRef) { + // If the cached git ref is expired in fetch() below, and the 'git fetch' + // fails, it falls back to continuing with the most recent version. + // This function must behave the same way, so we return the expired + // cached ref here. + warn("could not get HEAD ref for repository '%s'; using expired cached ref '%s'", actualUrl, *cachedRef); + return *cachedRef; + } + + return std::nullopt; } static bool isNotDotGitDirectory(const Path & path) @@ -331,7 +416,14 @@ struct GitInputScheme : InputScheme } } - if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master"); + if (!input.getRef()) { + auto head = isLocal ? readHead(actualUrl) : readHeadCached(actualUrl); + if (!head) { + warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); + head = "master"; + } + input.attrs.insert_or_assign("ref", *head); + } Attrs unlockedAttrs({ {"type", cacheType}, @@ -360,7 +452,7 @@ struct GitInputScheme : InputScheme } } - Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); + Path cacheDir = getCachePath(actualUrl); repoDir = cacheDir; gitDir = "."; @@ -400,7 +492,7 @@ struct GitInputScheme : InputScheme git fetch to update the local ref to the remote ref. */ struct stat st; doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + !isCacheFileWithinTtl(now, st); } } -- cgit v1.2.3 From de54e1cd3fce6eb456048b6a346a0be2b88660ae Mon Sep 17 00:00:00 2001 From: Kjetil Orbekk Date: Sat, 29 Jan 2022 14:12:01 -0500 Subject: Refactor fetching of dirty workdir Extract the handling of a local dirty workdir to a helper function. --- src/libfetchers/git.cc | 224 +++++++++++++++++++++++++++---------------------- 1 file changed, 123 insertions(+), 101 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 6b83638f6..00a7b85d8 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -16,14 +16,15 @@ using namespace std::string_literals; namespace nix::fetchers { +namespace { // Explicit initial branch of our bare repo to suppress warnings from new version of git. // The value itself does not matter, since we always fetch a specific revision or branch. // It is set with `-c init.defaultBranch=` instead of `--initial-branch=` to stay compatible with // old version of git, which will ignore unrecognized `-c` options. -static const std::string gitInitialBranch = "__nix_dummy_branch"; +const std::string gitInitialBranch = "__nix_dummy_branch"; -static std::string getGitDir() +std::string getGitDir() { auto gitDir = getEnv("GIT_DIR"); if (!gitDir) { @@ -32,11 +33,11 @@ static std::string getGitDir() return *gitDir; } -static bool isCacheFileWithinTtl(const time_t now, const struct stat& st) { +bool isCacheFileWithinTtl(const time_t now, const struct stat& st) { return st.st_mtime + settings.tarballTtl > now; } -static Path getCachePath(std::string key) { +Path getCachePath(std::string key) { return getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, key).to_string(Base32, false); } @@ -48,7 +49,7 @@ static Path getCachePath(std::string key) { // // ref: refs/heads/main HEAD // ... -static std::optional readHead(const Path & path) +std::optional readHead(const Path & path) { auto [exit_code, output] = runProgram(RunOptions { .program = "git", @@ -79,7 +80,7 @@ static std::optional readHead(const Path & path) return std::nullopt; } -static std::optional readHeadCached(std::string actualUrl) +std::optional readHeadCached(std::string actualUrl) { // Create a cache path to store the branch of the HEAD ref. Append something // in front of the URL to prevent collision with the repository itself. @@ -120,11 +121,120 @@ static std::optional readHeadCached(std::string actualUrl) return std::nullopt; } -static bool isNotDotGitDirectory(const Path & path) +bool isNotDotGitDirectory(const Path & path) { return baseNameOf(path) != ".git"; } +struct WorkdirInfo +{ + bool clean = false; + bool hasHead = false; +}; + +// Returns whether a git workdir is clean and has commits. +WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) +{ + const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + auto gitDir = getGitDir(); + + auto env = getEnv(); + // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong + // that way unknown errors can lead to a failure instead of continuing through the wrong code path + env["LC_ALL"] = "C"; + + /* Check whether HEAD points to something that looks like a commit, + since that is the refrence we want to use later on. */ + auto result = runProgram(RunOptions { + .program = "git", + .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, + .environment = env, + .mergeStderrToStdout = true + }); + auto exitCode = WEXITSTATUS(result.first); + auto errorMessage = result.second; + + if (errorMessage.find("fatal: not a git repository") != std::string::npos) { + throw Error("'%s' is not a Git repository", workdir); + } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) { + // indicates that the repo does not have any commits + // we want to proceed and will consider it dirty later + } else if (exitCode != 0) { + // any other errors should lead to a failure + throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage); + } + + bool clean = false; + bool hasHead = exitCode == 0; + + try { + if (hasHead) { + // Using git diff is preferrable over lower-level operations here, + // because its conceptually simpler and we only need the exit code anyways. + auto gitDiffOpts = Strings({ "-C", workdir, "diff", "HEAD", "--quiet"}); + if (!submodules) { + // Changes in submodules should only make the tree dirty + // when those submodules will be copied as well. + gitDiffOpts.emplace_back("--ignore-submodules"); + } + gitDiffOpts.emplace_back("--"); + runProgram("git", true, gitDiffOpts); + + clean = true; + } + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + } + + return WorkdirInfo { .clean = clean, .hasHead = hasHead }; +} + +std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) +{ + const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + + if (!fetchSettings.allowDirty) + throw Error("Git tree '%s' is dirty", workdir); + + if (fetchSettings.warnDirty) + warn("Git tree '%s' is dirty", workdir); + + auto gitOpts = Strings({ "-C", workdir, "ls-files", "-z" }); + if (submodules) + gitOpts.emplace_back("--recurse-submodules"); + + auto files = tokenizeString>( + runProgram("git", true, gitOpts), "\0"s); + + Path actualPath(absPath(workdir)); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualPath)); + std::string file(p, actualPath.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + input.attrs.insert_or_assign( + "lastModified", + workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + + return {std::move(storePath), input}; +} +} // end namespace + struct GitInputScheme : InputScheme { std::optional inputFromURL(const ParsedURL & url) override @@ -319,100 +429,12 @@ struct GitInputScheme : InputScheme auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug - // If this is a local directory and no ref or revision is - // given, then allow the use of an unclean working tree. + /* If this is a local directory and no ref or revision is given, + allow fetching directly from a dirty workdir. */ if (!input.getRef() && !input.getRev() && isLocal) { - bool clean = false; - - auto env = getEnv(); - // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong - // that way unknown errors can lead to a failure instead of continuing through the wrong code path - env["LC_ALL"] = "C"; - - /* Check whether HEAD points to something that looks like a commit, - since that is the refrence we want to use later on. */ - auto result = runProgram(RunOptions { - .program = "git", - .args = { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, - .environment = env, - .mergeStderrToStdout = true - }); - auto exitCode = WEXITSTATUS(result.first); - auto errorMessage = result.second; - - if (errorMessage.find("fatal: not a git repository") != std::string::npos) { - throw Error("'%s' is not a Git repository", actualUrl); - } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) { - // indicates that the repo does not have any commits - // we want to proceed and will consider it dirty later - } else if (exitCode != 0) { - // any other errors should lead to a failure - throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", actualUrl, exitCode, errorMessage); - } - - bool hasHead = exitCode == 0; - try { - if (hasHead) { - // Using git diff is preferrable over lower-level operations here, - // because its conceptually simpler and we only need the exit code anyways. - auto gitDiffOpts = Strings({ "-C", actualUrl, "--git-dir", gitDir, "diff", "HEAD", "--quiet"}); - if (!submodules) { - // Changes in submodules should only make the tree dirty - // when those submodules will be copied as well. - gitDiffOpts.emplace_back("--ignore-submodules"); - } - gitDiffOpts.emplace_back("--"); - runProgram("git", true, gitDiffOpts); - - clean = true; - } - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - } - - if (!clean) { - - /* This is an unclean working tree. So copy all tracked files. */ - - if (!fetchSettings.allowDirty) - throw Error("Git tree '%s' is dirty", actualUrl); - - if (fetchSettings.warnDirty) - warn("Git tree '%s' is dirty", actualUrl); - - auto gitOpts = Strings({ "-C", actualUrl, "--git-dir", gitDir, "ls-files", "-z" }); - if (submodules) - gitOpts.emplace_back("--recurse-submodules"); - - auto files = tokenizeString>( - runProgram("git", true, gitOpts), "\0"s); - - Path actualPath(absPath(actualUrl)); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualPath)); - std::string file(p, actualPath.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); - - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - input.attrs.insert_or_assign( - "lastModified", - hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); - - return {std::move(storePath), input}; + auto workdirInfo = getWorkdirInfo(input, actualUrl); + if (!workdirInfo.clean) { + return fetchFromWorkdir(store, input, actualUrl, workdirInfo); } } @@ -425,7 +447,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("ref", *head); } - Attrs unlockedAttrs({ + const Attrs unlockedAttrs({ {"type", cacheType}, {"name", name}, {"url", actualUrl}, -- cgit v1.2.3 From 1203e489263fe867d0a1ae3fd9270f40c8a1c24e Mon Sep 17 00:00:00 2001 From: Kjetil Orbekk Date: Sat, 29 Jan 2022 14:22:55 -0500 Subject: Store cached head in cached git repo The previous head caching implementation stored two paths in the local cache; one for the cached git repo and another textfile containing the resolved HEAD ref. This commit instead stores the resolved HEAD by setting the HEAD ref in the local cache appropriately. --- src/libfetchers/git.cc | 89 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 31 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 00a7b85d8..968cd642a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -10,6 +10,7 @@ #include "fetch-settings.hh" #include +#include #include #include @@ -37,7 +38,19 @@ bool isCacheFileWithinTtl(const time_t now, const struct stat& st) { return st.st_mtime + settings.tarballTtl > now; } -Path getCachePath(std::string key) { +bool touchCacheFile(const Path& path, const time_t& touch_time) +{ + struct timeval times[2]; + times[0].tv_sec = touch_time; + times[0].tv_usec = 0; + times[1].tv_sec = touch_time; + times[1].tv_usec = 0; + + return lutimes(path.c_str(), times) == 0; +} + +Path getCachePath(std::string key) +{ return getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, key).to_string(Base32, false); } @@ -80,32 +93,42 @@ std::optional readHead(const Path & path) return std::nullopt; } -std::optional readHeadCached(std::string actualUrl) +// Persist the HEAD ref from the remote repo in the local cached repo. +bool storeCachedHead(const std::string& actualUrl, const std::string& headRef) +{ + Path cacheDir = getCachePath(actualUrl); + try { + runProgram("git", true, { "-C", cacheDir, "symbolic-ref", "--", "HEAD", headRef }); + } catch (ExecError &e) { + if (!WIFEXITED(e.status)) throw; + return false; + } + /* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */ + return true; +} + +std::optional readHeadCached(const std::string& actualUrl) { // Create a cache path to store the branch of the HEAD ref. Append something // in front of the URL to prevent collision with the repository itself. - Path cachePath = getCachePath("|" + actualUrl); + Path cacheDir = getCachePath(actualUrl); + Path headRefFile = cacheDir + "/HEAD"; + time_t now = time(0); struct stat st; std::optional cachedRef; - if (stat(cachePath.c_str(), &st) == 0 && - // The file may be empty, because writeFile() is not atomic. - st.st_size > 0) { - - // The cached ref is persisted unconditionally (see below). - cachedRef = readFile(cachePath); - if (isCacheFileWithinTtl(now, st)) { + if (stat(headRefFile.c_str(), &st) == 0) { + cachedRef = readHead(cacheDir); + if (cachedRef != std::nullopt && + *cachedRef != gitInitialBranch && + isCacheFileWithinTtl(now, st)) { debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl); return cachedRef; } } auto ref = readHead(actualUrl); - if (ref) { - debug("storing cached HEAD ref '%s' for repo '%s' at '%s'", *ref, actualUrl, cachePath); - createDirs(dirOf(cachePath)); - writeFile(cachePath, *ref); return ref; } @@ -438,15 +461,6 @@ struct GitInputScheme : InputScheme } } - if (!input.getRef()) { - auto head = isLocal ? readHead(actualUrl) : readHeadCached(actualUrl); - if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); - head = "master"; - } - input.attrs.insert_or_assign("ref", *head); - } - const Attrs unlockedAttrs({ {"type", cacheType}, {"name", name}, @@ -457,14 +471,30 @@ struct GitInputScheme : InputScheme Path repoDir; if (isLocal) { + if (!input.getRef()) { + auto head = readHead(actualUrl); + if (!head) { + warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); + head = "master"; + } + input.attrs.insert_or_assign("ref", *head); + } if (!input.getRev()) input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; - } else { + const bool useHeadRef = !input.getRef(); + if (useHeadRef) { + auto head = readHeadCached(actualUrl); + if (!head) { + warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); + head = "master"; + } + input.attrs.insert_or_assign("ref", *head); + } if (auto res = getCache()->lookup(store, unlockedAttrs)) { auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); @@ -538,13 +568,10 @@ struct GitInputScheme : InputScheme warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); } - struct timeval times[2]; - times[0].tv_sec = now; - times[0].tv_usec = 0; - times[1].tv_sec = now; - times[1].tv_usec = 0; - - utimes(localRefFile.c_str(), times); + if (!touchCacheFile(localRefFile, now)) + warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno)); + if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef())) + warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl); } if (!input.getRev()) -- cgit v1.2.3 From 9bf296c970bf33b7ed53d7e2d8fbe44197482518 Mon Sep 17 00:00:00 2001 From: Kjetil Orbekk Date: Fri, 29 Apr 2022 18:30:00 -0400 Subject: Extract git reference parsing to a shared library These utility functions can be shared between the git and github fetchers. --- src/libfetchers/git-utils.cc | 25 +++++++++++++++++++++++++ src/libfetchers/git-utils.hh | 23 +++++++++++++++++++++++ src/libfetchers/git.cc | 29 +++++++++++------------------ src/libfetchers/github.cc | 28 +++++++++++----------------- 4 files changed, 70 insertions(+), 35 deletions(-) create mode 100644 src/libfetchers/git-utils.cc create mode 100644 src/libfetchers/git-utils.hh (limited to 'src/libfetchers') diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc new file mode 100644 index 000000000..060077098 --- /dev/null +++ b/src/libfetchers/git-utils.cc @@ -0,0 +1,25 @@ +#include "git-utils.hh" + +#include + +std::optional parseListReferenceHeadRef(std::string_view line) { + const static std::regex head_ref_regex("^ref: ([^\\s]+)\\t+HEAD$"); + std::match_results match; + if (std::regex_match(line.cbegin(), line.cend(), match, head_ref_regex)) { + return match[1]; + } else { + return std::nullopt; + } +} + +std::optional parseListReferenceForRev(std::string_view rev, std::string_view line) { + const static std::regex rev_regex("^([^\\t]+)\\t+(.*)$"); + std::match_results match; + if (!std::regex_match(line.cbegin(), line.cend(), match, rev_regex)) { + return std::nullopt; + } + if (rev != match[2].str()) { + return std::nullopt; + } + return match[1]; +} diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh new file mode 100644 index 000000000..946a68a9e --- /dev/null +++ b/src/libfetchers/git-utils.hh @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include + +// Parses the HEAD ref as reported by `git ls-remote --symref` +// +// Returns the head branch name as reported by `git ls-remote --symref`, e.g., if +// ls-remote returns the output below, "main" is returned based on the ref line. +// +// ref: refs/heads/main HEAD +// +// If the repository is in 'detached head' state (HEAD is pointing to a rev +// instead of a branch), parseListReferenceForRev("HEAD") may be used instead. +std::optional parseListReferenceHeadRef(std::string_view line); + +// Parses a reference line from `git ls-remote --symref`, e.g., +// parseListReferenceForRev("refs/heads/master", line) will return 6926... +// given the line below. +// +// 6926beab444c33fb57b21819b6642d032016bb1e refs/heads/master +std::optional parseListReferenceForRev(std::string_view rev, std::string_view line); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 968cd642a..9d4348cf1 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -6,6 +6,7 @@ #include "url-parts.hh" #include "pathlocks.hh" #include "util.hh" +#include "git-utils.hh" #include "fetch-settings.hh" @@ -69,27 +70,19 @@ std::optional readHead(const Path & path) .args = {"ls-remote", "--symref", path}, }); if (exit_code != 0) { - return std::nullopt; + return std::nullopt; } - // Matches the common case when HEAD points to a branch, e.g.: - // "ref: refs/heads/main HEAD". - const static std::regex head_ref_regex("^ref:\\s*([^\\s]+)\\s*HEAD$"); - // Matches when HEAD points directly at a commit, e.g.: - // "71abcd... HEAD". - const static std::regex head_rev_regex("^([^\\s]+)\\s*HEAD$"); - - for (const auto & line : tokenizeString>(output, "\n")) { - std::smatch match; - if (std::regex_match(line, match, head_ref_regex)) { - debug("resolved HEAD ref '%s' for repo '%s'", match[1], path); - return match[1]; - } else if (std::regex_match(line, match, head_rev_regex)) { - debug("resolved HEAD ref '%s' for repo '%s'", match[1], path); - return match[1]; - } + std::string_view line = output; + line = line.substr(0, line.find("\n")); + if (const auto ref = parseListReferenceHeadRef(line); ref) { + debug("resolved HEAD ref '%s' for repo '%s'", *ref, path); + return *ref; + } + if (const auto rev = parseListReferenceForRev("HEAD", line); rev) { + debug("resolved HEAD rev '%s' for repo '%s'", *rev, path); + return *rev; } - return std::nullopt; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 58b6e7c04..1bdf2759f 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -4,7 +4,7 @@ #include "store-api.hh" #include "types.hh" #include "url-parts.hh" - +#include "git-utils.hh" #include "fetchers.hh" #include "fetch-settings.hh" @@ -383,35 +383,29 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string line; getline(is, line); - auto ref_index = line.find("ref: "); - if (ref_index == std::string::npos) { + auto r = parseListReferenceHeadRef(line); + if (!r) { throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref); } - - ref_uri = line.substr(ref_index+5, line.length()-1); - } else + ref_uri = *r; + } else { ref_uri = fmt("refs/(heads|tags)/%s", ref); + } auto file = store->toRealPath( downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); std::ifstream is(file); std::string line; - std::string id; - while(getline(is, line)) { - // Append $ to avoid partial name matches - std::regex pattern(fmt("%s$", ref_uri)); - - if (std::regex_search(line, pattern)) { - id = line.substr(0, line.find('\t')); - break; - } + std::optional id; + while(!id && getline(is, line)) { + id = parseListReferenceForRev(ref_uri, line); } - if(id.empty()) + if(!id) throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); - auto rev = Hash::parseAny(id, htSHA1); + auto rev = Hash::parseAny(*id, htSHA1); debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev()); return rev; } -- cgit v1.2.3 From 1849e6a1f64734c488c2b1469249d65ce08cef93 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sat, 30 Apr 2022 15:56:12 +0200 Subject: libfetchers/git: fix every occasion of a permission error I'm afraid I missed a few problematic `git(1)`-calls while implementing PR #6440, sorry for that! Upon investigating what went wrong, I realized that I only tested against the "cached"-case by accident because my git-checkout with my system's flake was apparently cached during my debugging. I managed to trigger the original issue again by running: $ git commit --allow-empty -m "tmp" $ sudo nixos-rebuild switch --flake .# -L --builders '' Since `repoDir` points to the checkout that's potentially owned by another user, I decided to add `--git-dir` to each call affecting `repoDir`. Since the `tmpDir` for the temporary submodule-checkout is created by Nix itself, it doesn't seem to be an issue. Sorry for that, it should be fine now. --- src/libfetchers/git.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index af40990e5..2ff27795d 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -383,7 +383,7 @@ struct GitInputScheme : InputScheme repo. */ if (input.getRev()) { try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() }); + runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -418,7 +418,7 @@ struct GitInputScheme : InputScheme : ref == "HEAD" ? *ref : "refs/heads/" + *ref; - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); + runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); @@ -459,7 +459,7 @@ struct GitInputScheme : InputScheme auto result = runProgram(RunOptions { .program = "git", - .args = { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }, + .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() }, .mergeStderrToStdout = true }); if (WEXITSTATUS(result.first) == 128 @@ -498,7 +498,7 @@ struct GitInputScheme : InputScheme auto source = sinkToSource([&](Sink & sink) { runProgram2({ .program = "git", - .args = { "-C", repoDir, "archive", input.getRev()->gitRev() }, + .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() }, .standardOut = &sink }); }); @@ -508,7 +508,7 @@ struct GitInputScheme : InputScheme auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, @@ -517,7 +517,7 @@ struct GitInputScheme : InputScheme if (!shallow) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); + std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() }))); if (!_input.getRev()) getCache()->add( -- cgit v1.2.3 From 61289ceee3417cd2aa8e8db9fe117a5813969aed Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 2 May 2022 13:37:53 +0200 Subject: Style fixes --- src/libfetchers/git-utils.cc | 6 ++++-- src/libfetchers/git.cc | 9 +++------ 2 files changed, 7 insertions(+), 8 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 060077098..b2d6b7893 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -2,7 +2,8 @@ #include -std::optional parseListReferenceHeadRef(std::string_view line) { +std::optional parseListReferenceHeadRef(std::string_view line) +{ const static std::regex head_ref_regex("^ref: ([^\\s]+)\\t+HEAD$"); std::match_results match; if (std::regex_match(line.cbegin(), line.cend(), match, head_ref_regex)) { @@ -12,7 +13,8 @@ std::optional parseListReferenceHeadRef(std::string_view line) { } } -std::optional parseListReferenceForRev(std::string_view rev, std::string_view line) { +std::optional parseListReferenceForRev(std::string_view rev, std::string_view line) +{ const static std::regex rev_regex("^([^\\t]+)\\t+(.*)$"); std::match_results match; if (!std::regex_match(line.cbegin(), line.cend(), match, rev_regex)) { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 2b81900fa..266246fe9 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -28,14 +28,11 @@ const std::string gitInitialBranch = "__nix_dummy_branch"; std::string getGitDir() { - auto gitDir = getEnv("GIT_DIR"); - if (!gitDir) { - return ".git"; - } - return *gitDir; + return getEnv("GIT_DIR").value_or(".git"); } -bool isCacheFileWithinTtl(const time_t now, const struct stat& st) { +bool isCacheFileWithinTtl(const time_t now, const struct stat & st) +{ return st.st_mtime + settings.tarballTtl > now; } -- cgit v1.2.3 From e68676e6c859815f40079b6340399d82cc1913b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 4 May 2022 14:32:21 +0200 Subject: Fix the parsing of the sourcehut refs file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since a26be9f3b89be2ee90c6358250b9889b37f95cf8, the same parser is used to parse the result of sourcehut’s `HEAD` endpoint (coming from [git dumb protocol]) and the output of `git ls-remote`. However, they are very slightly different (the former doesn’t specify the current reference since it’s implied to be `HEAD`). Unify both, and make the parser a bit more robust and understandable (by making it more typed and adding tests for it) [git dumb protocol]: https://git-scm.com/book/en/v2/Git-Internals-Transfer-Protocols#_the_dumb_protocol --- src/libfetchers/git-utils.cc | 27 --------------------------- src/libfetchers/git-utils.hh | 23 ----------------------- src/libfetchers/git.cc | 19 +++++++++++-------- src/libfetchers/github.cc | 12 +++++++----- 4 files changed, 18 insertions(+), 63 deletions(-) delete mode 100644 src/libfetchers/git-utils.cc delete mode 100644 src/libfetchers/git-utils.hh (limited to 'src/libfetchers') diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc deleted file mode 100644 index b2d6b7893..000000000 --- a/src/libfetchers/git-utils.cc +++ /dev/null @@ -1,27 +0,0 @@ -#include "git-utils.hh" - -#include - -std::optional parseListReferenceHeadRef(std::string_view line) -{ - const static std::regex head_ref_regex("^ref: ([^\\s]+)\\t+HEAD$"); - std::match_results match; - if (std::regex_match(line.cbegin(), line.cend(), match, head_ref_regex)) { - return match[1]; - } else { - return std::nullopt; - } -} - -std::optional parseListReferenceForRev(std::string_view rev, std::string_view line) -{ - const static std::regex rev_regex("^([^\\t]+)\\t+(.*)$"); - std::match_results match; - if (!std::regex_match(line.cbegin(), line.cend(), match, rev_regex)) { - return std::nullopt; - } - if (rev != match[2].str()) { - return std::nullopt; - } - return match[1]; -} diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh deleted file mode 100644 index 946a68a9e..000000000 --- a/src/libfetchers/git-utils.hh +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include -#include -#include - -// Parses the HEAD ref as reported by `git ls-remote --symref` -// -// Returns the head branch name as reported by `git ls-remote --symref`, e.g., if -// ls-remote returns the output below, "main" is returned based on the ref line. -// -// ref: refs/heads/main HEAD -// -// If the repository is in 'detached head' state (HEAD is pointing to a rev -// instead of a branch), parseListReferenceForRev("HEAD") may be used instead. -std::optional parseListReferenceHeadRef(std::string_view line); - -// Parses a reference line from `git ls-remote --symref`, e.g., -// parseListReferenceForRev("refs/heads/master", line) will return 6926... -// given the line below. -// -// 6926beab444c33fb57b21819b6642d032016bb1e refs/heads/master -std::optional parseListReferenceForRev(std::string_view rev, std::string_view line); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 266246fe9..d23a820a4 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -6,7 +6,7 @@ #include "url-parts.hh" #include "pathlocks.hh" #include "util.hh" -#include "git-utils.hh" +#include "git.hh" #include "fetch-settings.hh" @@ -72,13 +72,16 @@ std::optional readHead(const Path & path) std::string_view line = output; line = line.substr(0, line.find("\n")); - if (const auto ref = parseListReferenceHeadRef(line); ref) { - debug("resolved HEAD ref '%s' for repo '%s'", *ref, path); - return *ref; - } - if (const auto rev = parseListReferenceForRev("HEAD", line); rev) { - debug("resolved HEAD rev '%s' for repo '%s'", *rev, path); - return *rev; + if (const auto parseResult = git::parseLsRemoteLine(line)) { + switch (parseResult->kind) { + case git::LsRemoteRefLine::Kind::Symbolic: + debug("resolved HEAD ref '%s' for repo '%s'", parseResult->target, path); + break; + case git::LsRemoteRefLine::Kind::Object: + debug("resolved HEAD rev '%s' for repo '%s'", parseResult->target, path); + break; + } + return parseResult->target; } return std::nullopt; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1bdf2759f..a1084c984 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -4,7 +4,7 @@ #include "store-api.hh" #include "types.hh" #include "url-parts.hh" -#include "git-utils.hh" +#include "git.hh" #include "fetchers.hh" #include "fetch-settings.hh" @@ -383,11 +383,11 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string line; getline(is, line); - auto r = parseListReferenceHeadRef(line); - if (!r) { + auto remoteLine = git::parseLsRemoteLine(line); + if (!remoteLine) { throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref); } - ref_uri = *r; + ref_uri = remoteLine->target; } else { ref_uri = fmt("refs/(heads|tags)/%s", ref); } @@ -399,7 +399,9 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string line; std::optional id; while(!id && getline(is, line)) { - id = parseListReferenceForRev(ref_uri, line); + auto parsedLine = git::parseLsRemoteLine(line); + if (parsedLine && parsedLine->reference == ref_uri) + id = parsedLine->target; } if(!id) -- cgit v1.2.3 From 7a3d5b2ff0e39ae0d7b393f454671a08da56776b Mon Sep 17 00:00:00 2001 From: Daniel Fullmer Date: Thu, 5 May 2022 10:46:48 -0700 Subject: Use correct URL for GitHub Enterprise For GitHub Enterprise, the API is accessed through a slightly different URL. See [1], where it says: > Use http(s)://[hostname]/api/v3 to access the API for GitHub > Enterprise Server. Also tested working on a GHE instance. [1] https://docs.github.com/en/enterprise-server@3.3/rest/guides/getting-started-with-the-rest-api --- src/libfetchers/github.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1bdf2759f..50b3150ee 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -243,7 +243,10 @@ struct GitHubInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check + auto url = fmt( + host == "github.com" + ? "https://api.%s/repos/%s/%s/commits/%s" + : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); Headers headers = makeHeadersWithAuthTokens(host); @@ -262,7 +265,10 @@ struct GitHubInputScheme : GitArchiveInputScheme // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances + auto url = fmt( + host == "github.com" + ? "https://api.%s/repos/%s/%s/commits/%s" + : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); -- cgit v1.2.3 From 831e2743ea0bac57ebccb73f532667a194b938d7 Mon Sep 17 00:00:00 2001 From: Norbert Melzer Date: Thu, 12 May 2022 00:56:39 +0200 Subject: fix GitHub URL template --- src/libfetchers/github.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 0721a13f2..0631fb6e8 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -267,8 +267,8 @@ struct GitHubInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); auto url = fmt( host == "github.com" - ? "https://api.%s/repos/%s/%s/commits/%s" - : "https://%s/api/v3/repos/%s/%s/commits/%s", + ? "https://api.%s/repos/%s/%s/tarball/%s" + : "https://%s/api/v3/repos/%s/%s/tarball/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); -- cgit v1.2.3 From 5b8c1deb18e0e6fc7a83fb8101cf5fc8dba38843 Mon Sep 17 00:00:00 2001 From: Tony Olagbaiye Date: Fri, 16 Oct 2020 00:35:24 +0100 Subject: fetchTree: Allow fetching plain files Add a new `file` fetcher type, which will fetch a plain file over http(s), or from the local file. Because plain `http(s)://` or `file://` urls can already correspond to `tarball` inputs (if the path ends-up with a know archive extension), the URL parsing logic is a bit convuluted in that: - {http,https,file}:// urls will be interpreted as either a tarball or a file input, depending on the extensions of the path part (so `https://foo.com/bar` will be a `file` input and `https://foo.com/bar.tar.gz` as a `tarball` input) - `file+{something}://` urls will be interpreted as `file` urls (with the `file+` part removed) - `tarball+{something}://` urls will be interpreted as `tarball` urls (with the `tarball+` part removed) Fix #3785 Co-Authored-By: Tony Olagbaiye --- src/libfetchers/tarball.cc | 90 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 22 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index dde0ad761..09acb74d3 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -6,6 +6,7 @@ #include "archive.hh" #include "tarfile.hh" #include "types.hh" +#include "split.hh" namespace nix::fetchers { @@ -168,24 +169,34 @@ std::pair downloadTarball( }; } -struct TarballInputScheme : InputScheme +// An input scheme corresponding to a curable ressource +struct CurlInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + virtual const std::string inputType() const = 0; + const std::set transportUrlSchemes = {"file", "http", "https"}; + + const bool hasTarballExtension(std::string_view path) const { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {}; + return hasSuffix(path, ".zip") || hasSuffix(path, ".tar") + || hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz") + || hasSuffix(path, ".tar.xz") || hasSuffix(path, ".tar.bz2") + || hasSuffix(path, ".tar.zst"); + } - if (!hasSuffix(url.path, ".zip") - && !hasSuffix(url.path, ".tar") - && !hasSuffix(url.path, ".tgz") - && !hasSuffix(url.path, ".tar.gz") - && !hasSuffix(url.path, ".tar.xz") - && !hasSuffix(url.path, ".tar.bz2") - && !hasSuffix(url.path, ".tar.zst")) - return {}; + virtual bool isValidURL(const ParsedURL & url) const = 0; + + std::optional inputFromURL(const ParsedURL & url) override + { + if (!isValidURL(url)) + return std::nullopt; Input input; - input.attrs.insert_or_assign("type", "tarball"); - input.attrs.insert_or_assign("url", url.to_string()); + + auto urlWithoutApplicationScheme = url; + urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport; + + input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string()); auto narHash = url.query.find("narHash"); if (narHash != url.query.end()) input.attrs.insert_or_assign("narHash", narHash->second); @@ -194,14 +205,17 @@ struct TarballInputScheme : InputScheme std::optional inputFromAttrs(const Attrs & attrs) override { - if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + auto type = maybeGetStrAttr(attrs, "type"); + if (type != inputType()) return {}; + std::set allowedNames = {"type", "url", "narHash", "name", "unpack"}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name") - throw Error("unsupported tarball input attribute '%s'", name); + if (!allowedNames.count(name)) + throw Error("unsupported %s input attribute '%s'", *type, name); Input input; input.attrs = attrs; + //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); return input; } @@ -209,14 +223,9 @@ struct TarballInputScheme : InputScheme ParsedURL toURL(const Input & input) override { auto url = parseURL(getStrAttr(input.attrs, "url")); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. + // NAR hashes are preferred over file hashes since tar/zip files // don't have a canonical representation. if (auto narHash = input.getNarHash()) url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); - /* - else if (auto hash = maybeGetStrAttr(input.attrs, "hash")) - url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true)); - */ return url; } @@ -225,6 +234,42 @@ struct TarballInputScheme : InputScheme return true; } +}; + +struct FileInputScheme : CurlInputScheme +{ + const std::string inputType() const override { return "file"; } + + bool isValidURL(const ParsedURL & url) const override + { + auto parsedUrlScheme = parseUrlScheme(url.scheme); + return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) + && (parsedUrlScheme.application + ? parsedUrlScheme.application.value() == inputType() + : !hasTarballExtension(url.path)); + } + + std::pair fetch(ref store, const Input & input) override + { + auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); + return {std::move(file.storePath), input}; + } +}; + +struct TarballInputScheme : CurlInputScheme +{ + const std::string inputType() const override { return "tarball"; } + + bool isValidURL(const ParsedURL & url) const override + { + auto parsedUrlScheme = parseUrlScheme(url.scheme); + + return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) + && (parsedUrlScheme.application + ? parsedUrlScheme.application.value() == inputType() + : hasTarballExtension(url.path)); + } + std::pair fetch(ref store, const Input & input) override { auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first; @@ -233,5 +278,6 @@ struct TarballInputScheme : InputScheme }; static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto rFileInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); } -- cgit v1.2.3 From b916c08feba5173c3455890cff615fd46464409a Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Tue, 24 May 2022 14:20:48 +0200 Subject: libfetchers: drop `getGitDir` and hardcode `.git` As discussed[1] this is most likely not desirable. [1] https://github.com/NixOS/nix/pull/6440#issuecomment-1120876248 --- src/libfetchers/git.cc | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d23a820a4..a71bff76f 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -26,11 +26,6 @@ namespace { // old version of git, which will ignore unrecognized `-c` options. const std::string gitInitialBranch = "__nix_dummy_branch"; -std::string getGitDir() -{ - return getEnv("GIT_DIR").value_or(".git"); -} - bool isCacheFileWithinTtl(const time_t now, const struct stat & st) { return st.st_mtime + settings.tarballTtl > now; @@ -152,7 +147,7 @@ struct WorkdirInfo WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) { const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - auto gitDir = getGitDir(); + std::string gitDir(".git"); auto env = getEnv(); // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong @@ -370,7 +365,7 @@ struct GitInputScheme : InputScheme { auto sourcePath = getSourcePath(input); assert(sourcePath); - auto gitDir = getGitDir(); + auto gitDir = ".git"; runProgram("git", true, { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) }); @@ -396,7 +391,7 @@ struct GitInputScheme : InputScheme std::pair fetch(ref store, const Input & _input) override { Input input(_input); - auto gitDir = getGitDir(); + auto gitDir = ".git"; std::string name = input.getName(); -- cgit v1.2.3 From d8398d33c9a09e1f5599127ae6d477e7e0868b55 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 May 2022 15:29:27 +0200 Subject: Typo --- src/libfetchers/tarball.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 09acb74d3..6c551bd93 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -169,7 +169,7 @@ std::pair downloadTarball( }; } -// An input scheme corresponding to a curable ressource +// An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { virtual const std::string inputType() const = 0; -- cgit v1.2.3 From 027fd45230b74c67e65d06e7073c04b62c60eb4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 27 May 2022 16:15:28 +0200 Subject: Fix a segfault in the git fetcher The git fetcher code used to dereference the (potentially empty) `ref` input attribute. This was magically working, probably because the compiler somehow outsmarted us, but is now blowing up with newer nixpkgs versions. Fix that by not trying to access this field while we don't know for sure that it has been defined. Fix #6554 --- src/libfetchers/git.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a71bff76f..9cbd39247 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -449,11 +449,10 @@ struct GitInputScheme : InputScheme } } - const Attrs unlockedAttrs({ + Attrs unlockedAttrs({ {"type", cacheType}, {"name", name}, {"url", actualUrl}, - {"ref", *input.getRef()}, }); Path repoDir; @@ -466,6 +465,7 @@ struct GitInputScheme : InputScheme head = "master"; } input.attrs.insert_or_assign("ref", *head); + unlockedAttrs.insert_or_assign("ref", *head); } if (!input.getRev()) @@ -482,6 +482,7 @@ struct GitInputScheme : InputScheme head = "master"; } input.attrs.insert_or_assign("ref", *head); + unlockedAttrs.insert_or_assign("ref", *head); } if (auto res = getCache()->lookup(store, unlockedAttrs)) { -- cgit v1.2.3 From da8f8668ca0efaad5a4134c55bf801448cec3cd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Fri, 10 Jun 2022 12:57:13 +0200 Subject: libfetchers/git: add missing `--git-dir` flags --- src/libfetchers/git.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 9cbd39247..35fdf807a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -85,8 +85,9 @@ std::optional readHead(const Path & path) bool storeCachedHead(const std::string& actualUrl, const std::string& headRef) { Path cacheDir = getCachePath(actualUrl); + auto gitDir = "."; try { - runProgram("git", true, { "-C", cacheDir, "symbolic-ref", "--", "HEAD", headRef }); + runProgram("git", true, { "-C", cacheDir, "--git-dir", gitDir, "symbolic-ref", "--", "HEAD", headRef }); } catch (ExecError &e) { if (!WIFEXITED(e.status)) throw; return false; @@ -182,7 +183,7 @@ WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) if (hasHead) { // Using git diff is preferrable over lower-level operations here, // because its conceptually simpler and we only need the exit code anyways. - auto gitDiffOpts = Strings({ "-C", workdir, "diff", "HEAD", "--quiet"}); + auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"}); if (!submodules) { // Changes in submodules should only make the tree dirty // when those submodules will be copied as well. @@ -203,6 +204,7 @@ WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) { const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + auto gitDir = ".git"; if (!fetchSettings.allowDirty) throw Error("Git tree '%s' is dirty", workdir); @@ -210,7 +212,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co if (fetchSettings.warnDirty) warn("Git tree '%s' is dirty", workdir); - auto gitOpts = Strings({ "-C", workdir, "ls-files", "-z" }); + auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" }); if (submodules) gitOpts.emplace_back("--recurse-submodules"); @@ -240,7 +242,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co // modified dirty file? input.attrs.insert_or_assign( "lastModified", - workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return {std::move(storePath), input}; } -- cgit v1.2.3 From 9f6b4639c2060aa6d7f7336222dad4ea350ccdf8 Mon Sep 17 00:00:00 2001 From: Gabriel Fontes Date: Sat, 11 Jun 2022 16:52:20 -0300 Subject: fix sourcehut brach/tag resolving regression nixos/nix#6290 introduced a regex pattern to account for tags when resolving sourcehut refs. nixos/nix#4638 reafactored the code, accidentally treating the pattern as a regular string, causing all non-HEAD ref resolving to break. This fixes the regression and adds more test cases to avoid future breakage. --- src/libfetchers/github.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 0631fb6e8..a491d82a6 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -381,7 +381,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme Headers headers = makeHeadersWithAuthTokens(host); - std::string ref_uri; + std::string refUri; if (ref == "HEAD") { auto file = store->toRealPath( downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath); @@ -393,10 +393,11 @@ struct SourceHutInputScheme : GitArchiveInputScheme if (!remoteLine) { throw BadURL("in '%d', couldn't resolve HEAD ref '%d'", input.to_string(), ref); } - ref_uri = remoteLine->target; + refUri = remoteLine->target; } else { - ref_uri = fmt("refs/(heads|tags)/%s", ref); + refUri = fmt("refs/(heads|tags)/%s", ref); } + std::regex refRegex(refUri); auto file = store->toRealPath( downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); @@ -406,7 +407,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::optional id; while(!id && getline(is, line)) { auto parsedLine = git::parseLsRemoteLine(line); - if (parsedLine && parsedLine->reference == ref_uri) + if (parsedLine && parsedLine->reference && std::regex_match(*parsedLine->reference, refRegex)) id = parsedLine->target; } -- cgit v1.2.3 From 4b6cc3da62ba33c6861a3aa85353d75e8ac82464 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 23 Jun 2022 23:56:26 +0200 Subject: Fetch flake-registry.json from channels.nixos.org Using fastly is slightly faster, provides some resilience due to a high stale TTL, and allows some usage metrics. --- src/libfetchers/fetch-settings.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 04c9feda0..6452143a1 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -70,7 +70,7 @@ struct FetchSettings : public Config Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; - Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", + Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", "Path or URI of the global flake registry."}; Setting useRegistries{this, true, "use-registries", -- cgit v1.2.3 From e8109cf405d672c50b1e5a25c632ddcb1d517233 Mon Sep 17 00:00:00 2001 From: Guillaume Girol Date: Sun, 26 Jun 2022 12:00:00 +0000 Subject: fetchGit: document `shallow` argument --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 35fdf807a..7d01aaa7a 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -574,7 +574,7 @@ struct GitInputScheme : InputScheme bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true"; if (isShallow && !shallow) - throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); + throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl); // FIXME: check whether rev is an ancestor of ref. -- cgit v1.2.3 From c80a74b7d52f0cf5fc5c1c7497e059ebf301f789 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Aug 2022 16:59:02 +0200 Subject: Don't pass --force to 'git add' Fixes #5810. --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7d01aaa7a..c1a21e764 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -370,7 +370,7 @@ struct GitInputScheme : InputScheme auto gitDir = ".git"; runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--force", "--intent-to-add", "--", std::string(file) }); + { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); if (commitMsg) runProgram("git", true, -- cgit v1.2.3 From d9abce4ad4b6888183271c0a4051981dee5fffe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 12 Sep 2022 22:50:18 +0200 Subject: libfetchers: avoid api.github.com ratelimit if no github token is set If we don't have any github token, we won't be able to fetch private repos, but we are also more likely to run into API limits since we don't have a token. To mitigate this only ever use the github api if we actually have a token. --- src/libfetchers/github.cc | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index a491d82a6..2115ce2f5 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -262,17 +262,20 @@ struct GitHubInputScheme : GitArchiveInputScheme DownloadUrl getDownloadUrl(const Input & input) const override { - // FIXME: use regular /archive URLs instead? api.github.com - // might have stricter rate limits. auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - auto url = fmt( - host == "github.com" - ? "https://api.%s/repos/%s/%s/tarball/%s" - : "https://%s/api/v3/repos/%s/%s/tarball/%s", - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + Headers headers = makeHeadersWithAuthTokens(host); + // If we have no auth headers then we default to the public archive + // urls so we do not run into rate limits. + const auto urlFmt = + host != "github.com" + ? "https://%s/api/v3/repos/%s/%s/tarball/%s" + : headers.empty() + ? "https://%s/%s/%s/archive/%s.tar.gz" + : "https://api.%s/repos/%s/%s/tarball/%s"; + + const auto url = fmt(urlFmt, host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; } -- cgit v1.2.3 From a259084c50cdb692a6ba6818bb56ee381e67715b Mon Sep 17 00:00:00 2001 From: Andrew Brooks Date: Fri, 14 Oct 2022 18:04:47 -0500 Subject: Fix #7146 When fetching a non-local git repo by ref (and no rev), don't consider unrelated cached revs for the same repository. --- src/libfetchers/git.cc | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c1a21e764..7b7a1be35 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -485,6 +485,10 @@ struct GitInputScheme : InputScheme } input.attrs.insert_or_assign("ref", *head); unlockedAttrs.insert_or_assign("ref", *head); + } else { + if (!input.getRev()) { + unlockedAttrs.insert_or_assign("ref", input.getRef().value()); + } } if (auto res = getCache()->lookup(store, unlockedAttrs)) { -- cgit v1.2.3 From 703d863a48f549b2626382eda407ffae779f8725 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Dec 2022 12:58:58 +0100 Subject: Trivial changes from the lazy-trees branch --- src/libfetchers/fetchers.cc | 6 ++--- src/libfetchers/fetchers.hh | 13 +++++----- src/libfetchers/git.cc | 60 +++++++++++++++++++++----------------------- src/libfetchers/github.cc | 53 +++++++++++++++++++++++++------------- src/libfetchers/indirect.cc | 10 ++++---- src/libfetchers/mercurial.cc | 10 ++++---- src/libfetchers/path.cc | 8 +++--- src/libfetchers/tarball.cc | 11 ++++---- 8 files changed, 93 insertions(+), 78 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 6957d2da4..c767e72e5 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -266,7 +266,7 @@ std::optional Input::getLastModified() const return {}; } -ParsedURL InputScheme::toURL(const Input & input) +ParsedURL InputScheme::toURL(const Input & input) const { throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs)); } @@ -274,7 +274,7 @@ ParsedURL InputScheme::toURL(const Input & input) Input InputScheme::applyOverrides( const Input & input, std::optional ref, - std::optional rev) + std::optional rev) const { if (ref) throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref); @@ -293,7 +293,7 @@ void InputScheme::markChangedFile(const Input & input, std::string_view file, st assert(false); } -void InputScheme::clone(const Input & input, const Path & destDir) +void InputScheme::clone(const Input & input, const Path & destDir) const { throw Error("do not know how to clone input '%s'", input.to_string()); } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index bc9a76b0b..17da37f47 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -107,26 +107,25 @@ public: * recognized. The Input object contains the information the fetcher * needs to actually perform the "fetch()" when called. */ - struct InputScheme { virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url) = 0; + virtual std::optional inputFromURL(const ParsedURL & url) const = 0; - virtual std::optional inputFromAttrs(const Attrs & attrs) = 0; + virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; - virtual ParsedURL toURL(const Input & input); + virtual ParsedURL toURL(const Input & input) const; - virtual bool hasAllInfo(const Input & input) = 0; + virtual bool hasAllInfo(const Input & input) const = 0; virtual Input applyOverrides( const Input & input, std::optional ref, - std::optional rev); + std::optional rev) const; - virtual void clone(const Input & input, const Path & destDir); + virtual void clone(const Input & input, const Path & destDir) const; virtual std::optional getSourcePath(const Input & input); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7b7a1be35..1f7d7c07d 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -18,6 +18,7 @@ using namespace std::string_literals; namespace nix::fetchers { + namespace { // Explicit initial branch of our bare repo to suppress warnings from new version of git. @@ -26,23 +27,23 @@ namespace { // old version of git, which will ignore unrecognized `-c` options. const std::string gitInitialBranch = "__nix_dummy_branch"; -bool isCacheFileWithinTtl(const time_t now, const struct stat & st) +bool isCacheFileWithinTtl(time_t now, const struct stat & st) { return st.st_mtime + settings.tarballTtl > now; } -bool touchCacheFile(const Path& path, const time_t& touch_time) +bool touchCacheFile(const Path & path, time_t touch_time) { - struct timeval times[2]; - times[0].tv_sec = touch_time; - times[0].tv_usec = 0; - times[1].tv_sec = touch_time; - times[1].tv_usec = 0; + struct timeval times[2]; + times[0].tv_sec = touch_time; + times[0].tv_usec = 0; + times[1].tv_sec = touch_time; + times[1].tv_usec = 0; - return lutimes(path.c_str(), times) == 0; + return lutimes(path.c_str(), times) == 0; } -Path getCachePath(std::string key) +Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, key).to_string(Base32, false); @@ -57,13 +58,12 @@ Path getCachePath(std::string key) // ... std::optional readHead(const Path & path) { - auto [exit_code, output] = runProgram(RunOptions { + auto [status, output] = runProgram(RunOptions { .program = "git", + // FIXME: use 'HEAD' to avoid returning all refs .args = {"ls-remote", "--symref", path}, }); - if (exit_code != 0) { - return std::nullopt; - } + if (status != 0) return std::nullopt; std::string_view line = output; line = line.substr(0, line.find("\n")); @@ -82,12 +82,11 @@ std::optional readHead(const Path & path) } // Persist the HEAD ref from the remote repo in the local cached repo. -bool storeCachedHead(const std::string& actualUrl, const std::string& headRef) +bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) { Path cacheDir = getCachePath(actualUrl); - auto gitDir = "."; try { - runProgram("git", true, { "-C", cacheDir, "--git-dir", gitDir, "symbolic-ref", "--", "HEAD", headRef }); + runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef }); } catch (ExecError &e) { if (!WIFEXITED(e.status)) throw; return false; @@ -96,7 +95,7 @@ bool storeCachedHead(const std::string& actualUrl, const std::string& headRef) return true; } -std::optional readHeadCached(const std::string& actualUrl) +std::optional readHeadCached(const std::string & actualUrl) { // Create a cache path to store the branch of the HEAD ref. Append something // in front of the URL to prevent collision with the repository itself. @@ -110,16 +109,15 @@ std::optional readHeadCached(const std::string& actualUrl) cachedRef = readHead(cacheDir); if (cachedRef != std::nullopt && *cachedRef != gitInitialBranch && - isCacheFileWithinTtl(now, st)) { + isCacheFileWithinTtl(now, st)) + { debug("using cached HEAD ref '%s' for repo '%s'", *cachedRef, actualUrl); return cachedRef; } } auto ref = readHead(actualUrl); - if (ref) { - return ref; - } + if (ref) return ref; if (cachedRef) { // If the cached git ref is expired in fetch() below, and the 'git fetch' @@ -250,7 +248,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "git" && url.scheme != "git+http" && @@ -265,7 +263,7 @@ struct GitInputScheme : InputScheme Attrs attrs; attrs.emplace("type", "git"); - for (auto &[name, value] : url.query) { + for (auto & [name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); else if (name == "shallow" || name == "submodules") @@ -279,7 +277,7 @@ struct GitInputScheme : InputScheme return inputFromAttrs(attrs); } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "git") return {}; @@ -302,7 +300,7 @@ struct GitInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme != "git") url.scheme = "git+" + url.scheme; @@ -313,7 +311,7 @@ struct GitInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { bool maybeDirty = !input.getRef(); bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); @@ -325,7 +323,7 @@ struct GitInputScheme : InputScheme Input applyOverrides( const Input & input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto res(input); if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); @@ -335,7 +333,7 @@ struct GitInputScheme : InputScheme return res; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto [isLocal, actualUrl] = getActualUrl(input); @@ -603,9 +601,9 @@ struct GitInputScheme : InputScheme { throw Error( "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " - "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " - ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD - "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", input.getRev()->gitRev(), *input.getRef(), actualUrl diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 2115ce2f5..1ed09d30d 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -26,11 +26,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { - virtual std::string type() = 0; + virtual std::string type() const = 0; virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != type()) return {}; @@ -100,7 +100,7 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != type()) return {}; @@ -116,7 +116,7 @@ struct GitArchiveInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto owner = getStrAttr(input.attrs, "owner"); auto repo = getStrAttr(input.attrs, "repo"); @@ -132,7 +132,7 @@ struct GitArchiveInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); } @@ -140,7 +140,7 @@ struct GitArchiveInputScheme : InputScheme Input applyOverrides( const Input & _input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto input(_input); if (rev && ref) @@ -227,7 +227,7 @@ struct GitArchiveInputScheme : InputScheme struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() override { return "github"; } + std::string type() const override { return "github"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -240,14 +240,29 @@ struct GitHubInputScheme : GitArchiveInputScheme return std::pair("Authorization", fmt("token %s", token)); } + std::string getHost(const Input & input) const + { + return maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + } + + std::string getOwner(const Input & input) const + { + return getStrAttr(input.attrs, "owner"); + } + + std::string getRepo(const Input & input) const + { + return getStrAttr(input.attrs, "repo"); + } + Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto host = getHost(input); auto url = fmt( host == "github.com" ? "https://api.%s/repos/%s/%s/commits/%s" : "https://%s/api/v3/repos/%s/%s/commits/%s", - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + host, getOwner(input), getRepo(input), *input.getRef()); Headers headers = makeHeadersWithAuthTokens(host); @@ -262,8 +277,10 @@ struct GitHubInputScheme : GitArchiveInputScheme DownloadUrl getDownloadUrl(const Input & input) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto host = getHost(input); + Headers headers = makeHeadersWithAuthTokens(host); + // If we have no auth headers then we default to the public archive // urls so we do not run into rate limits. const auto urlFmt = @@ -273,17 +290,17 @@ struct GitHubInputScheme : GitArchiveInputScheme ? "https://%s/%s/%s/archive/%s.tar.gz" : "https://api.%s/repos/%s/%s/tarball/%s"; - const auto url = fmt(urlFmt, host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), input.getRev()->to_string(Base16, false)); return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { - auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); + auto host = getHost(input); Input::fromURL(fmt("git+https://%s/%s/%s.git", - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); } @@ -291,7 +308,7 @@ struct GitHubInputScheme : GitArchiveInputScheme struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() override { return "gitlab"; } + std::string type() const override { return "gitlab"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -346,7 +363,7 @@ struct GitLabInputScheme : GitArchiveInputScheme return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere @@ -359,7 +376,7 @@ struct GitLabInputScheme : GitArchiveInputScheme struct SourceHutInputScheme : GitArchiveInputScheme { - std::string type() override { return "sourcehut"; } + std::string type() const override { return "sourcehut"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -433,7 +450,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme return DownloadUrl { url, headers }; } - void clone(const Input & input, const Path & destDir) override + void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); Input::fromURL(fmt("git+https://%s/%s/%s", diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 9288fc6cf..b99504a16 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -7,7 +7,7 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "flake") return {}; @@ -50,7 +50,7 @@ struct IndirectInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; @@ -68,7 +68,7 @@ struct IndirectInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { ParsedURL url; url.scheme = "flake"; @@ -78,7 +78,7 @@ struct IndirectInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return false; } @@ -86,7 +86,7 @@ struct IndirectInputScheme : InputScheme Input applyOverrides( const Input & _input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto input(_input); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 5c5671681..86e8f81f4 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -43,7 +43,7 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && @@ -69,7 +69,7 @@ struct MercurialInputScheme : InputScheme return inputFromAttrs(attrs); } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "hg") return {}; @@ -89,7 +89,7 @@ struct MercurialInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); url.scheme = "hg+" + url.scheme; @@ -98,7 +98,7 @@ struct MercurialInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { // FIXME: ugly, need to distinguish between dirty and clean // default trees. @@ -108,7 +108,7 @@ struct MercurialInputScheme : InputScheme Input applyOverrides( const Input & input, std::optional ref, - std::optional rev) override + std::optional rev) const override { auto res(input); if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index f0ef97da5..61541e69d 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -6,7 +6,7 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (url.scheme != "path") return {}; @@ -32,7 +32,7 @@ struct PathInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { if (maybeGetStrAttr(attrs, "type") != "path") return {}; @@ -54,7 +54,7 @@ struct PathInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto query = attrsToQuery(input.attrs); query.erase("path"); @@ -66,7 +66,7 @@ struct PathInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return true; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 6c551bd93..e9686262a 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -185,7 +185,7 @@ struct CurlInputScheme : InputScheme virtual bool isValidURL(const ParsedURL & url) const = 0; - std::optional inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) const override { if (!isValidURL(url)) return std::nullopt; @@ -203,7 +203,7 @@ struct CurlInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) const override { auto type = maybeGetStrAttr(attrs, "type"); if (type != inputType()) return {}; @@ -220,16 +220,17 @@ struct CurlInputScheme : InputScheme return input; } - ParsedURL toURL(const Input & input) override + ParsedURL toURL(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); - // NAR hashes are preferred over file hashes since tar/zip files // don't have a canonical representation. + // NAR hashes are preferred over file hashes since tar/zip + // files don't have a canonical representation. if (auto narHash = input.getNarHash()) url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); return url; } - bool hasAllInfo(const Input & input) override + bool hasAllInfo(const Input & input) const override { return true; } -- cgit v1.2.3 From a456630a5a93db170bbc5570ae0e1a48a16cbe09 Mon Sep 17 00:00:00 2001 From: Benoit de Chezelles Date: Thu, 28 Oct 2021 00:56:36 +0200 Subject: Allow to disable global flake-registry with "" --- src/libfetchers/fetch-settings.hh | 7 ++++++- src/libfetchers/registry.cc | 3 +++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 6452143a1..f33cbdcfc 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -71,7 +71,12 @@ struct FetchSettings : public Config "Whether to warn about dirty Git/Mercurial trees."}; Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", - "Path or URI of the global flake registry."}; + R"( + Path or URI of the global flake registry. + + When empty, disables the global flake registry. + )"}; + Setting useRegistries{this, true, "use-registries", "Whether to use flake registries to resolve flake references."}; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index acd1ff866..43c03beec 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -153,6 +153,9 @@ static std::shared_ptr getGlobalRegistry(ref store) { static auto reg = [&]() { auto path = fetchSettings.flakeRegistry.get(); + if (path == "") { + return std::make_shared(Registry::Global); // empty registry + } if (!hasPrefix(path, "/")) { auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; -- cgit v1.2.3