From e0a0ae0467fa8cdcc542f593b9d94283f04508ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 14:03:28 +0200 Subject: Move fetchers from libstore to libfetchers --- src/libfetchers/git.cc | 458 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 458 insertions(+) create mode 100644 src/libfetchers/git.cc (limited to 'src/libfetchers/git.cc') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc new file mode 100644 index 000000000..e578ed731 --- /dev/null +++ b/src/libfetchers/git.cc @@ -0,0 +1,458 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +static std::string readHead(const Path & path) +{ + return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); +} + +struct GitInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + bool shallow = false; + + GitInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "git"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + if (shallow) url2.query.insert_or_assign("shallow", "1"); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (shallow) + attrs.emplace("shallow", true); + return attrs; + } + + void clone(const Path & destDir) const override + { + auto [isLocal, actualUrl] = getActualUrl(); + + Strings args = {"clone"}; + + args.push_back(actualUrl); + + if (ref) { + args.push_back("--branch"); + args.push_back(*ref); + } + + if (rev) throw Error("cloning a specific revision is not implemented"); + + args.push_back(destDir); + + runProgram("git", true, args); + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + if (!res->ref && res->rev) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); + + return res; + } + + std::optional getSourcePath() const override + { + if (url.scheme == "file" && !ref && !rev) + return url.path; + return {}; + } + + void markChangedFile(std::string_view file, std::optional commitMsg) const override + { + auto sourcePath = getSourcePath(); + assert(sourcePath); + + runProgram("git", true, + { "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) }); + + if (commitMsg) + runProgram("git", true, + { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); + } + + std::pair getActualUrl() const + { + // Don't clone file:// URIs (but otherwise treat them the + // same as remote URIs, i.e. don't use the working tree or + // HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + bool isLocal = url.scheme == "file" && !forceHttp; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + assert(!rev || rev->type == htSHA1); + + auto cacheType = shallow ? "git-shallow" : "git"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", cacheType}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree { + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), + .lastModified = getIntAttr(infoAttrs, "lastModified"), + }, + }, + input + }; + }; + + if (rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input->ref && !input->rev && isLocal) { + bool clean = false; + + /* Check whether this repo has any commits. There are + probably better ways to do this. */ + auto gitDir = actualUrl + "/.git"; + auto commonGitDir = chomp(runProgram( + "git", + true, + { "-C", actualUrl, "rev-parse", "--git-common-dir" } + )); + if (commonGitDir != ".git") + gitDir = commonGitDir; + + bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty(); + + try { + if (haveCommits) { + runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); + clean = true; + } + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + } + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked files. */ + + if (!settings.allowDirty) + throw Error("Git tree '%s' is dirty", actualUrl); + + if (settings.warnDirty) + warn("Git tree '%s' is dirty", actualUrl); + + auto files = tokenizeString>( + runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + + auto tree = Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, + } + }; + + return {std::move(tree), input}; + } + } + + if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + + Attrs mutableAttrs({ + {"type", cacheType}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + Path repoDir; + + if (isLocal) { + + if (!input->rev) + input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + + repoDir = actualUrl; + + } else { + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); + repoDir = cacheDir; + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } + + Path localRefFile = + input->ref->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input->ref + : cacheDir + "/refs/heads/" + *input->ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (input->rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } + + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + try { + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + } + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + if (!input->rev) + input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + } + + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; + + if (isShallow && !shallow) + throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); + + // FIXME: check whether rev is an ancestor of ref. + + printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + + /* Now that we know the ref, check again whether we have it in + the store. */ + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + unpackTarfile(*source, tmpDir); + + auto storePath = store->addToStore(name, tmpDir); + + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"lastModified", lastModified}, + }); + + if (!shallow) + infoAttrs.insert_or_assign("revCount", + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct GitInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return nullptr; + + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "git"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "git") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow") + throw Error("unsupported Git input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + + input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} -- cgit v1.2.3 From a6ff66b658b61aef80d936f0183447fe4cb46000 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 19:04:33 +0200 Subject: Respect the narHash attribute in more input types call-flake.nix now passes node.info.narHash to fetchTree. This ensures that dirty Git trees work even in pure mode. --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libfetchers/git.cc') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index e578ed731..dbf57aa33 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -39,7 +39,7 @@ struct GitInput : Input bool isImmutable() const override { - return (bool) rev; + return (bool) rev || narHash; } std::optional getRef() const override { return ref; } -- cgit v1.2.3 From 849d3968dbe8c18651e1a7fbf071fc3304257517 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 May 2020 10:41:21 +0200 Subject: Update src/libfetchers/git.cc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörg Thalheim --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libfetchers/git.cc') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 210e29193..c9f9a4b23 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -129,7 +129,7 @@ struct GitInput : Input assert(sourcePath); runProgram("git", true, - { "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) }); + { "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) }); if (commitMsg) runProgram("git", true, -- cgit v1.2.3 From 950b46821f644eb3f92725460584a3102f356179 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 30 May 2020 00:44:11 +0200 Subject: Remove TreeInfo The attributes previously stored in TreeInfo (narHash, revCount, lastModified) are now stored in Input. This makes it less arbitrary what attributes are stored where. As a result, the lock file format has changed. An entry like "info": { "lastModified": 1585405475, "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE=" }, "locked": { "owner": "NixOS", "repo": "nixpkgs", "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be", "type": "github" }, is now stored as "locked": { "owner": "NixOS", "repo": "nixpkgs", "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be", "type": "github", "lastModified": 1585405475, "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE=" }, The 'Input' class is now a dumb set of attributes. All the fetcher implementations subclass InputScheme, not Input. This simplifies the API. Also, fix substitution of flake inputs. This was broken since lazy flake fetching started using fetchTree internally. --- src/libfetchers/git.cc | 292 +++++++++++++++++++++---------------------------- 1 file changed, 126 insertions(+), 166 deletions(-) (limited to 'src/libfetchers/git.cc') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c9f9a4b23..4fcf3f542 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -22,110 +22,121 @@ static bool isNotDotGitDirectory(const Path & path) return not std::regex_match(path, gitDirRegex); } -struct GitInput : Input +struct GitInputScheme : InputScheme { - ParsedURL url; - std::optional ref; - std::optional rev; - bool shallow = false; - bool submodules = false; + std::optional inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return {}; - GitInput(const ParsedURL & url) : url(url) - { } + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); - std::string type() const override { return "git"; } + Attrs attrs; + attrs.emplace("type", "git"); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != "git") return {}; - std::optional getRef() const override { return ref; } + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash") + throw Error("unsupported Git input attribute '%s'", name); + + parseURL(getStrAttr(attrs, "url")); + maybeGetBoolAttr(attrs, "shallow"); + maybeGetBoolAttr(attrs, "submodules"); + + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + } - std::optional getRev() const override { return rev; } + Input input; + input.attrs = attrs; + return input; + } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - ParsedURL url2(url); - if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); - if (shallow) url2.query.insert_or_assign("shallow", "1"); - return url2; + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme != "git") url.scheme = "git+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); + if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + url.query.insert_or_assign("shallow", "1"); + return url; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (shallow) - attrs.emplace("shallow", true); - if (submodules) - attrs.emplace("submodules", true); - return attrs; + bool maybeDirty = !input.getRef(); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + return + maybeGetIntAttr(input.attrs, "lastModified") + && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); } - void clone(const Path & destDir) const override + Input applyOverrides( + const Input & input, + std::optional ref, + std::optional rev) override + { + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); + if (!res.getRef() && res.getRev()) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string()); + return res; + } + + void clone(const Input & input, const Path & destDir) override { - auto [isLocal, actualUrl] = getActualUrl(); + auto [isLocal, actualUrl] = getActualUrl(input); Strings args = {"clone"}; args.push_back(actualUrl); - if (ref) { + if (auto ref = input.getRef()) { args.push_back("--branch"); args.push_back(*ref); } - if (rev) throw Error("cloning a specific revision is not implemented"); + if (input.getRev()) throw Error("cloning a specific revision is not implemented"); args.push_back(destDir); runProgram("git", true, args); } - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - if (!res->ref && res->rev) - throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); - - return res; - } - - std::optional getSourcePath() const override + std::optional getSourcePath(const Input & input) override { - if (url.scheme == "file" && !ref && !rev) + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) return url.path; return {}; } - void markChangedFile(std::string_view file, std::optional commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override { - auto sourcePath = getSourcePath(); + auto sourcePath = getSourcePath(input); assert(sourcePath); runProgram("git", true, @@ -136,23 +147,25 @@ struct GitInput : Input { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); } - std::pair getActualUrl() const + std::pair getActualUrl(const Input & input) const { // Don't clone file:// URIs (but otherwise treat them the // same as remote URIs, i.e. don't use the working tree or // HEAD). static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file" && !forceHttp; return {isLocal, isLocal ? url.path : url.base}; } - std::pair> fetchTreeInternal(nix::ref store) const override + std::pair fetch(ref store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared(*this); + Input input(_input); - assert(!rev || rev->type == htSHA1); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -163,39 +176,38 @@ struct GitInput : Input return Attrs({ {"type", cacheType}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair> + -> std::pair { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + if (!shallow) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return { Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), - .lastModified = getIntAttr(infoAttrs, "lastModified"), - }, }, input }; }; - if (rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. - if (!input->ref && !input->rev && isLocal) { + if (!input.getRef() && !input.getRev() && isLocal) { bool clean = false; /* Check whether this repo has any commits. There are @@ -254,35 +266,37 @@ struct GitInput : Input auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); - auto tree = Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, - } + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + input.attrs.insert_or_assign( + "lastModified", + haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0); + + return { + Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input }; - - return {std::move(tree), input}; } } - if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master"); Attrs mutableAttrs({ {"type", cacheType}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); Path repoDir; if (isLocal) { - if (!input->rev) - input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", + Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; @@ -290,8 +304,8 @@ struct GitInput : Input if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -305,18 +319,18 @@ struct GitInput : Input } Path localRefFile = - input->ref->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input->ref - : cacheDir + "/refs/heads/" + *input->ref; + input.getRef()->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input.getRef() + : cacheDir + "/refs/heads/" + *input.getRef(); bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input->rev) { + if (input.getRev()) { try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -339,7 +353,7 @@ struct GitInput : Input // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. try { - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input.getRef(), *input.getRef()) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); @@ -354,8 +368,8 @@ struct GitInput : Input utimes(localRefFile.c_str(), times); } - if (!input->rev) - input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev()); } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; @@ -365,7 +379,7 @@ struct GitInput : Input // FIXME: check whether rev is an ancestor of ref. - printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); /* Now that we know the ref, check again whether we have it in the store. */ @@ -387,7 +401,7 @@ struct GitInput : Input runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); @@ -396,7 +410,7 @@ struct GitInput : Input // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() }); gitOptions.standardOut = &sink; runProgram2(gitOptions); }); @@ -406,18 +420,18 @@ struct GitInput : Input auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() })); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"lastModified", lastModified}, }); if (!shallow) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -436,60 +450,6 @@ struct GitInput : Input } }; -struct GitInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "git" && - url.scheme != "git+http" && - url.scheme != "git+https" && - url.scheme != "git+ssh" && - url.scheme != "git+file") return nullptr; - - auto url2(url); - if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "git"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules") - throw Error("unsupported Git input attribute '%s'", name); - - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Git branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - - input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); - - input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false); - - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); } -- cgit v1.2.3 From c254254a8088ca303e97b9430484cf25cf8f7806 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Mon, 1 Jun 2020 08:59:26 -0600 Subject: use Tree ctor --- src/libfetchers/git.cc | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'src/libfetchers/git.cc') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 4fcf3f542..af4efc9b1 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -189,10 +189,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return { - Tree { - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - }, + Tree(store->toRealPath(storePath), std::move(storePath)), input }; }; @@ -273,10 +270,8 @@ struct GitInputScheme : InputScheme haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0); return { - Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - }, input + Tree(store->printStorePath(storePath), std::move(storePath)), + input }; } } -- cgit v1.2.3 From 7d554f295ca0a8947042f367adb4f198730696be Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Jul 2020 14:57:59 +0200 Subject: Support building flakes from a shallow Git repo Fixes #3756. --- src/libfetchers/git.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libfetchers/git.cc') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 0b6155e71..5d38e0c2b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -42,6 +42,8 @@ struct GitInputScheme : InputScheme for (auto &[name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); + else if (name == "shallow") + attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); } -- cgit v1.2.3