Merge remote-tracking branch 'origin/master' into best-effort-supplementary-groups

7 files changed, 103 insertions, 40 deletions

diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index 1a14bb023..9f885a793 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -2,6 +2,7 @@
 ///@file
 
 #include "types.hh"
+#include "hash.hh"
 
 #include <variant>
 
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 91db3a9eb..2860c1ceb 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -159,6 +159,12 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
                 input.to_string(), *prevLastModified);
     }
 
+    if (auto prevRev = getRev()) {
+        if (input.getRev() != prevRev)
+            throw Error("'rev' attribute mismatch in input '%s', expected %s",
+                input.to_string(), prevRev->gitRev());
+    }
+
     if (auto prevRevCount = getRevCount()) {
         if (input.getRevCount() != prevRevCount)
             throw Error("'revCount' attribute mismatch in input '%s', expected %d",
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 498ad7e4d..d0738f619 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -158,6 +158,7 @@ struct DownloadFileResult
     StorePath storePath;
     std::string etag;
     std::string effectiveUrl;
+    std::optional<std::string> immutableUrl;
 };
 
 DownloadFileResult downloadFile(
@@ -167,7 +168,14 @@ DownloadFileResult downloadFile(
     bool locked,
     const Headers & headers = {});
 
-std::pair<Tree, time_t> downloadTarball(
+struct DownloadTarballResult
+{
+    Tree tree;
+    time_t lastModified;
+    std::optional<std::string> immutableUrl;
+};
+
+DownloadTarballResult downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 1da8c9609..be5842d53 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -62,6 +62,7 @@ std::optional<std::string> readHead(const Path & path)
         .program = "git",
         // FIXME: use 'HEAD' to avoid returning all refs
         .args = {"ls-remote", "--symref", path},
+        .isInteractive = true,
     });
     if (status != 0) return std::nullopt;
 
@@ -242,6 +243,13 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co
         "lastModified",
         workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
 
+    if (workdirInfo.hasHead) {
+        input.attrs.insert_or_assign("dirtyRev", chomp(
+            runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty");
+        input.attrs.insert_or_assign("dirtyShortRev", chomp(
+            runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty");
+    }
+
     return {std::move(storePath), input};
 }
 }  // end namespace
@@ -282,7 +290,7 @@ struct GitInputScheme : InputScheme
         if (maybeGetStrAttr(attrs, "type") != "git") return {};
 
         for (auto & [name, value] : attrs)
-            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name")
+            if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev")
                 throw Error("unsupported Git input attribute '%s'", name);
 
         parseURL(getStrAttr(attrs, "url"));
@@ -350,7 +358,7 @@ struct GitInputScheme : InputScheme
 
         args.push_back(destDir);
 
-        runProgram("git", true, args);
+        runProgram("git", true, args, {}, true);
     }
 
     std::optional<Path> getSourcePath(const Input & input) override
@@ -555,7 +563,7 @@ struct GitInputScheme : InputScheme
                             : ref == "HEAD"
                                 ? *ref
                                 : "refs/heads/" + *ref;
-                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
+                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
@@ -622,7 +630,7 @@ struct GitInputScheme : InputScheme
                 // everything to ensure we get the rev.
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
                 runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
-                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" });
+                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
             }
 
             runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
@@ -649,7 +657,7 @@ struct GitInputScheme : InputScheme
 
             {
                 Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
-                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
+                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
             }
 
             filter = isNotDotGitDirectory;
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 6c1d573ce..80598e7f8 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -207,21 +207,21 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers);
+        auto result = downloadTarball(store, url.url, input.getName(), true, url.headers);
 
-        input.attrs.insert_or_assign("lastModified", uint64_t(lastModified));
+        input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
 
         getCache()->add(
             store,
             lockedAttrs,
             {
                 {"rev", rev->gitRev()},
-                {"lastModified", uint64_t(lastModified)}
+                {"lastModified", uint64_t(result.lastModified)}
             },
-            tree.storePath,
+            result.tree.storePath,
             true);
 
-        return {std::move(tree.storePath), input};
+        return {result.tree.storePath, input};
     }
 };
 
diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc
index f9909c218..f37a8058b 100644
--- a/src/libfetchers/input-accessor.cc
+++ b/src/libfetchers/input-accessor.cc
@@ -75,22 +75,28 @@ SourcePath SourcePath::resolveSymlinks() const
 
     int linksAllowed = 1024;
 
-    for (auto & component : path) {
-        res.path.push(component);
-        while (true) {
-            if (auto st = res.maybeLstat()) {
+    std::list<std::string> todo;
+    for (auto & c : path)
+        todo.push_back(std::string(c));
+
+    while (!todo.empty()) {
+        auto c = *todo.begin();
+        todo.pop_front();
+        if (c == "" || c == ".")
+            ;
+        else if (c == "..")
+            res.path.pop();
+        else {
+            res.path.push(c);
+            if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) {
                 if (!linksAllowed--)
                     throw Error("infinite symlink recursion in path '%s'", path);
-                if (st->type != InputAccessor::tSymlink) break;
                 auto target = res.readLink();
+                res.path.pop();
                 if (hasPrefix(target, "/"))
-                    res = CanonPath(target);
-                else {
-                    res.path.pop();
-                    res.path.extend(CanonPath(target));
-                }
-            } else
-                break;
+                    res.path = CanonPath::root;
+                todo.splice(todo.begin(), tokenizeString<std::list<std::string>>(target, "/"));
+            }
         }
     }
 
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 96fe5faca..e42aca6db 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -32,7 +32,8 @@ DownloadFileResult downloadFile(
         return {
             .storePath = std::move(cached->storePath),
             .etag = getStrAttr(cached->infoAttrs, "etag"),
-            .effectiveUrl = getStrAttr(cached->infoAttrs, "url")
+            .effectiveUrl = getStrAttr(cached->infoAttrs, "url"),
+            .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
         };
     };
 
@@ -55,12 +56,14 @@ DownloadFileResult downloadFile(
     }
 
     // FIXME: write to temporary file.
-
     Attrs infoAttrs({
         {"etag", res.etag},
         {"url", res.effectiveUri},
     });
 
+    if (res.immutableUrl)
+        infoAttrs.emplace("immutableUrl", *res.immutableUrl);
+
     std::optional<StorePath> storePath;
 
     if (res.cached) {
@@ -111,10 +114,11 @@ DownloadFileResult downloadFile(
         .storePath = std::move(*storePath),
         .etag = res.etag,
         .effectiveUrl = res.effectiveUri,
+        .immutableUrl = res.immutableUrl,
     };
 }
 
-std::pair<Tree, time_t> downloadTarball(
+DownloadTarballResult downloadTarball(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
@@ -131,8 +135,9 @@ std::pair<Tree, time_t> downloadTarball(
 
     if (cached && !cached->expired)
         return {
-            Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) },
-            getIntAttr(cached->infoAttrs, "lastModified")
+            .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) },
+            .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"),
+            .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
         };
 
     auto res = downloadFile(store, url, name, locked, headers);
@@ -160,6 +165,9 @@ std::pair<Tree, time_t> downloadTarball(
         {"etag", res.etag},
     });
 
+    if (res.immutableUrl)
+        infoAttrs.emplace("immutableUrl", *res.immutableUrl);
+
     getCache()->add(
         store,
         inAttrs,
@@ -168,8 +176,9 @@ std::pair<Tree, time_t> downloadTarball(
         locked);
 
     return {
-        Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) },
-        lastModified,
+        .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) },
+        .lastModified = lastModified,
+        .immutableUrl = res.immutableUrl,
     };
 }
 
@@ -189,21 +198,33 @@ struct CurlInputScheme : InputScheme
 
     virtual bool isValidURL(const ParsedURL & url) const = 0;
 
-    std::optional<Input> inputFromURL(const ParsedURL & url) const override
+    std::optional<Input> inputFromURL(const ParsedURL & _url) const override
     {
-        if (!isValidURL(url))
+        if (!isValidURL(_url))
             return std::nullopt;
 
         Input input;
 
-        auto urlWithoutApplicationScheme = url;
-        urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport;
+        auto url = _url;
+
+        url.scheme = parseUrlScheme(url.scheme).transport;
 
-        input.attrs.insert_or_assign("type", inputType());
-        input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string());
         auto narHash = url.query.find("narHash");
         if (narHash != url.query.end())
             input.attrs.insert_or_assign("narHash", narHash->second);
+
+        if (auto i = get(url.query, "rev"))
+            input.attrs.insert_or_assign("rev", *i);
+
+        if (auto i = get(url.query, "revCount"))
+            if (auto n = string2Int<uint64_t>(*i))
+                input.attrs.insert_or_assign("revCount", *n);
+
+        url.query.erase("rev");
+        url.query.erase("revCount");
+
+        input.attrs.insert_or_assign("type", inputType());
+        input.attrs.insert_or_assign("url", url.to_string());
         return input;
     }
 
@@ -212,7 +233,8 @@ struct CurlInputScheme : InputScheme
         auto type = maybeGetStrAttr(attrs, "type");
         if (type != inputType()) return {};
 
-        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack"};
+        // FIXME: some of these only apply to TarballInputScheme.
+        std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount"};
         for (auto & [name, value] : attrs)
             if (!allowedNames.count(name))
                 throw Error("unsupported %s input attribute '%s'", *type, name);
@@ -275,10 +297,22 @@ struct TarballInputScheme : CurlInputScheme
                     : hasTarballExtension(url.path));
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
+    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
     {
-        auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first;
-        return {std::move(tree.storePath), input};
+        Input input(_input);
+        auto url = getStrAttr(input.attrs, "url");
+        auto result = downloadTarball(store, url, input.getName(), false);
+
+        if (result.immutableUrl) {
+            auto immutableInput = Input::fromURL(*result.immutableUrl);
+            // FIXME: would be nice to support arbitrary flakerefs
+            // here, e.g. git flakes.
+            if (immutableInput.getType() != "tarball")
+                throw Error("tarball 'Link' headers that redirect to non-tarball URLs are not supported");
+            input = immutableInput;
+        }
+
+        return {result.tree.storePath, std::move(input)};
     }
 };