Update to use access-tokens configuration for github/gitlab access.

This change provides support for using access tokens with other
instances of GitHub and GitLab beyond just github.com and
gitlab.com (especially company-specific or foundation-specific
instances).

This change also provides the ability to specify the type of access
token being used, where different types may have different handling,
based on the forge type.

1 files changed, 59 insertions, 41 deletions

diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index eb758bc5f..0e0655367 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -5,6 +5,7 @@
 #include "store-api.hh"
 #include "types.hh"
 
+#include <optional>
 #include <nlohmann/json.hpp>
 
 namespace nix::fetchers {
@@ -12,13 +13,10 @@ namespace nix::fetchers {
 struct DownloadUrl
 {
     std::string url;
-    std::optional<std::pair<std::string, std::string>> access_token_header;
+    Headers headers;
 
-    DownloadUrl(const std::string & url)
-        : url(url) { }
-
-    DownloadUrl(const std::string & url, const std::pair<std::string, std::string> & access_token_header)
-        : url(url), access_token_header(access_token_header) { }
+    DownloadUrl(const std::string & url, const Headers & headers)
+        : url(url), headers(headers) { }
 };
 
 // A github or gitlab url
@@ -29,7 +27,7 @@ struct GitArchiveInputScheme : InputScheme
 {
     virtual std::string type() = 0;
 
-    virtual std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const = 0;
+    virtual std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const = 0;
 
     std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
@@ -144,6 +142,27 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
+    std::optional<std::string> getAccessToken(const std::string &host) const {
+        auto tokens = settings.accessTokens.get();
+        auto pat = tokens.find(host);
+        if (pat == tokens.end())
+            return std::nullopt;
+        return pat->second;
+    }
+
+    Headers makeHeadersWithAuthTokens(const std::string & host) const {
+        Headers headers;
+        auto accessToken = getAccessToken(host);
+        if (accessToken) {
+            auto hdr = accessHeaderFromToken(*accessToken);
+            if (hdr)
+                headers.push_back(*hdr);
+            else
+                warn("Unrecognized access token for host '%s'", host);
+        }
+        return headers;
+    }
+
     virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
 
     virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
@@ -175,12 +194,7 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        Headers headers;
-        if (url.access_token_header) {
-            headers.push_back(*url.access_token_header);
-        }
-
-        auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true);
+        auto [tree, lastModified] = downloadTarball(store, url.url, url.headers, "source", true);
 
         input.attrs.insert_or_assign("lastModified", lastModified);
 
@@ -202,7 +216,13 @@ struct GitHubInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "github"; }
 
-    std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
+    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
+        // Github supports PAT/OAuth2 tokens and HTTP Basic
+        // Authentication.  The former simply specifies the token, the
+        // latter can use the token as the password.  Only the first
+        // is used here. See
+        // https://developer.github.com/v3/#authentication and
+        // https://docs.github.com/en/developers/apps/authorizing-oath-apps
         return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
     }
 
@@ -212,10 +232,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
         auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers;
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            headers.push_back(accessHeaderFromToken(accessToken));
+        Headers headers = makeHeadersWithAuthTokens(host);
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -235,13 +252,8 @@ struct GitHubInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "") {
-            auto auth_header = accessHeaderFromToken(accessToken);
-            return DownloadUrl(url, auth_header);
-        } else {
-            return DownloadUrl(url);
-        }
+        Headers headers = makeHeadersWithAuthTokens(host);
+        return DownloadUrl(url, headers);
     }
 
     void clone(const Input & input, const Path & destDir) override
@@ -258,20 +270,32 @@ struct GitLabInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "gitlab"; }
 
-    std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
-        return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
+    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
+        // Gitlab supports 4 kinds of authorization, two of which are
+        // relevant here: OAuth2 and PAT (Private Access Token).  The
+        // user can indicate which token is used by specifying the
+        // token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
+        // If the <TYPE> is unrecognized, this will fall back to
+        // treating this simply has <HDRNAME>:<HDRVAL>.  See
+        // https://docs.gitlab.com/12.10/ee/api/README.html#authentication
+        auto fldsplit = token.find_first_of(':');
+        // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
+        if ("OAuth2" == token.substr(0, fldsplit))
+            return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
+        if ("PAT" == token.substr(0, fldsplit))
+            return std::make_pair("Private-token", token.substr(fldsplit+1));
+        warn("Unrecognized GitLab token type %s",  token.substr(0, fldsplit));
+        return std::nullopt;
     }
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+        // See rate limiting note below
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
-            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers;
-        std::string accessToken = settings.gitlabAccessToken.get();
-        if (accessToken != "")
-            headers.push_back(accessHeaderFromToken(accessToken));
+        Headers headers = makeHeadersWithAuthTokens(host);
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -294,14 +318,8 @@ struct GitLabInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        std::string accessToken = settings.gitlabAccessToken.get();
-        if (accessToken != "") {
-            auto auth_header = accessHeaderFromToken(accessToken);
-            return DownloadUrl(url, auth_header);
-        } else {
-            return DownloadUrl(url);
-        }
-
+        Headers headers = makeHeadersWithAuthTokens(host);
+        return DownloadUrl(url, headers);
     }
 
     void clone(const Input & input, const Path & destDir) override