aboutsummaryrefslogtreecommitdiff
path: root/src/libfetchers
diff options
context:
space:
mode:
Diffstat (limited to 'src/libfetchers')
-rw-r--r--src/libfetchers/fetchers.cc8
-rw-r--r--src/libfetchers/fetchers.hh10
-rw-r--r--src/libfetchers/github.cc124
-rw-r--r--src/libfetchers/tarball.cc10
4 files changed, 113 insertions, 39 deletions
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index eaa635595..49851f7bc 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -69,6 +69,14 @@ ParsedURL Input::toURL() const
return scheme->toURL(*this);
}
+std::string Input::toURLString(const std::map<std::string, std::string> & extraQuery) const
+{
+ auto url = toURL();
+ for (auto & attr : extraQuery)
+ url.query.insert(attr);
+ return url.to_string();
+}
+
std::string Input::to_string() const
{
return toURL().to_string();
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 89b1e6e7d..cc31a31b9 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -39,6 +39,8 @@ public:
ParsedURL toURL() const;
+ std::string toURLString(const std::map<std::string, std::string> & extraQuery = {}) const;
+
std::string to_string() const;
Attrs toAttrs() const;
@@ -73,7 +75,7 @@ public:
StorePath computeStorePath(Store & store) const;
- // Convience functions for common attributes.
+ // Convenience functions for common attributes.
std::string getType() const;
std::optional<Hash> getNarHash() const;
std::optional<std::string> getRef() const;
@@ -119,12 +121,14 @@ DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
- bool immutable);
+ bool immutable,
+ const Headers & headers = {});
std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
- bool immutable);
+ bool immutable,
+ const Headers & headers = {});
}
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 1737658a7..8610fe447 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -3,12 +3,20 @@
#include "fetchers.hh"
#include "globals.hh"
#include "store-api.hh"
+#include "types.hh"
#include "url-parts.hh"
+#include <optional>
#include <nlohmann/json.hpp>
namespace nix::fetchers {
+struct DownloadUrl
+{
+ std::string url;
+ Headers headers;
+};
+
// A github or gitlab host
const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
@@ -17,6 +25,8 @@ struct GitArchiveInputScheme : InputScheme
{
virtual std::string type() = 0;
+ virtual std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const = 0;
+
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != type()) return {};
@@ -130,9 +140,31 @@ struct GitArchiveInputScheme : InputScheme
return input;
}
+ std::optional<std::string> getAccessToken(const std::string & host) const
+ {
+ auto tokens = settings.accessTokens.get();
+ if (auto token = get(tokens, host))
+ return *token;
+ return {};
+ }
+
+ Headers makeHeadersWithAuthTokens(const std::string & host) const
+ {
+ Headers headers;
+ auto accessToken = getAccessToken(host);
+ if (accessToken) {
+ auto hdr = accessHeaderFromToken(*accessToken);
+ if (hdr)
+ headers.push_back(*hdr);
+ else
+ warn("Unrecognized access token for host '%s'", host);
+ }
+ return headers;
+ }
+
virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
- virtual std::string getDownloadUrl(const Input & input) const = 0;
+ virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
@@ -161,7 +193,7 @@ struct GitArchiveInputScheme : InputScheme
auto url = getDownloadUrl(input);
- auto [tree, lastModified] = downloadTarball(store, url, "source", true);
+ auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
input.attrs.insert_or_assign("lastModified", lastModified);
@@ -183,49 +215,52 @@ struct GitHubInputScheme : GitArchiveInputScheme
{
std::string type() override { return "github"; }
- void addAccessToken(std::string & url) const
+ std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const
{
- std::string accessToken = settings.githubAccessToken.get();
- if (accessToken != "")
- url += "?access_token=" + accessToken;
+ // Github supports PAT/OAuth2 tokens and HTTP Basic
+ // Authentication. The former simply specifies the token, the
+ // latter can use the token as the password. Only the first
+ // is used here. See
+ // https://developer.github.com/v3/#authentication and
+ // https://docs.github.com/en/developers/apps/authorizing-oath-apps
+ return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
}
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
- auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
- addAccessToken(url);
+ Headers headers = makeHeadersWithAuthTokens(host);
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
- downloadFile(store, url, "source", false).storePath)));
+ downloadFile(store, url, "source", false, headers).storePath)));
auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
- std::string getDownloadUrl(const Input & input) const override
+ DownloadUrl getDownloadUrl(const Input & input) const override
{
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
- auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
- addAccessToken(url);
-
- return url;
+ Headers headers = makeHeadersWithAuthTokens(host);
+ return DownloadUrl { url, headers };
}
void clone(const Input & input, const Path & destDir) override
{
- auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.clone(destDir);
}
@@ -235,42 +270,65 @@ struct GitLabInputScheme : GitArchiveInputScheme
{
std::string type() override { return "gitlab"; }
+ std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const
+ {
+ // Gitlab supports 4 kinds of authorization, two of which are
+ // relevant here: OAuth2 and PAT (Private Access Token). The
+ // user can indicate which token is used by specifying the
+ // token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
+ // If the <TYPE> is unrecognized, this will fall back to
+ // treating this simply has <HDRNAME>:<HDRVAL>. See
+ // https://docs.gitlab.com/12.10/ee/api/README.html#authentication
+ auto fldsplit = token.find_first_of(':');
+ // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
+ if ("OAuth2" == token.substr(0, fldsplit))
+ return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
+ if ("PAT" == token.substr(0, fldsplit))
+ return std::make_pair("Private-token", token.substr(fldsplit+1));
+ warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit));
+ return std::nullopt;
+ }
+
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
- auto host_url = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+ // See rate limiting note below
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+
+ Headers headers = makeHeadersWithAuthTokens(host);
+
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
- downloadFile(store, url, "source", false).storePath)));
+ downloadFile(store, url, "source", false, headers).storePath)));
auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
- std::string getDownloadUrl(const Input & input) const override
+ DownloadUrl getDownloadUrl(const Input & input) const override
{
- // FIXME: This endpoint has a rate limit threshold of 5 requests per minute
- auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ // This endpoint has a rate limit threshold that may be
+ // server-specific and vary based whether the user is
+ // authenticated via an accessToken or not, but the usual rate
+ // is 10 reqs/sec/ip-addr. See
+ // https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
- /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
- std::string accessToken = settings.githubAccessToken.get();
- if (accessToken != "")
- url += "?access_token=" + accessToken;*/
-
- return url;
+ Headers headers = makeHeadersWithAuthTokens(host);
+ return DownloadUrl { url, headers };
}
void clone(const Input & input, const Path & destDir) override
{
- auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.clone(destDir);
}
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index a2d16365e..ca49482a9 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -5,6 +5,7 @@
#include "store-api.hh"
#include "archive.hh"
#include "tarfile.hh"
+#include "types.hh"
namespace nix::fetchers {
@@ -12,7 +13,8 @@ DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
- bool immutable)
+ bool immutable,
+ const Headers & headers)
{
// FIXME: check store
@@ -37,6 +39,7 @@ DownloadFileResult downloadFile(
return useCached();
FileTransferRequest request(url);
+ request.headers = headers;
if (cached)
request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
FileTransferResult res;
@@ -111,7 +114,8 @@ std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
- bool immutable)
+ bool immutable,
+ const Headers & headers)
{
Attrs inAttrs({
{"type", "tarball"},
@@ -127,7 +131,7 @@ std::pair<Tree, time_t> downloadTarball(
getIntAttr(cached->infoAttrs, "lastModified")
};
- auto res = downloadFile(store, url, name, immutable);
+ auto res = downloadFile(store, url, name, immutable, headers);
std::optional<StorePath> unpackedStorePath;
time_t lastModified;
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 14:09:58 +0000