aboutsummaryrefslogtreecommitdiff
path: root/src/libfetchers
diff options
context:
space:
mode:
Diffstat (limited to 'src/libfetchers')
-rw-r--r--src/libfetchers/fetchers.cc8
-rw-r--r--src/libfetchers/fetchers.hh15
-rw-r--r--src/libfetchers/git.cc3
-rw-r--r--src/libfetchers/github.cc145
-rw-r--r--src/libfetchers/indirect.cc3
-rw-r--r--src/libfetchers/mercurial.cc3
-rw-r--r--src/libfetchers/path.cc2
-rw-r--r--src/libfetchers/tarball.cc12
8 files changed, 136 insertions, 55 deletions
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index eaa635595..49851f7bc 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -69,6 +69,14 @@ ParsedURL Input::toURL() const
return scheme->toURL(*this);
}
+std::string Input::toURLString(const std::map<std::string, std::string> & extraQuery) const
+{
+ auto url = toURL();
+ for (auto & attr : extraQuery)
+ url.query.insert(attr);
+ return url.to_string();
+}
+
std::string Input::to_string() const
{
return toURL().to_string();
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index be71b786b..e8ae59143 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -23,7 +23,7 @@ struct InputScheme;
struct Input
{
- friend class InputScheme;
+ friend struct InputScheme;
std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs;
@@ -39,6 +39,8 @@ public:
ParsedURL toURL() const;
+ std::string toURLString(const std::map<std::string, std::string> & extraQuery = {}) const;
+
std::string to_string() const;
Attrs toAttrs() const;
@@ -73,7 +75,7 @@ public:
StorePath computeStorePath(Store & store) const;
- // Convience functions for common attributes.
+ // Convenience functions for common attributes.
std::string getType() const;
std::optional<Hash> getNarHash() const;
std::optional<std::string> getRef() const;
@@ -84,6 +86,9 @@ public:
struct InputScheme
{
+ virtual ~InputScheme()
+ { }
+
virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;
virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0;
@@ -119,12 +124,14 @@ DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
- bool immutable);
+ bool immutable,
+ const Headers & headers = {});
std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
- bool immutable);
+ bool immutable,
+ const Headers & headers = {});
}
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 5ca0f8521..a6411b02b 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -3,6 +3,7 @@
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
+#include "url-parts.hh"
#include <sys/time.h>
@@ -451,6 +452,6 @@ struct GitInputScheme : InputScheme
}
};
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
+static auto rGitInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
}
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 1cc0c5e2e..92ff224f7 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -3,19 +3,30 @@
#include "fetchers.hh"
#include "globals.hh"
#include "store-api.hh"
+#include "types.hh"
+#include "url-parts.hh"
+#include <optional>
#include <nlohmann/json.hpp>
namespace nix::fetchers {
-// A github or gitlab url
-const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
-std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
+struct DownloadUrl
+{
+ std::string url;
+ Headers headers;
+};
+
+// A github or gitlab host
+const static std::string hostRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
+std::regex hostRegex(hostRegexS, std::regex::ECMAScript);
struct GitArchiveInputScheme : InputScheme
{
virtual std::string type() = 0;
+ virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
+
std::optional<Input> inputFromURL(const ParsedURL & url) override
{
if (url.scheme != type()) return {};
@@ -50,9 +61,9 @@ struct GitArchiveInputScheme : InputScheme
throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
ref = value;
}
- else if (name == "url") {
- if (!std::regex_match(value, urlRegex))
- throw BadURL("URL '%s' contains an invalid instance url", url.url);
+ else if (name == "host") {
+ if (!std::regex_match(value, hostRegex))
+ throw BadURL("URL '%s' contains an invalid instance host", url.url);
host_url = value;
}
// FIXME: barf on unsupported attributes
@@ -67,7 +78,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.insert_or_assign("repo", path[1]);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
if (ref) input.attrs.insert_or_assign("ref", *ref);
- if (host_url) input.attrs.insert_or_assign("url", *host_url);
+ if (host_url) input.attrs.insert_or_assign("host", *host_url);
return input;
}
@@ -77,7 +88,7 @@ struct GitArchiveInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != type()) return {};
for (auto & [name, value] : attrs)
- if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
+ if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host")
throw Error("unsupported input attribute '%s'", name);
getStrAttr(attrs, "owner");
@@ -129,9 +140,31 @@ struct GitArchiveInputScheme : InputScheme
return input;
}
+ std::optional<std::string> getAccessToken(const std::string & host) const
+ {
+ auto tokens = settings.accessTokens.get();
+ if (auto token = get(tokens, host))
+ return *token;
+ return {};
+ }
+
+ Headers makeHeadersWithAuthTokens(const std::string & host) const
+ {
+ Headers headers;
+ auto accessToken = getAccessToken(host);
+ if (accessToken) {
+ auto hdr = accessHeaderFromToken(*accessToken);
+ if (hdr)
+ headers.push_back(*hdr);
+ else
+ warn("Unrecognized access token for host '%s'", host);
+ }
+ return headers;
+ }
+
virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
- virtual std::string getDownloadUrl(const Input & input) const = 0;
+ virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
@@ -160,7 +193,7 @@ struct GitArchiveInputScheme : InputScheme
auto url = getDownloadUrl(input);
- auto [tree, lastModified] = downloadTarball(store, url, "source", true);
+ auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
input.attrs.insert_or_assign("lastModified", lastModified);
@@ -182,49 +215,52 @@ struct GitHubInputScheme : GitArchiveInputScheme
{
std::string type() override { return "github"; }
- void addAccessToken(std::string & url) const
+ std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
- std::string accessToken = settings.githubAccessToken.get();
- if (accessToken != "")
- url += "?access_token=" + accessToken;
+ // Github supports PAT/OAuth2 tokens and HTTP Basic
+ // Authentication. The former simply specifies the token, the
+ // latter can use the token as the password. Only the first
+ // is used here. See
+ // https://developer.github.com/v3/#authentication and
+ // https://docs.github.com/en/developers/apps/authorizing-oath-apps
+ return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
}
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
- auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
- addAccessToken(url);
+ Headers headers = makeHeadersWithAuthTokens(host);
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
- downloadFile(store, url, "source", false).storePath)));
+ downloadFile(store, url, "source", false, headers).storePath)));
auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
- std::string getDownloadUrl(const Input & input) const override
+ DownloadUrl getDownloadUrl(const Input & input) const override
{
// FIXME: use regular /archive URLs instead? api.github.com
// might have stricter rate limits.
- auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
- addAccessToken(url);
-
- return url;
+ Headers headers = makeHeadersWithAuthTokens(host);
+ return DownloadUrl { url, headers };
}
void clone(const Input & input, const Path & destDir) override
{
- auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.clone(destDir);
}
@@ -234,48 +270,71 @@ struct GitLabInputScheme : GitArchiveInputScheme
{
std::string type() override { return "gitlab"; }
+ std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
+ {
+ // Gitlab supports 4 kinds of authorization, two of which are
+ // relevant here: OAuth2 and PAT (Private Access Token). The
+ // user can indicate which token is used by specifying the
+ // token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
+ // If the <TYPE> is unrecognized, this will fall back to
+ // treating this simply has <HDRNAME>:<HDRVAL>. See
+ // https://docs.gitlab.com/12.10/ee/api/README.html#authentication
+ auto fldsplit = token.find_first_of(':');
+ // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
+ if ("OAuth2" == token.substr(0, fldsplit))
+ return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
+ if ("PAT" == token.substr(0, fldsplit))
+ return std::make_pair("Private-token", token.substr(fldsplit+1));
+ warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit));
+ return std::nullopt;
+ }
+
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
- auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+ // See rate limiting note below
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+
+ Headers headers = makeHeadersWithAuthTokens(host);
+
auto json = nlohmann::json::parse(
readFile(
store->toRealPath(
- downloadFile(store, url, "source", false).storePath)));
+ downloadFile(store, url, "source", false, headers).storePath)));
auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1);
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
return rev;
}
- std::string getDownloadUrl(const Input & input) const override
+ DownloadUrl getDownloadUrl(const Input & input) const override
{
- // FIXME: This endpoint has a rate limit threshold of 5 requests per minute
- auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ // This endpoint has a rate limit threshold that may be
+ // server-specific and vary based whether the user is
+ // authenticated via an accessToken or not, but the usual rate
+ // is 10 reqs/sec/ip-addr. See
+ // https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
- /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
- std::string accessToken = settings.githubAccessToken.get();
- if (accessToken != "")
- url += "?access_token=" + accessToken;*/
-
- return url;
+ Headers headers = makeHeadersWithAuthTokens(host);
+ return DownloadUrl { url, headers };
}
void clone(const Input & input, const Path & destDir) override
{
- auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
// FIXME: get username somewhere
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
- host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+ host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
.clone(destDir);
}
};
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
-static auto r2 = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
+static auto rGitHubInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+static auto rGitLabInputScheme = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
}
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
index b981d4d8e..10e59919a 100644
--- a/src/libfetchers/indirect.cc
+++ b/src/libfetchers/indirect.cc
@@ -1,4 +1,5 @@
#include "fetchers.hh"
+#include "url-parts.hh"
namespace nix::fetchers {
@@ -99,6 +100,6 @@ struct IndirectInputScheme : InputScheme
}
};
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
+static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
}
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 3e76ffc4d..7d3d52751 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -3,6 +3,7 @@
#include "globals.hh"
#include "tarfile.hh"
#include "store-api.hh"
+#include "url-parts.hh"
#include <sys/time.h>
@@ -292,6 +293,6 @@ struct MercurialInputScheme : InputScheme
}
};
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
+static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
}
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index 99d4b4e8f..bcb904c0d 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -102,6 +102,6 @@ struct PathInputScheme : InputScheme
}
};
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
+static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
}
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index a2d16365e..8c0f20475 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -5,6 +5,7 @@
#include "store-api.hh"
#include "archive.hh"
#include "tarfile.hh"
+#include "types.hh"
namespace nix::fetchers {
@@ -12,7 +13,8 @@ DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
- bool immutable)
+ bool immutable,
+ const Headers & headers)
{
// FIXME: check store
@@ -37,6 +39,7 @@ DownloadFileResult downloadFile(
return useCached();
FileTransferRequest request(url);
+ request.headers = headers;
if (cached)
request.expectedETag = getStrAttr(cached->infoAttrs, "etag");
FileTransferResult res;
@@ -111,7 +114,8 @@ std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
- bool immutable)
+ bool immutable,
+ const Headers & headers)
{
Attrs inAttrs({
{"type", "tarball"},
@@ -127,7 +131,7 @@ std::pair<Tree, time_t> downloadTarball(
getIntAttr(cached->infoAttrs, "lastModified")
};
- auto res = downloadFile(store, url, name, immutable);
+ auto res = downloadFile(store, url, name, immutable, headers);
std::optional<StorePath> unpackedStorePath;
time_t lastModified;
@@ -227,6 +231,6 @@ struct TarballInputScheme : InputScheme
}
};
-static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
+static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 05:31:08 +0000