From 9f4d8c6170517c9452e25dc29c56a6fbb43d40a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Jan 2020 16:27:53 +0100 Subject: Pluggable fetchers Flakes are now fetched using an extensible mechanism. Also lots of other flake cleanups. --- src/libstore/fetchers/fetchers.cc | 56 ++++++ src/libstore/fetchers/fetchers.hh | 75 ++++++++ src/libstore/fetchers/git.cc | 382 ++++++++++++++++++++++++++++++++++++++ src/libstore/fetchers/github.cc | 183 ++++++++++++++++++ src/libstore/fetchers/indirect.cc | 114 ++++++++++++ src/libstore/fetchers/parse.cc | 129 +++++++++++++ src/libstore/fetchers/parse.hh | 28 +++ src/libstore/fetchers/regex.hh | 32 ++++ src/libstore/fetchers/registry.cc | 145 +++++++++++++++ src/libstore/fetchers/registry.hh | 47 +++++ src/libstore/globals.hh | 9 + src/libstore/local.mk | 2 +- src/libstore/store-api.cc | 23 +-- 13 files changed, 1203 insertions(+), 22 deletions(-) create mode 100644 src/libstore/fetchers/fetchers.cc create mode 100644 src/libstore/fetchers/fetchers.hh create mode 100644 src/libstore/fetchers/git.cc create mode 100644 src/libstore/fetchers/github.cc create mode 100644 src/libstore/fetchers/indirect.cc create mode 100644 src/libstore/fetchers/parse.cc create mode 100644 src/libstore/fetchers/parse.hh create mode 100644 src/libstore/fetchers/regex.hh create mode 100644 src/libstore/fetchers/registry.cc create mode 100644 src/libstore/fetchers/registry.hh (limited to 'src/libstore') diff --git a/src/libstore/fetchers/fetchers.cc b/src/libstore/fetchers/fetchers.cc new file mode 100644 index 000000000..7f82d5af0 --- /dev/null +++ b/src/libstore/fetchers/fetchers.cc @@ -0,0 +1,56 @@ +#include "fetchers.hh" +#include "parse.hh" +#include "store-api.hh" + +namespace nix::fetchers { + +std::unique_ptr>> inputSchemes = nullptr; + +void registerInputScheme(std::unique_ptr && inputScheme) +{ + if (!inputSchemes) inputSchemes = std::make_unique>>(); + inputSchemes->push_back(std::move(inputScheme)); +} + +std::unique_ptr inputFromURL(const ParsedURL & url) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromURL(url); + if (res) return res; + } + throw Error("input '%s' is unsupported", url.url); +} + +std::unique_ptr inputFromURL(const std::string & url) +{ + return inputFromURL(parseURL(url)); +} + +std::pair> Input::fetchTree(ref store) const +{ + auto [tree, input] = fetchTreeInternal(store); + + if (tree.actualPath == "") + tree.actualPath = store->toRealPath(store->printStorePath(tree.storePath)); + + if (!tree.narHash) + tree.narHash = store->queryPathInfo(tree.storePath)->narHash; + + if (input->narHash) + assert(input->narHash == tree.narHash); + + return {std::move(tree), input}; +} + +std::shared_ptr Input::applyOverrides( + std::optional ref, + std::optional rev) const +{ + if (ref) + throw Error("don't know how to apply '%s' to '%s'", *ref, to_string()); + if (rev) + throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string()); + return shared_from_this(); +} + +} diff --git a/src/libstore/fetchers/fetchers.hh b/src/libstore/fetchers/fetchers.hh new file mode 100644 index 000000000..b59b328cc --- /dev/null +++ b/src/libstore/fetchers/fetchers.hh @@ -0,0 +1,75 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "path.hh" + +#include + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Input; + +struct Tree +{ + Path actualPath; + StorePath storePath; + Hash narHash; + std::optional rev; + std::optional revCount; + std::optional lastModified; +}; + +struct Input : std::enable_shared_from_this +{ + std::string type; + std::optional narHash; + + virtual bool operator ==(const Input & other) const { return false; } + + virtual bool isDirect() const { return true; } + + virtual bool isImmutable() const { return (bool) narHash; } + + virtual bool contains(const Input & other) const { return false; } + + virtual std::optional getRef() const { return {}; } + + virtual std::optional getRev() const { return {}; } + + virtual std::string to_string() const = 0; + + std::pair> fetchTree(ref store) const; + + virtual std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const; + + virtual std::optional getSourcePath() const { return {}; } + + virtual void clone(const Path & destDir) const + { + throw Error("do not know how to clone input '%s'", to_string()); + } + +private: + + virtual std::pair> fetchTreeInternal(ref store) const = 0; +}; + +struct ParsedURL; + +struct InputScheme +{ + virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; +}; + +std::unique_ptr inputFromURL(const ParsedURL & url); + +std::unique_ptr inputFromURL(const std::string & url); + +void registerInputScheme(std::unique_ptr && fetcher); + +} diff --git a/src/libstore/fetchers/git.cc b/src/libstore/fetchers/git.cc new file mode 100644 index 000000000..bfa862cf5 --- /dev/null +++ b/src/libstore/fetchers/git.cc @@ -0,0 +1,382 @@ +#include "fetchers.hh" +#include "parse.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" +#include "regex.hh" + +#include + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +static Path getCacheInfoPathFor(const std::string & name, const Hash & rev) +{ + Path cacheDir = getCacheDir() + "/nix/git-revs-v2"; + std::string linkName = + name == "source" + ? rev.gitRev() + : hashString(htSHA512, name + std::string("\0"s) + rev.gitRev()).to_string(Base32, false); + return cacheDir + "/" + linkName + ".link"; +} + +static void cacheGitInfo(Store & store, const std::string & name, const Tree & tree) +{ + nlohmann::json json; + json["storePath"] = store.printStorePath(tree.storePath); + json["name"] = name; + json["rev"] = tree.rev->gitRev(); + json["revCount"] = *tree.revCount; + json["lastModified"] = *tree.lastModified; + + auto cacheInfoPath = getCacheInfoPathFor(name, *tree.rev); + createDirs(dirOf(cacheInfoPath)); + writeFile(cacheInfoPath, json.dump()); +} + +static std::optional lookupGitInfo( + ref store, + const std::string & name, + const Hash & rev) +{ + try { + auto json = nlohmann::json::parse(readFile(getCacheInfoPathFor(name, rev))); + + assert(json["name"] == name && Hash((std::string) json["rev"], htSHA1) == rev); + + auto storePath = store->parseStorePath((std::string) json["storePath"]); + + if (store->isValidPath(storePath)) { + Tree tree{ + .actualPath = store->toRealPath(store->printStorePath(storePath)), + .storePath = std::move(storePath), + .rev = rev, + .revCount = json["revCount"], + .lastModified = json["lastModified"], + }; + return tree; + } + + } catch (SysError & e) { + if (e.errNo != ENOENT) throw; + } + + return {}; +} + +struct GitInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + + GitInput(const ParsedURL & url) : url(url) + { + type = "git"; + } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url.url == other2->url.url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + std::string to_string() const override + { + ParsedURL url2(url); + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + return url2.to_string(); + } + + void clone(const Path & destDir) const override + { + auto [isLocal, actualUrl] = getActualUrl(); + + Strings args = {"clone"}; + + args.push_back(actualUrl); + + if (ref) { + args.push_back("--branch"); + args.push_back(*ref); + } + + if (rev) throw Error("cloning a specific revision is not implemented"); + + args.push_back(destDir); + + runProgram("git", true, args); + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + if (!res->ref && res->rev) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); + + return res; + } + + std::optional getSourcePath() const + { + if (url.scheme == "git+file" && !ref && !rev) + return url.path; + return {}; + } + + std::pair getActualUrl() const + { + // Don't clone git+file:// URIs (but otherwise treat them the + // same as remote URIs, i.e. don't use the working tree or + // HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + bool isLocal = url.scheme == "git+file" && !forceHttp; + return {isLocal, isLocal ? url.path : std::string(url.base, 4)}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + assert(!rev || rev->type == htSHA1); + + if (rev) { + if (auto tree = lookupGitInfo(store, name, *rev)) + return {std::move(*tree), input}; + } + + auto [isLocal, actualUrl] = getActualUrl(); + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input->ref && !input->rev && isLocal) { + bool clean = false; + + /* Check whether this repo has any commits. There are + probably better ways to do this. */ + bool haveCommits = !readDirectory(actualUrl + "/.git/refs/heads").empty(); + + try { + if (haveCommits) { + runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); + clean = true; + } + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + } + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked files. */ + + if (!settings.allowDirty) + throw Error("Git tree '%s' is dirty", actualUrl); + + if (settings.warnDirty) + warn("Git tree '%s' is dirty", actualUrl); + + auto files = tokenizeString>( + runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + + auto tree = Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .revCount = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "rev-list", "--count", "HEAD" })) : 0, + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, + }; + + return {std::move(tree), input}; + } + } + + if (!input->ref) input->ref = isLocal ? "HEAD" : "master"; + + Path repoDir; + + if (isLocal) { + + if (!input->rev) + input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + + repoDir = actualUrl; + + } else { + + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); + repoDir = cacheDir; + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } + + Path localRefFile = + input->ref->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input->ref + : cacheDir + "/refs/heads/" + *input->ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (input->rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } + + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + try { + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + } + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + if (!input->rev) + input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + } + + if (auto tree = lookupGitInfo(store, name, *input->rev)) + return {std::move(*tree), input}; + + // FIXME: check whether rev is an ancestor of ref. + + printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + unpackTarfile(*source, tmpDir); + + auto storePath = store->addToStore(name, tmpDir); + auto revCount = std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + + auto tree = Tree { + .actualPath = store->toRealPath(store->printStorePath(storePath)), + .storePath = std::move(storePath), + .rev = input->rev, + .revCount = revCount, + .lastModified = lastModified, + }; + + cacheGitInfo(*store, name, tree); + + return {std::move(tree), input}; + } +}; + +struct GitInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return nullptr; + + auto input = std::make_unique(url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (!std::regex_match(value, revRegex)) + throw BadURL("Git URL '%s' contains an invalid commit hash", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("Git URL '%s' contains an invalid branch/tag name", url.url); + input->ref = value; + } + } + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libstore/fetchers/github.cc b/src/libstore/fetchers/github.cc new file mode 100644 index 000000000..c75680649 --- /dev/null +++ b/src/libstore/fetchers/github.cc @@ -0,0 +1,183 @@ +#include "fetchers.hh" +#include "download.hh" +#include "globals.hh" +#include "parse.hh" +#include "regex.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); +std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct GitHubInput : Input +{ + std::string owner; + std::string repo; + std::optional ref; + std::optional rev; + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && owner == other2->owner + && repo == other2->repo + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + std::string to_string() const override + { + auto s = fmt("github:%s/%s", owner, repo); + assert(!(ref && rev)); + if (ref) s += "/" + *ref; + if (rev) s += "/" + rev->to_string(Base16, false); + return s; + } + + void clone(const Path & destDir) const override + { + std::shared_ptr input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); + input = input->applyOverrides(ref.value_or("master"), rev); + input->clone(destDir); + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto rev = this->rev; + + #if 0 + if (rev) { + if (auto gitInfo = lookupGitInfo(store, "source", *rev)) + return *gitInfo; + } + #endif + + if (!rev) { + auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", + owner, repo, ref ? *ref : "master"); + CachedDownloadRequest request(url); + request.ttl = rev ? 1000000000 : settings.tarballTtl; + auto result = getDownloader()->downloadCached(store, request); + auto json = nlohmann::json::parse(readFile(result.path)); + rev = Hash(json["sha"], htSHA1); + debug("HEAD revision for '%s' is %s", url, rev->gitRev()); + } + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + owner, repo, rev->to_string(Base16, false)); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + CachedDownloadRequest request(url); + request.unpack = true; + request.name = "source"; + request.ttl = 1000000000; + request.getLastModified = true; + auto dresult = getDownloader()->downloadCached(store, request); + + assert(dresult.lastModified); + + Tree result{ + .actualPath = dresult.path, + .storePath = store->parseStorePath(dresult.storePath), + .rev = *rev, + .lastModified = *dresult.lastModified + }; + + #if 0 + // FIXME: this can overwrite a cache file that contains a revCount. + cacheGitInfo("source", gitInfo); + #endif + + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = *rev; + + return {std::move(result), input}; + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } +}; + +struct GitHubInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "github") return nullptr; + + auto path = tokenizeString>(url.path, "/"); + auto input = std::make_unique(); + input->type = "github"; + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + input->rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + input->ref = path[2]; + else + throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (!std::regex_match(value, revRegex)) + throw BadURL("GitHub URL '%s' contains an invalid commit hash", url.url); + if (input->rev) + throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); + if (input->ref) + throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); + input->ref = value; + } + } + + if (input->ref && input->rev) + throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); + + input->owner = path[0]; + input->repo = path[1]; + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libstore/fetchers/indirect.cc b/src/libstore/fetchers/indirect.cc new file mode 100644 index 000000000..1f9d1e24f --- /dev/null +++ b/src/libstore/fetchers/indirect.cc @@ -0,0 +1,114 @@ +#include "fetchers.hh" +#include "parse.hh" +#include "regex.hh" + +namespace nix::fetchers { + +std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct IndirectInput : Input +{ + std::string id; + std::optional rev; + std::optional ref; + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && id == other2->id + && rev == other2->rev + && ref == other2->ref; + } + + bool isDirect() const override + { + return false; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + bool contains(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && id == other2->id + && (!ref || ref == other2->ref) + && (!rev || rev == other2->rev); + } + + std::string to_string() const override + { + ParsedURL url; + url.scheme = "flake"; + url.path = id; + if (ref) { url.path += '/'; url.path += *ref; }; + if (rev) { url.path += '/'; url.path += rev->gitRev(); }; + return url.to_string(); + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + throw Error("indirect input '%s' cannot be fetched directly", to_string()); + } +}; + +struct IndirectInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "flake") return nullptr; + + auto path = tokenizeString>(url.path, "/"); + auto input = std::make_unique(); + input->type = "indirect"; + + if (path.size() == 1) { + } else if (path.size() == 2) { + if (std::regex_match(path[1], revRegex)) + input->rev = Hash(path[1], htSHA1); + else if (std::regex_match(path[1], refRegex)) + input->ref = path[1]; + else + throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); + } else if (path.size() == 3) { + if (!std::regex_match(path[1], refRegex)) + throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); + input->ref = path[1]; + if (!std::regex_match(path[2], revRegex)) + throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); + input->rev = Hash(path[2], htSHA1); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + // FIXME: forbid query params? + + input->id = path[0]; + if (!std::regex_match(input->id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", input->id); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libstore/fetchers/parse.cc b/src/libstore/fetchers/parse.cc new file mode 100644 index 000000000..96a0681e5 --- /dev/null +++ b/src/libstore/fetchers/parse.cc @@ -0,0 +1,129 @@ +#include "parse.hh" +#include "util.hh" +#include "regex.hh" + +namespace nix::fetchers { + +std::regex refRegex(refRegexS, std::regex::ECMAScript); +std::regex revRegex(revRegexS, std::regex::ECMAScript); + +ParsedURL parseURL(const std::string & url) +{ + static std::regex uriRegex( + "(((" + schemeRegex + "):" + + "(//(" + authorityRegex + "))?" + + "(" + pathRegex + "))" + + "(?:\\?(" + queryRegex + "))?" + + "(?:#(" + queryRegex + "))?" + + ")", + std::regex::ECMAScript); + + std::smatch match; + + if (std::regex_match(url, match, uriRegex)) { + auto & base = match[2]; + std::string scheme = match[3]; + auto authority = match[4].matched + ? std::optional(match[5]) : std::nullopt; + std::string path = match[6]; + auto & query = match[7]; + auto & fragment = match[8]; + + auto isFile = scheme.find("file") != std::string::npos; + + if (authority && *authority != "" && isFile) + throw Error("file:// URL '%s' has unexpected authority '%s'", + url, *authority); + + if (isFile && path.empty()) + path = "/"; + + return ParsedURL{ + .url = url, + .base = base, + .scheme = scheme, + .authority = authority, + .path = path, + .query = decodeQuery(query), + .fragment = percentDecode(std::string(fragment)) + }; + } + + else + throw BadURL("'%s' is not a valid URL", url); +} + +std::string percentDecode(std::string_view in) +{ + std::string decoded; + for (size_t i = 0; i < in.size(); ) { + if (in[i] == '%') { + if (i + 2 >= in.size()) + throw BadURL("invalid URI parameter '%s'", in); + try { + decoded += std::stoul(std::string(in, i + 1, 2), 0, 16); + i += 3; + } catch (...) { + throw BadURL("invalid URI parameter '%s'", in); + } + } else + decoded += in[i++]; + } + return decoded; +} + +std::map decodeQuery(const std::string & query) +{ + std::map result; + + for (auto s : tokenizeString(query, "&")) { + auto e = s.find('='); + if (e != std::string::npos) + result.emplace( + s.substr(0, e), + percentDecode(std::string_view(s).substr(e + 1))); + } + + return result; +} + +std::string percentEncode(std::string_view s) +{ + std::string res; + for (auto & c : s) + if ((c >= 'a' && c <= 'z') + || (c >= 'A' && c <= 'Z') + || (c >= '0' && c <= '9') + || strchr("-._~!$&'()*+,;=:@", c)) + res += c; + else + res += fmt("%%%02x", (unsigned int) c); + return res; +} + +std::string encodeQuery(const std::map & ss) +{ + std::string res; + bool first = true; + for (auto & [name, value] : ss) { + if (!first) res += '&'; + first = false; + res += percentEncode(name); + res += '='; + res += percentEncode(value); + } + return res; +} + +std::string ParsedURL::to_string() const +{ + return + scheme + + ":" + + (authority ? "//" + *authority : "") + + path + + (query.empty() ? "" : "?" + encodeQuery(query)) + + (fragment.empty() ? "" : "#" + percentEncode(fragment)); +} + +} diff --git a/src/libstore/fetchers/parse.hh b/src/libstore/fetchers/parse.hh new file mode 100644 index 000000000..22cc57816 --- /dev/null +++ b/src/libstore/fetchers/parse.hh @@ -0,0 +1,28 @@ +#pragma once + +#include "types.hh" + +namespace nix::fetchers { + +struct ParsedURL +{ + std::string url; + std::string base; // URL without query/fragment + std::string scheme; + std::optional authority; + std::string path; + std::map query; + std::string fragment; + + std::string to_string() const; +}; + +MakeError(BadURL, Error); + +std::string percentDecode(std::string_view in); + +std::map decodeQuery(const std::string & query); + +ParsedURL parseURL(const std::string & url); + +} diff --git a/src/libstore/fetchers/regex.hh b/src/libstore/fetchers/regex.hh new file mode 100644 index 000000000..eb061a048 --- /dev/null +++ b/src/libstore/fetchers/regex.hh @@ -0,0 +1,32 @@ +#pragma once + +#include + +namespace nix::fetchers { + +// URI stuff. +const static std::string pctEncoded = "%[0-9a-fA-F][0-9a-fA-F]"; +const static std::string schemeRegex = "[a-z+]+"; +const static std::string authorityRegex = + "(?:(?:[a-z])*@)?" + "[a-zA-Z0-9._~-]*"; +const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; +const static std::string pathRegex = "(?:/?" + segmentRegex + "(?:/" + segmentRegex + ")*|/?)"; +const static std::string pcharRegex = + "(?:[a-zA-Z0-9-._~!$&'()*+,;=:@ ]|" + pctEncoded + ")"; +const static std::string queryRegex = "(?:" + pcharRegex + "|[/?])*"; + +// A Git ref (i.e. branch or tag name). +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +extern std::regex refRegex; + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +extern std::regex revRegex; + +// A ref or revision, or a ref followed by a revision. +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*"; + +} diff --git a/src/libstore/fetchers/registry.cc b/src/libstore/fetchers/registry.cc new file mode 100644 index 000000000..dd74e16c1 --- /dev/null +++ b/src/libstore/fetchers/registry.cc @@ -0,0 +1,145 @@ +#include "registry.hh" +#include "util.hh" +#include "fetchers.hh" +#include "globals.hh" +#include "download.hh" + +#include + +namespace nix::fetchers { + +std::shared_ptr Registry::read( + const Path & path, RegistryType type) +{ + auto registry = std::make_shared(); + registry->type = type; + + if (!pathExists(path)) + return std::make_shared(); + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", path, version); + + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + // FIXME: remove 'uri' soon. + auto url = i->value("url", i->value("uri", "")); + if (url.empty()) + throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", + path, i.key()); + registry->entries.push_back( + {inputFromURL(i.key()), inputFromURL(url)}); + } + + return registry; +} + +void Registry::write(const Path & path) +{ + nlohmann::json json; + json["version"] = 1; + for (auto & elem : entries) + json["flakes"][elem.first->to_string()] = { {"url", elem.second->to_string()} }; + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); +} + +void Registry::add( + const std::shared_ptr & from, + const std::shared_ptr & to) +{ + entries.emplace_back(from, to); +} + +void Registry::remove(const std::shared_ptr & input) +{ + // FIXME: use C++20 std::erase. + for (auto i = entries.begin(); i != entries.end(); ) + if (*i->first == *input) + i = entries.erase(i); + else + ++i; +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr getUserRegistry() +{ + return Registry::read(getUserRegistryPath(), Registry::User); +} + +#if 0 +std::shared_ptr getFlagRegistry(RegistryOverrides registryOverrides) +{ + auto flagRegistry = std::make_shared(); + for (auto const & x : registryOverrides) + flagRegistry->entries.insert_or_assign( + parseFlakeRef2(x.first), + parseFlakeRef2(x.second)); + return flagRegistry; +} +#endif + +static std::shared_ptr getGlobalRegistry(ref store) +{ + static auto reg = [&]() { + auto path = settings.flakeRegistry; + + if (!hasPrefix(path, "/")) { + CachedDownloadRequest request(path); + request.name = "flake-registry.json"; + request.gcRoot = true; + path = getDownloader()->downloadCached(store, request).path; + } + + return Registry::read(path, Registry::Global); + }(); + + return reg; +} + +Registries getRegistries(ref store) +{ + Registries registries; + //registries.push_back(getFlagRegistry(registryOverrides)); + registries.push_back(getUserRegistry()); + registries.push_back(getGlobalRegistry(store)); + return registries; +} + +std::shared_ptr lookupInRegistries( + ref store, + std::shared_ptr input) +{ + int n = 0; + + restart: + + n++; + if (n > 100) throw Error("cycle detected in flake registr for '%s'", input); + + for (auto & registry : getRegistries(store)) { + // FIXME: O(n) + for (auto & entry : registry->entries) { + if (entry.first->contains(*input)) { + input = entry.second->applyOverrides( + !entry.first->getRef() && input->getRef() ? input->getRef() : std::optional(), + !entry.first->getRev() && input->getRev() ? input->getRev() : std::optional()); + goto restart; + } + } + } + + if (!input->isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input->to_string()); + + return input; +} + +} diff --git a/src/libstore/fetchers/registry.hh b/src/libstore/fetchers/registry.hh new file mode 100644 index 000000000..1757ce323 --- /dev/null +++ b/src/libstore/fetchers/registry.hh @@ -0,0 +1,47 @@ +#pragma once + +#include "types.hh" + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Input; + +struct Registry +{ + enum RegistryType { + Flag = 0, + User = 1, + Global = 2, + }; + + RegistryType type; + + std::vector, std::shared_ptr>> entries; + + static std::shared_ptr read( + const Path & path, RegistryType type); + + void write(const Path & path); + + void add( + const std::shared_ptr & from, + const std::shared_ptr & to); + + void remove(const std::shared_ptr & input); +}; + +typedef std::vector> Registries; + +std::shared_ptr getUserRegistry(); + +Path getUserRegistryPath(); + +Registries getRegistries(ref store); + +std::shared_ptr lookupInRegistries( + ref store, + std::shared_ptr input); + +} diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 247fba2f8..d0500be22 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -365,6 +365,15 @@ public: bool isExperimentalFeatureEnabled(const std::string & name); void requireExperimentalFeature(const std::string & name); + + Setting flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", + "Path or URI of the global flake registry."}; + + Setting allowDirty{this, true, "allow-dirty", + "Whether to allow dirty Git/Mercurial trees."}; + + Setting warnDirty{this, true, "warn-dirty", + "Whether to warn about dirty Git/Mercurial trees."}; }; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index ac68c2342..e803ff85a 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -4,7 +4,7 @@ libstore_NAME = libnixstore libstore_DIR := $(d) -libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) +libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/fetchers/*.cc) libstore_LIBS = libutil libnixrust diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c29ca5a12..e37829b17 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -6,6 +6,7 @@ #include "thread-pool.hh" #include "json.hh" #include "derivations.hh" +#include "fetchers/parse.hh" #include @@ -864,27 +865,7 @@ std::pair splitUriAndParams(const std::string & uri_ Store::Params params; auto q = uri.find('?'); if (q != std::string::npos) { - for (auto s : tokenizeString(uri.substr(q + 1), "&")) { - auto e = s.find('='); - if (e != std::string::npos) { - auto value = s.substr(e + 1); - std::string decoded; - for (size_t i = 0; i < value.size(); ) { - if (value[i] == '%') { - if (i + 2 >= value.size()) - throw Error("invalid URI parameter '%s'", value); - try { - decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); - i += 3; - } catch (...) { - throw Error("invalid URI parameter '%s'", value); - } - } else - decoded += value[i++]; - } - params[s.substr(0, e)] = decoded; - } - } + params = fetchers::decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } return {uri, params}; -- cgit v1.2.3