Pluggable fetchers

Flakes are now fetched using an extensible mechanism. Also lots of
other flake cleanups.

1 files changed, 122 insertions, 240 deletions

diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc
index ff7c725cb..397b1b84b 100644
--- a/src/libexpr/flake/flakeref.cc
+++ b/src/libexpr/flake/flakeref.cc
@@ -1,283 +1,165 @@
 #include "flakeref.hh"
 #include "store-api.hh"
-
-#include <regex>
+#include "fetchers/parse.hh"
+#include "fetchers/fetchers.hh"
+#include "fetchers/registry.hh"
+#include "fetchers/regex.hh"
 
 namespace nix {
 
-// A Git ref (i.e. branch or tag name).
-const static std::string refRegex = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
-
-// A Git revision (a SHA-1 commit hash).
-const static std::string revRegexS = "[0-9a-fA-F]{40}";
-std::regex revRegex(revRegexS, std::regex::ECMAScript);
-
-// A Git ref or revision.
-const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + "))";
-
-// A rev ("e72daba8250068216d79d2aeef40d4d95aff6666"), or a ref
-// optionally followed by a rev (e.g. "master" or
-// "master/e72daba8250068216d79d2aeef40d4d95aff6666").
-const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))";
-
-const static std::string flakeId = "[a-zA-Z][a-zA-Z0-9_-]*";
-
-// GitHub references.
-const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*";
-const static std::string repoRegex = "[a-zA-Z][a-zA-Z0-9_-]*";
-
-// URI stuff.
-const static std::string schemeRegex = "[a-z+]+";
-const static std::string authorityRegex = "[a-zA-Z0-9._~-]*";
-const static std::string segmentRegex = "[a-zA-Z0-9._~-]+";
-const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*";
-
+#if 0
 // 'dir' path elements cannot start with a '.'. We also reject
 // potentially dangerous characters like ';'.
 const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
 const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
+#endif
 
 
-FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative)
+std::string FlakeRef::to_string() const
 {
-    // FIXME: could combine this into one regex.
-
-    static std::regex flakeRegex(
-        "(?:flake:)?(" + flakeId + ")(?:/(?:" + refAndOrRevRegex + "))?",
-        std::regex::ECMAScript);
-
-    static std::regex githubRegex(
-        "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?",
-        std::regex::ECMAScript);
-
-    static std::regex uriRegex(
-        "((" + schemeRegex + "):" +
-        "(?://(" + authorityRegex + "))?" +
-        "(" + pathRegex + "))",
-        std::regex::ECMAScript);
+    return input->to_string();
+}
 
-    static std::regex refRegex2(refRegex, std::regex::ECMAScript);
+bool FlakeRef::isDirect() const
+{
+    return input->isDirect();
+}
 
-    static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript);
+bool FlakeRef::isImmutable() const
+{
+    return input->isImmutable();
+}
 
-    auto [uri2, params] = splitUriAndParams(uri_);
-    std::string uri(uri2);
+std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
+{
+    str << flakeRef.to_string();
+    return str;
+}
 
-    auto handleSubdir = [&](const std::string & name, const std::string & value) {
-        if (name == "dir") {
-            if (value != "" && !std::regex_match(value, subDirRegex2))
-                throw BadFlakeRef("flake '%s' has invalid subdirectory '%s'", uri, value);
-            subdir = value;
-            return true;
-        } else
-            return false;
-    };
+bool FlakeRef::operator==(const FlakeRef & other) const
+{
+    return *input == *other.input && subdir == other.subdir;
+}
 
-    auto handleGitParams = [&](const std::string & name, const std::string & value) {
-        if (name == "rev") {
-            if (!std::regex_match(value, revRegex))
-                throw BadFlakeRef("invalid Git revision '%s'", value);
-            rev = Hash(value, htSHA1);
-        } else if (name == "ref") {
-            if (!std::regex_match(value, refRegex2))
-                throw BadFlakeRef("invalid Git ref '%s'", value);
-            ref = value;
-        } else if (handleSubdir(name, value))
-            ;
-        else return false;
-        return true;
-    };
+FlakeRef FlakeRef::resolve(ref<Store> store) const
+{
+    return FlakeRef(lookupInRegistries(store, input), subdir);
+}
 
-    std::smatch match;
-    if (std::regex_match(uri, match, flakeRegex)) {
-        IsId d;
-        d.id = match[1];
-        if (match[2].matched)
-            rev = Hash(match[2], htSHA1);
-        else if (match[3].matched) {
-            ref = match[3];
-            if (match[4].matched)
-                rev = Hash(match[4], htSHA1);
-        }
-        data = d;
-    }
+FlakeRef parseFlakeRef(
+    const std::string & url, const std::optional<Path> & baseDir)
+{
+    auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir);
+    if (fragment != "")
+        throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
+    return flakeRef;
+}
 
-    else if (std::regex_match(uri, match, githubRegex)) {
-        IsGitHub d;
-        d.owner = match[1];
-        d.repo = match[2];
-        if (match[3].matched)
-            rev = Hash(match[3], htSHA1);
-        else if (match[4].matched) {
-            ref = match[4];
-        }
-        for (auto & param : params) {
-            if (handleSubdir(param.first, param.second))
-                ;
-            else
-                throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri);
-        }
-        data = d;
+std::optional<FlakeRef> maybeParseFlakeRef(
+    const std::string & url, const std::optional<Path> & baseDir)
+{
+    try {
+        return parseFlakeRef(url, baseDir);
+    } catch (Error &) {
+        return {};
     }
+}
 
-    else if (std::regex_match(uri, match, uriRegex)) {
-        auto & scheme = match[2];
-        if (scheme == "git" ||
-            scheme == "git+http" ||
-            scheme == "git+https" ||
-            scheme == "git+ssh" ||
-            scheme == "git+file" ||
-            scheme == "file")
-        {
-            IsGit d;
-            d.uri = match[1];
-            for (auto & param : params) {
-                if (handleGitParams(param.first, param.second))
-                    ;
-                else
-                    // FIXME: should probably pass through unknown parameters
-                    throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri);
-            }
-            if (rev && !ref)
-                throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri);
-            data = d;
-        } else
-            throw BadFlakeRef("unsupported URI scheme '%s' in flake reference '%s'", scheme, uri);
-    }
+std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
+    const std::string & url, const std::optional<Path> & baseDir)
+{
+    using namespace fetchers;
 
-    else if ((hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == ".")))
-        && uri.find(':') == std::string::npos)
-    {
-        IsPath d;
-        if (allowRelative) {
-            d.path = absPath(uri);
-            try {
-                if (!S_ISDIR(lstat(d.path).st_mode))
-                    throw MissingFlake("path '%s' is not a flake (sub)directory", d.path);
-            } catch (SysError & e) {
-                if (e.errNo == ENOENT || e.errNo == EISDIR)
-                    throw MissingFlake("flake '%s' does not exist", d.path);
-                throw;
-            }
-            while (true) {
-                if (pathExists(d.path + "/.git")) break;
-                subdir = std::string(baseNameOf(d.path)) + (subdir.empty() ? "" : "/" + subdir);
-                d.path = dirOf(d.path);
-                if (d.path == "/")
-                    throw MissingFlake("path '%s' is not a flake (because it does not reference a Git repository)", uri);
-            }
-        } else
-            d.path = canonPath(uri);
-        data = d;
-        for (auto & param : params) {
-            if (handleGitParams(param.first, param.second))
-                ;
-            else
-                throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri);
-        }
-    }
+    static std::regex pathUrlRegex(
+        "(" + pathRegex + "/?)"
+        + "(?:\\?(" + queryRegex + "))?"
+        + "(?:#(" + queryRegex + "))?",
+        std::regex::ECMAScript);
 
-    else
-        throw BadFlakeRef("'%s' is not a valid flake reference", uri);
-}
+    static std::regex flakeRegex(
+        "((" + flakeId + ")(?:/(?:" + refAndOrRevRegex + "))?)"
+        + "(?:#(" + queryRegex + "))?",
+        std::regex::ECMAScript);
 
-std::string FlakeRef::to_string() const
-{
-    std::string string;
-    bool first = true;
+    std::smatch match;
 
-    auto addParam =
-        [&](const std::string & name, std::string value) {
-            string += first ? '?' : '&';
-            first = false;
-            string += name;
-            string += '=';
-            string += value; // FIXME: escaping
+    /* Check if 'url' is a flake ID. This is an abbreviated syntax for
+       'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
+
+    if (std::regex_match(url, match, flakeRegex)) {
+        auto parsedURL = ParsedURL{
+            .url = url,
+            .base = "flake:" + std::string(match[1]),
+            .scheme = "flake",
+            .authority = "",
+            .path = match[1],
+            .fragment = percentDecode(std::string(match[6]))
         };
 
-    if (auto refData = std::get_if<FlakeRef::IsId>(&data)) {
-        string = refData->id;
-        if (ref) string += '/' + *ref;
-        if (rev) string += '/' + rev->gitRev();
+        return std::make_pair(
+            FlakeRef(inputFromURL(parsedURL), ""),
+            parsedURL.fragment);
     }
 
-    else if (auto refData = std::get_if<FlakeRef::IsPath>(&data)) {
-        string = refData->path;
-        if (ref) addParam("ref", *ref);
-        if (rev) addParam("rev", rev->gitRev());
-        if (subdir != "") addParam("dir", subdir);
-    }
+    /* Check if 'url' is a path (either absolute or relative to
+       'baseDir'). If so, search upward to the root of the repo
+       (i.e. the directory containing .git). */
+
+    else if (std::regex_match(url, match, pathUrlRegex)) {
+        std::string path = match[1];
+        if (!baseDir && !hasPrefix(path, "/"))
+            throw BadURL("flake reference '%s' is not an absolute path", url);
+        path = absPath(path, baseDir, true);
+
+        auto flakeRoot = path;
+        std::string subdir;
+
+        while (true) {
+            if (pathExists(flakeRoot + "/.git")) break;
+            subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
+            flakeRoot = dirOf(flakeRoot);
+            if (flakeRoot == "/")
+                throw BadURL("path '%s' is not a flake (because it does not reference a Git repository)", path);
+        }
 
-    else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data)) {
-        assert(!(ref && rev));
-        string = "github:" + refData->owner + "/" + refData->repo;
-        if (ref) { string += '/'; string += *ref; }
-        if (rev) { string += '/'; string += rev->gitRev(); }
-        if (subdir != "") addParam("dir", subdir);
-    }
+        auto base = std::string("git+file://") + flakeRoot;
 
-    else if (auto refData = std::get_if<FlakeRef::IsGit>(&data)) {
-        assert(!rev || ref);
-        string = refData->uri;
+        auto parsedURL = ParsedURL{
+            .url = base, // FIXME
+            .base = base,
+            .scheme = "git+file",
+            .authority = "",
+            .path = flakeRoot,
+            .query = decodeQuery(match[2]),
+            .fragment = percentDecode(std::string(match[3]))
+        };
 
-        if (ref) {
-            addParam("ref", *ref);
-            if (rev)
-                addParam("rev", rev->gitRev());
+        if (subdir != "") {
+            if (parsedURL.query.count("subdir"))
+                throw Error("flake URL '%s' has an inconsistent 'subdir' parameter", url);
+            parsedURL.query.insert_or_assign("subdir", subdir);
         }
 
-        if (subdir != "") addParam("dir", subdir);
+        return std::make_pair(
+            FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "subdir").value_or("")),
+            parsedURL.fragment);
     }
 
-    else abort();
-
-    assert(FlakeRef(string) == *this);
-
-    return string;
-}
-
-std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
-{
-    str << flakeRef.to_string();
-    return str;
-}
-
-bool FlakeRef::isImmutable() const
-{
-    return (bool) rev;
-}
-
-FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef.
-{
-    FlakeRef result(*this);
-    result.ref = std::nullopt;
-    result.rev = std::nullopt;
-    return result;
-}
-
-bool FlakeRef::contains(const FlakeRef & other) const
-{
-    if (!(data == other.data))
-        return false;
-
-    if (ref && ref != other.ref)
-        return false;
-
-    if (rev && rev != other.rev)
-        return false;
-
-    if (subdir != other.subdir)
-        return false;
-
-    return true;
+    else {
+        auto parsedURL = parseURL(url);
+        return std::make_pair(
+            FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "subdir").value_or("")),
+            parsedURL.fragment);
+    }
 }
 
-std::optional<FlakeRef> parseFlakeRef(
-    const std::string & uri, bool allowRelative)
+std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
+    const std::string & url, const std::optional<Path> & baseDir)
 {
     try {
-        return FlakeRef(uri, allowRelative);
-    } catch (BadFlakeRef & e) {
+        return parseFlakeRefWithFragment(url, baseDir);
+    } catch (Error & e) {
+        printError("FOO: %s", e.what());
         return {};
     }
 }