Pluggable fetchers

Flakes are now fetched using an extensible mechanism. Also lots of
other flake cleanups.

4 files changed, 94 insertions, 119 deletions

diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 22e994e58..5bec5903f 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -9,6 +9,8 @@
 #include "store-api.hh"
 #include "derivations.hh"
 #include "attr-path.hh"
+#include "fetchers/fetchers.hh"
+#include "fetchers/registry.hh"
 
 #include <nlohmann/json.hpp>
 #include <queue>
@@ -30,10 +32,7 @@ public:
 
     FlakeRef getFlakeRef()
     {
-        if (flakeUrl.find('/') != std::string::npos || flakeUrl == ".")
-            return FlakeRef(flakeUrl, true);
-        else
-            return FlakeRef(flakeUrl);
+        return parseFlakeRef(flakeUrl, absPath(".")); //FIXME
     }
 
     Flake getFlake()
@@ -57,63 +56,54 @@ struct CmdFlakeList : EvalCommand
 
     void run(nix::ref<nix::Store> store) override
     {
-        auto registries = getEvalState()->getFlakeRegistries();
-
-        stopProgressBar();
+        using namespace fetchers;
 
-        for (auto & entry : registries[FLAG_REGISTRY]->entries)
-            std::cout << entry.first.to_string() << " flags " << entry.second.to_string() << "\n";
+        auto registries = getRegistries(store);
 
-        for (auto & entry : registries[USER_REGISTRY]->entries)
-            std::cout << entry.first.to_string() << " user " << entry.second.to_string() << "\n";
+        stopProgressBar();
 
-        for (auto & entry : registries[GLOBAL_REGISTRY]->entries)
-            std::cout << entry.first.to_string() << " global " << entry.second.to_string() << "\n";
+        for (auto & registry : registries) {
+            for (auto & entry : registry->entries) {
+                // FIXME: format nicely
+                std::cout << fmt("%s %s %s\n",
+                    registry->type == Registry::Flag ? "flags " :
+                    registry->type == Registry::User ? "user  " :
+                    "global",
+                    entry.first->to_string(),
+                    entry.second->to_string());
+            }
+        }
     }
 };
 
-static void printSourceInfo(const SourceInfo & sourceInfo)
-{
-    std::cout << fmt("URL:           %s\n", sourceInfo.resolvedRef.to_string());
-    if (sourceInfo.resolvedRef.ref)
-        std::cout << fmt("Branch:        %s\n",*sourceInfo.resolvedRef.ref);
-    if (sourceInfo.resolvedRef.rev)
-        std::cout << fmt("Revision:      %s\n", sourceInfo.resolvedRef.rev->to_string(Base16, false));
-    if (sourceInfo.revCount)
-        std::cout << fmt("Revisions:     %s\n", *sourceInfo.revCount);
-    if (sourceInfo.lastModified)
-        std::cout << fmt("Last modified: %s\n",
-            std::put_time(std::localtime(&*sourceInfo.lastModified), "%F %T"));
-    std::cout << fmt("Path:          %s\n", sourceInfo.storePath);
-}
-
-static void sourceInfoToJson(const SourceInfo & sourceInfo, nlohmann::json & j)
-{
-    j["url"] = sourceInfo.resolvedRef.to_string();
-    if (sourceInfo.resolvedRef.ref)
-        j["branch"] = *sourceInfo.resolvedRef.ref;
-    if (sourceInfo.resolvedRef.rev)
-        j["revision"] = sourceInfo.resolvedRef.rev->to_string(Base16, false);
-    if (sourceInfo.revCount)
-        j["revCount"] = *sourceInfo.revCount;
-    if (sourceInfo.lastModified)
-        j["lastModified"] = *sourceInfo.lastModified;
-    j["path"] = sourceInfo.storePath;
-}
-
-static void printFlakeInfo(const Flake & flake)
+static void printFlakeInfo(const Store & store, const Flake & flake)
 {
-    std::cout << fmt("Description:   %s\n", flake.description);
+    std::cout << fmt("URL:           %s\n", flake.resolvedRef.input->to_string());
     std::cout << fmt("Edition:       %s\n", flake.edition);
-    printSourceInfo(flake.sourceInfo);
+    std::cout << fmt("Description:   %s\n", flake.description);
+    std::cout << fmt("Path:          %s\n", store.printStorePath(flake.sourceInfo->storePath));
+    if (flake.sourceInfo->rev)
+        std::cout << fmt("Revision:      %s\n", flake.sourceInfo->rev->to_string(Base16, false));
+    if (flake.sourceInfo->revCount)
+        std::cout << fmt("Revisions:     %s\n", *flake.sourceInfo->revCount);
+    if (flake.sourceInfo->lastModified)
+        std::cout << fmt("Last modified: %s\n",
+            std::put_time(std::localtime(&*flake.sourceInfo->lastModified), "%F %T"));
 }
 
-static nlohmann::json flakeToJson(const Flake & flake)
+static nlohmann::json flakeToJson(const Store & store, const Flake & flake)
 {
     nlohmann::json j;
     j["description"] = flake.description;
     j["edition"] = flake.edition;
-    sourceInfoToJson(flake.sourceInfo, j);
+    j["url"] = flake.resolvedRef.input->to_string();
+    if (flake.sourceInfo->rev)
+        j["revision"] = flake.sourceInfo->rev->to_string(Base16, false);
+    if (flake.sourceInfo->revCount)
+        j["revCount"] = *flake.sourceInfo->revCount;
+    if (flake.sourceInfo->lastModified)
+        j["lastModified"] = *flake.sourceInfo->lastModified;
+    j["path"] = store.printStorePath(flake.sourceInfo->storePath);
     return j;
 }
 
@@ -140,7 +130,7 @@ struct CmdFlakeDeps : FlakeCommand
             todo.pop();
 
             for (auto & info : resFlake.flakeDeps) {
-                printFlakeInfo(info.second.flake);
+                printFlakeInfo(*store, info.second.flake);
                 todo.push(info.second);
             }
         }
@@ -161,10 +151,12 @@ struct CmdFlakeUpdate : FlakeCommand
 
         auto flakeRef = getFlakeRef();
 
+#if 0
         if (std::get_if<FlakeRef::IsPath>(&flakeRef.data))
             updateLockFile(*evalState, flakeRef, true);
         else
             throw Error("cannot update lockfile of flake '%s'", flakeRef);
+#endif
     }
 };
 
@@ -195,7 +187,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON
             auto state = getEvalState();
             auto flake = resolveFlake();
 
-            auto json = flakeToJson(flake.flake);
+            auto json = flakeToJson(*store, flake.flake);
 
             auto vFlake = state->allocValue();
             flake::callFlake(*state, flake, *vFlake);
@@ -222,7 +214,7 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON
         } else {
             auto flake = getFlake();
             stopProgressBar();
-            printFlakeInfo(flake);
+            printFlakeInfo(*store, flake);
         }
     }
 };
@@ -495,8 +487,7 @@ struct CmdFlakeCheck : FlakeCommand, MixJSON
 
 struct CmdFlakeAdd : MixEvalArgs, Command
 {
-    FlakeUri alias;
-    FlakeUri url;
+    std::string fromUrl, toUrl;
 
     std::string description() override
     {
@@ -505,24 +496,24 @@ struct CmdFlakeAdd : MixEvalArgs, Command
 
     CmdFlakeAdd()
     {
-        expectArg("alias", &alias);
-        expectArg("flake-url", &url);
+        expectArg("from-url", &fromUrl);
+        expectArg("to-url", &toUrl);
     }
 
     void run() override
     {
-        FlakeRef aliasRef(alias);
-        Path userRegistryPath = getUserRegistryPath();
-        auto userRegistry = readRegistry(userRegistryPath);
-        userRegistry->entries.erase(aliasRef);
-        userRegistry->entries.insert_or_assign(aliasRef, FlakeRef(url));
-        writeRegistry(*userRegistry, userRegistryPath);
+        auto fromRef = parseFlakeRef(fromUrl);
+        auto toRef = parseFlakeRef(toUrl);
+        auto userRegistry = fetchers::getUserRegistry();
+        userRegistry->remove(fromRef.input);
+        userRegistry->add(fromRef.input, toRef.input);
+        userRegistry->write(fetchers::getUserRegistryPath());
     }
 };
 
 struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command
 {
-    FlakeUri alias;
+    std::string url;
 
     std::string description() override
     {
@@ -531,52 +522,38 @@ struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command
 
     CmdFlakeRemove()
     {
-        expectArg("alias", &alias);
+        expectArg("url", &url);
     }
 
     void run() override
     {
-        Path userRegistryPath = getUserRegistryPath();
-        auto userRegistry = readRegistry(userRegistryPath);
-        userRegistry->entries.erase(FlakeRef(alias));
-        writeRegistry(*userRegistry, userRegistryPath);
+        auto userRegistry = fetchers::getUserRegistry();
+        userRegistry->remove(parseFlakeRef(url).input);
+        userRegistry->write(fetchers::getUserRegistryPath());
     }
 };
 
 struct CmdFlakePin : virtual Args, EvalCommand
 {
-    FlakeUri alias;
+    std::string url;
 
     std::string description() override
     {
-        return "pin flake require in user flake registry";
+        return "pin a flake to its current version in user flake registry";
     }
 
     CmdFlakePin()
     {
-        expectArg("alias", &alias);
+        expectArg("url", &url);
     }
 
     void run(nix::ref<nix::Store> store) override
     {
-        auto evalState = getEvalState();
-
-        Path userRegistryPath = getUserRegistryPath();
-        FlakeRegistry userRegistry = *readRegistry(userRegistryPath);
-        auto it = userRegistry.entries.find(FlakeRef(alias));
-        if (it != userRegistry.entries.end()) {
-            it->second = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef;
-            writeRegistry(userRegistry, userRegistryPath);
-        } else {
-            std::shared_ptr<FlakeRegistry> globalReg = evalState->getGlobalFlakeRegistry();
-            it = globalReg->entries.find(FlakeRef(alias));
-            if (it != globalReg->entries.end()) {
-                auto newRef = getFlake(*evalState, it->second, true).sourceInfo.resolvedRef;
-                userRegistry.entries.insert_or_assign(alias, newRef);
-                writeRegistry(userRegistry, userRegistryPath);
-            } else
-                throw Error("the flake alias '%s' does not exist in the user or global registry", alias);
-        }
+        auto ref = parseFlakeRef(url);
+        auto userRegistry = fetchers::getUserRegistry();
+        userRegistry->remove(ref.input);
+        auto [tree, resolved] = ref.resolve(store).input->fetchTree(store);
+        userRegistry->add(ref.input, resolved);
     }
 };
 
@@ -616,15 +593,20 @@ struct CmdFlakeClone : FlakeCommand
 
     CmdFlakeClone()
     {
-        expectArg("dest-dir", &destDir, true);
+        mkFlag()
+            .shortName('f')
+            .longName("dest")
+            .label("path")
+            .description("destination path")
+            .dest(&destDir);
     }
 
     void run(nix::ref<nix::Store> store) override
     {
-        auto evalState = getEvalState();
+        if (destDir.empty())
+            throw Error("missing flag '--dest'");
 
-        Registries registries = evalState->getFlakeRegistries();
-        gitCloneFlake(getFlakeRef().to_string(), *evalState, registries, destDir);
+        getFlakeRef().resolve(store).input->clone(destDir);
     }
 };
 
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index 86b9fbdb9..74cd85380 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -10,6 +10,7 @@
 #include "shared.hh"
 #include "flake/flake.hh"
 #include "flake/eval-cache.hh"
+#include "fetchers/parse.hh"
 
 #include <regex>
 #include <queue>
@@ -80,10 +81,8 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
 
 ref<EvalState> EvalCommand::getEvalState()
 {
-    if (!evalState) {
+    if (!evalState)
         evalState = std::make_shared<EvalState>(searchPath, getStore());
-        evalState->addRegistryOverrides(registryOverrides);
-    }
     return ref<EvalState>(evalState);
 }
 
@@ -243,6 +242,7 @@ void makeFlakeClosureGCRoot(Store & store,
     const FlakeRef & origFlakeRef,
     const flake::ResolvedFlake & resFlake)
 {
+#if 0
     if (std::get_if<FlakeRef::IsPath>(&origFlakeRef.data)) return;
 
     /* Get the store paths of all non-local flakes. */
@@ -285,6 +285,7 @@ void makeFlakeClosureGCRoot(Store & store,
     debug("writing GC root '%s' for flake closure of '%s'", symlink, origFlakeRef);
     replaceSymlink(store.printStorePath(closurePath), symlink);
     store.addIndirectRoot(symlink);
+#endif
 }
 
 std::vector<std::string> InstallableFlake::getActualAttrPaths()
@@ -334,7 +335,7 @@ std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> InstallableFlake
         auto drv = evalCache.getDerivation(fingerprint, attrPath);
         if (drv) {
             if (state->store->isValidPath(drv->drvPath))
-                return {attrPath, resFlake.flake.sourceInfo.resolvedRef, std::move(*drv)};
+                return {attrPath, resFlake.flake.resolvedRef, std::move(*drv)};
         }
 
         if (!vOutputs)
@@ -356,7 +357,7 @@ std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> InstallableFlake
 
             evalCache.addDerivation(fingerprint, attrPath, drv);
 
-            return {attrPath, resFlake.flake.sourceInfo.resolvedRef, std::move(drv)};
+            return {attrPath, resFlake.flake.resolvedRef, std::move(drv)};
         } catch (AttrPathNotFound & e) {
         }
     }
@@ -440,27 +441,23 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
             if (hasPrefix(s, "nixpkgs.")) {
                 bool static warned;
                 warnOnce(warned, "the syntax 'nixpkgs.<attr>' is deprecated; use 'nixpkgs:<attr>' instead");
-                result.push_back(std::make_shared<InstallableFlake>(*this, FlakeRef("nixpkgs"),
-                        Strings{"legacyPackages." + settings.thisSystem.get() + "." + std::string(s, 8)}));
+                result.push_back(std::make_shared<InstallableFlake>(*this, parseFlakeRef("flake:nixpkgs"),
+                        Strings{"legacyPackages." + settings.thisSystem.get() + "." + std::string(s, 8)}, Strings{}));
             }
 
-            else if ((hash = s.rfind('#')) != std::string::npos)
-                result.push_back(std::make_shared<InstallableFlake>(
-                        *this,
-                        FlakeRef(std::string(s, 0, hash), true),
-                        std::string(s, hash + 1),
-                        getDefaultFlakeAttrPathPrefixes()));
-
             else {
-                try {
-                    auto flakeRef = FlakeRef(s, true);
+                auto res = maybeParseFlakeRefWithFragment(s, absPath("."));
+                if (res) {
+                    auto &[flakeRef, fragment] = *res;
                     result.push_back(std::make_shared<InstallableFlake>(
-                            *this, std::move(flakeRef), getDefaultFlakeAttrPaths()));
-                } catch (...) {
+                            *this, std::move(flakeRef),
+                            fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment},
+                            getDefaultFlakeAttrPathPrefixes()));
+                } else {
                     if (s.find('/') != std::string::npos && (storePath = follow(s)))
                         result.push_back(std::make_shared<InstallableStorePath>(store, store->printStorePath(*storePath)));
                     else
-                        throw;
+                        throw Error("unrecognized argument '%s'", s);
                 }
             }
         }
diff --git a/src/nix/installables.hh b/src/nix/installables.hh
index 22e4b38f9..340c2c9da 100644
--- a/src/nix/installables.hh
+++ b/src/nix/installables.hh
@@ -75,13 +75,9 @@ struct InstallableFlake : InstallableValue
     Strings attrPaths;
     Strings prefixes;
 
-    InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, Strings attrPaths)
-        : InstallableValue(cmd), flakeRef(flakeRef), attrPaths(std::move(attrPaths))
-    { }
-
     InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef,
-        std::string attrPath, Strings && prefixes)
-        : InstallableValue(cmd), flakeRef(flakeRef), attrPaths{attrPath},
+        Strings && attrPaths, Strings && prefixes)
+        : InstallableValue(cmd), flakeRef(flakeRef), attrPaths(attrPaths),
           prefixes(prefixes)
     { }
 
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 8ff0e4dd9..c94d92567 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -54,8 +54,8 @@ struct ProfileManifest
                 element.active = e["active"];
                 if (e.value("uri", "") != "") {
                     element.source = ProfileElementSource{
-                        FlakeRef(e["originalUri"]),
-                        FlakeRef(e["uri"]),
+                        parseFlakeRef(e["originalUri"]),
+                        parseFlakeRef(e["uri"]),
                         e["attrPath"]
                     };
                 }
@@ -336,7 +336,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
                 Activity act(*logger, lvlChatty, actUnknown,
                     fmt("checking '%s' for updates", element.source->attrPath));
 
-                InstallableFlake installable(*this, FlakeRef(element.source->originalRef), {element.source->attrPath});
+                InstallableFlake installable(*this, FlakeRef(element.source->originalRef), {element.source->attrPath}, {});
 
                 auto [attrPath, resolvedRef, drv] = installable.toDerivation();