diff options
Diffstat (limited to 'src/libfetchers')
-rw-r--r-- | src/libfetchers/fetchers.cc | 11 | ||||
-rw-r--r-- | src/libfetchers/fetchers.hh | 16 | ||||
-rw-r--r-- | src/libfetchers/git.cc | 57 | ||||
-rw-r--r-- | src/libfetchers/github.cc | 21 | ||||
-rw-r--r-- | src/libfetchers/indirect.cc | 140 | ||||
-rw-r--r-- | src/libfetchers/mercurial.cc | 35 | ||||
-rw-r--r-- | src/libfetchers/path.cc | 24 | ||||
-rw-r--r-- | src/libfetchers/registry.cc | 222 | ||||
-rw-r--r-- | src/libfetchers/registry.hh | 65 | ||||
-rw-r--r-- | src/libfetchers/tree-info.cc | 46 | ||||
-rw-r--r-- | src/libfetchers/tree-info.hh | 4 |
11 files changed, 640 insertions, 1 deletions
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 94ac30e38..83268b4bf 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -72,4 +72,15 @@ std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) return {std::move(tree), input}; } +std::shared_ptr<const Input> Input::applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const +{ + if (ref) + throw Error("don't know how to apply '%s' to '%s'", *ref, to_string()); + if (rev) + throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string()); + return shared_from_this(); +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 59a58ae67..b75dcffa5 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -57,6 +57,22 @@ struct Input : std::enable_shared_from_this<Input> std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const; + virtual std::shared_ptr<const Input> applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const; + + virtual std::optional<Path> getSourcePath() const { return {}; } + + virtual void markChangedFile( + std::string_view file, + std::optional<std::string> commitMsg) const + { assert(false); } + + virtual void clone(const Path & destDir) const + { + throw Error("do not know how to clone input '%s'", to_string()); + } + private: virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7c18cf67f..210e29193 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -79,6 +79,63 @@ struct GitInput : Input return attrs; } + void clone(const Path & destDir) const override + { + auto [isLocal, actualUrl] = getActualUrl(); + + Strings args = {"clone"}; + + args.push_back(actualUrl); + + if (ref) { + args.push_back("--branch"); + args.push_back(*ref); + } + + if (rev) throw Error("cloning a specific revision is not implemented"); + + args.push_back(destDir); + + runProgram("git", true, args); + } + + std::shared_ptr<const Input> applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared<GitInput>(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + if (!res->ref && res->rev) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); + + return res; + } + + std::optional<Path> getSourcePath() const override + { + if (url.scheme == "file" && !ref && !rev) + return url.path; + return {}; + } + + void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + { + auto sourcePath = getSourcePath(); + assert(sourcePath); + + runProgram("git", true, + { "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) }); + + if (commitMsg) + runProgram("git", true, + { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); + } + std::pair<bool, std::string> getActualUrl() const { // Don't clone file:// URIs (but otherwise treat them the diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8675a5a66..c01917dcc 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -64,6 +64,13 @@ struct GitHubInput : Input return attrs; } + void clone(const Path & destDir) const override + { + std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); + input = input->applyOverrides(ref.value_or("master"), rev); + input->clone(destDir); + } + std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override { auto rev = this->rev; @@ -126,6 +133,20 @@ struct GitHubInput : Input return {std::move(tree), input}; } + + std::shared_ptr<const Input> applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared<GitHubInput>(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } }; struct GitHubInputScheme : InputScheme diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc new file mode 100644 index 000000000..380b69fe0 --- /dev/null +++ b/src/libfetchers/indirect.cc @@ -0,0 +1,140 @@ +#include "fetchers.hh" + +namespace nix::fetchers { + +std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct IndirectInput : Input +{ + std::string id; + std::optional<Hash> rev; + std::optional<std::string> ref; + + std::string type() const override { return "indirect"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast<const IndirectInput *>(&other); + return + other2 + && id == other2->id + && rev == other2->rev + && ref == other2->ref; + } + + bool isDirect() const override + { + return false; + } + + std::optional<std::string> getRef() const override { return ref; } + + std::optional<Hash> getRev() const override { return rev; } + + bool contains(const Input & other) const override + { + auto other2 = dynamic_cast<const IndirectInput *>(&other); + return + other2 + && id == other2->id + && (!ref || ref == other2->ref) + && (!rev || rev == other2->rev); + } + + ParsedURL toURL() const override + { + ParsedURL url; + url.scheme = "flake"; + url.path = id; + if (ref) { url.path += '/'; url.path += *ref; }; + if (rev) { url.path += '/'; url.path += rev->gitRev(); }; + return url; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("id", id); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::shared_ptr<const Input> applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared<IndirectInput>(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } + + std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + { + throw Error("indirect input '%s' cannot be fetched directly", to_string()); + } +}; + +struct IndirectInputScheme : InputScheme +{ + std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "flake") return nullptr; + + auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); + auto input = std::make_unique<IndirectInput>(); + + if (path.size() == 1) { + } else if (path.size() == 2) { + if (std::regex_match(path[1], revRegex)) + input->rev = Hash(path[1], htSHA1); + else if (std::regex_match(path[1], refRegex)) + input->ref = path[1]; + else + throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); + } else if (path.size() == 3) { + if (!std::regex_match(path[1], refRegex)) + throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); + input->ref = path[1]; + if (!std::regex_match(path[2], revRegex)) + throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); + input->rev = Hash(path[2], htSHA1); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + // FIXME: forbid query params? + + input->id = path[0]; + if (!std::regex_match(input->id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", input->id); + + return input; + } + + std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "id" && name != "ref" && name != "rev") + throw Error("unsupported indirect input attribute '%s'", name); + + auto input = std::make_unique<IndirectInput>(); + input->id = getStrAttr(attrs, "id"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); }); + +} diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 1d6571571..5abb00172 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -60,6 +60,41 @@ struct MercurialInput : Input return attrs; } + std::shared_ptr<const Input> applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared<MercurialInput>(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } + + std::optional<Path> getSourcePath() const + { + if (url.scheme == "file" && !ref && !rev) + return url.path; + return {}; + } + + void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + { + auto sourcePath = getSourcePath(); + assert(sourcePath); + + // FIXME: shut up if file is already tracked. + runProgram("hg", true, + { "add", *sourcePath + "/" + std::string(file) }); + + if (commitMsg) + runProgram("hg", true, + { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); + } + std::pair<bool, std::string> getActualUrl() const { bool isLocal = url.scheme == "file"; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index ba2cc192e..77fe87d59 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -32,7 +32,7 @@ struct PathInput : Input bool isImmutable() const override { - return (bool) narHash; + return narHash || rev; } ParsedURL toURL() const override @@ -56,9 +56,20 @@ struct PathInput : Input attrs.emplace("revCount", *revCount); if (lastModified) attrs.emplace("lastModified", *lastModified); + if (!rev && narHash) + attrs.emplace("narHash", narHash->to_string(SRI)); return attrs; } + std::optional<Path> getSourcePath() const override + { + return path; + } + + void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + { + } + std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override { auto input = std::make_shared<PathInput>(*this); @@ -74,6 +85,8 @@ struct PathInput : Input // FIXME: try to substitute storePath. storePath = store->addToStore("source", path); + input->narHash = store->queryPathInfo(*storePath)->narHash; + return { Tree { @@ -99,6 +112,9 @@ struct PathInputScheme : InputScheme auto input = std::make_unique<PathInput>(); input->path = url.path; + if (url.authority && *url.authority != "") + throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); + for (auto & [name, value] : url.query) if (name == "rev") input->rev = Hash(value, htSHA1); @@ -114,6 +130,9 @@ struct PathInputScheme : InputScheme throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); input->lastModified = lastModified; } + else if (name == "narHash") + // FIXME: require SRI hash. + input->narHash = Hash(value); else throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); @@ -134,6 +153,9 @@ struct PathInputScheme : InputScheme input->revCount = getIntAttr(attrs, "revCount"); else if (name == "lastModified") input->lastModified = getIntAttr(attrs, "lastModified"); + else if (name == "narHash") + // FIXME: require SRI hash. + input->narHash = Hash(getStrAttr(attrs, "narHash")); else if (name == "type" || name == "path") ; else diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc new file mode 100644 index 000000000..77d3b3378 --- /dev/null +++ b/src/libfetchers/registry.cc @@ -0,0 +1,222 @@ +#include "registry.hh" +#include "fetchers.hh" +#include "util.hh" +#include "globals.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +namespace nix::fetchers { + +std::shared_ptr<Registry> Registry::read( + const Path & path, RegistryType type) +{ + auto registry = std::make_shared<Registry>(type); + + if (!pathExists(path)) + return std::make_shared<Registry>(type); + + try { + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + + // FIXME: remove soon + if (version == 1) { + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + auto url = i->value("url", i->value("uri", "")); + if (url.empty()) + throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", + path, i.key()); + registry->entries.push_back( + {inputFromURL(i.key()), inputFromURL(url), {}}); + } + } + + else if (version == 2) { + for (auto & i : json["flakes"]) { + auto toAttrs = jsonToAttrs(i["to"]); + Attrs extraAttrs; + auto j = toAttrs.find("dir"); + if (j != toAttrs.end()) { + extraAttrs.insert(*j); + toAttrs.erase(j); + } + auto exact = i.find("exact"); + registry->entries.push_back( + Entry { + .from = inputFromAttrs(jsonToAttrs(i["from"])), + .to = inputFromAttrs(toAttrs), + .extraAttrs = extraAttrs, + .exact = exact != i.end() && exact.value() + }); + } + } + + else + throw Error("flake registry '%s' has unsupported version %d", path, version); + + } catch (nlohmann::json::exception & e) { + warn("cannot parse flake registry '%s': %s", path, e.what()); + } catch (Error & e) { + warn("cannot read flake registry '%s': %s", path, e.what()); + } + + return registry; +} + +void Registry::write(const Path & path) +{ + nlohmann::json arr; + for (auto & entry : entries) { + nlohmann::json obj; + obj["from"] = attrsToJson(entry.from->toAttrs()); + obj["to"] = attrsToJson(entry.to->toAttrs()); + if (!entry.extraAttrs.empty()) + obj["to"].update(attrsToJson(entry.extraAttrs)); + if (entry.exact) + obj["exact"] = true; + arr.emplace_back(std::move(obj)); + } + + nlohmann::json json; + json["version"] = 2; + json["flakes"] = std::move(arr); + + createDirs(dirOf(path)); + writeFile(path, json.dump(2)); +} + +void Registry::add( + const std::shared_ptr<const Input> & from, + const std::shared_ptr<const Input> & to, + const Attrs & extraAttrs) +{ + entries.emplace_back( + Entry { + .from = from, + .to = to, + .extraAttrs = extraAttrs + }); +} + +void Registry::remove(const std::shared_ptr<const Input> & input) +{ + // FIXME: use C++20 std::erase. + for (auto i = entries.begin(); i != entries.end(); ) + if (*i->from == *input) + i = entries.erase(i); + else + ++i; +} + +static Path getSystemRegistryPath() +{ + return settings.nixConfDir + "/registry.json"; +} + +static std::shared_ptr<Registry> getSystemRegistry() +{ + static auto systemRegistry = + Registry::read(getSystemRegistryPath(), Registry::System); + return systemRegistry; +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr<Registry> getUserRegistry() +{ + static auto userRegistry = + Registry::read(getUserRegistryPath(), Registry::User); + return userRegistry; +} + +static std::shared_ptr<Registry> flagRegistry = + std::make_shared<Registry>(Registry::Flag); + +std::shared_ptr<Registry> getFlagRegistry() +{ + return flagRegistry; +} + +void overrideRegistry( + const std::shared_ptr<const Input> & from, + const std::shared_ptr<const Input> & to, + const Attrs & extraAttrs) +{ + flagRegistry->add(from, to, extraAttrs); +} + +static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store) +{ + static auto reg = [&]() { + auto path = settings.flakeRegistry; + + if (!hasPrefix(path, "/")) { + auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) + store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true); + path = store->toRealPath(storePath); + } + + return Registry::read(path, Registry::Global); + }(); + + return reg; +} + +Registries getRegistries(ref<Store> store) +{ + Registries registries; + registries.push_back(getFlagRegistry()); + registries.push_back(getUserRegistry()); + registries.push_back(getSystemRegistry()); + registries.push_back(getGlobalRegistry(store)); + return registries; +} + +std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries( + ref<Store> store, + std::shared_ptr<const Input> input) +{ + Attrs extraAttrs; + int n = 0; + + restart: + + n++; + if (n > 100) throw Error("cycle detected in flake registr for '%s'", input); + + for (auto & registry : getRegistries(store)) { + // FIXME: O(n) + for (auto & entry : registry->entries) { + if (entry.exact) { + if (*entry.from == *input) { + input = entry.to; + extraAttrs = entry.extraAttrs; + goto restart; + } + } else { + if (entry.from->contains(*input)) { + input = entry.to->applyOverrides( + !entry.from->getRef() && input->getRef() ? input->getRef() : std::optional<std::string>(), + !entry.from->getRev() && input->getRev() ? input->getRev() : std::optional<Hash>()); + extraAttrs = entry.extraAttrs; + goto restart; + } + } + } + } + + if (!input->isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input->to_string()); + + return {input, extraAttrs}; +} + +} diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh new file mode 100644 index 000000000..c3ce948a8 --- /dev/null +++ b/src/libfetchers/registry.hh @@ -0,0 +1,65 @@ +#pragma once + +#include "types.hh" +#include "fetchers.hh" + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Registry +{ + enum RegistryType { + Flag = 0, + User = 1, + System = 2, + Global = 3, + }; + + RegistryType type; + + struct Entry + { + std::shared_ptr<const Input> from; + std::shared_ptr<const Input> to; + Attrs extraAttrs; + bool exact = false; + }; + + std::vector<Entry> entries; + + Registry(RegistryType type) + : type(type) + { } + + static std::shared_ptr<Registry> read( + const Path & path, RegistryType type); + + void write(const Path & path); + + void add( + const std::shared_ptr<const Input> & from, + const std::shared_ptr<const Input> & to, + const Attrs & extraAttrs); + + void remove(const std::shared_ptr<const Input> & input); +}; + +typedef std::vector<std::shared_ptr<Registry>> Registries; + +std::shared_ptr<Registry> getUserRegistry(); + +Path getUserRegistryPath(); + +Registries getRegistries(ref<Store> store); + +void overrideRegistry( + const std::shared_ptr<const Input> & from, + const std::shared_ptr<const Input> & to, + const Attrs & extraAttrs); + +std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries( + ref<Store> store, + std::shared_ptr<const Input> input); + +} diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index 5788e94a1..42a19cbc8 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -11,4 +11,50 @@ StorePath TreeInfo::computeStorePath(Store & store) const return store.makeFixedOutputPath(true, narHash, "source"); } +TreeInfo TreeInfo::fromJson(const nlohmann::json & json) +{ + TreeInfo info; + + auto i = json.find("info"); + if (i != json.end()) { + const nlohmann::json & i2(*i); + + auto j = i2.find("narHash"); + if (j != i2.end()) + info.narHash = Hash((std::string) *j); + else + throw Error("attribute 'narHash' missing in lock file"); + + j = i2.find("revCount"); + if (j != i2.end()) + info.revCount = *j; + + j = i2.find("lastModified"); + if (j != i2.end()) + info.lastModified = *j; + + return info; + } + + i = json.find("narHash"); + if (i != json.end()) { + info.narHash = Hash((std::string) *i); + return info; + } + + throw Error("attribute 'info' missing in lock file"); +} + +nlohmann::json TreeInfo::toJson() const +{ + nlohmann::json json; + assert(narHash); + json["narHash"] = narHash.to_string(SRI); + if (revCount) + json["revCount"] = *revCount; + if (lastModified) + json["lastModified"] = *lastModified; + return json; +} + } diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh index 2c7347281..3b62151c6 100644 --- a/src/libfetchers/tree-info.hh +++ b/src/libfetchers/tree-info.hh @@ -24,6 +24,10 @@ struct TreeInfo } StorePath computeStorePath(Store & store) const; + + static TreeInfo fromJson(const nlohmann::json & json); + + nlohmann::json toJson() const; }; } |