aboutsummaryrefslogtreecommitdiff
path: root/src/libfetchers
diff options
context:
space:
mode:
Diffstat (limited to 'src/libfetchers')
-rw-r--r--src/libfetchers/attrs.cc14
-rw-r--r--src/libfetchers/attrs.hh11
-rw-r--r--src/libfetchers/fetchers.cc258
-rw-r--r--src/libfetchers/fetchers.hh93
-rw-r--r--src/libfetchers/git.cc303
-rw-r--r--src/libfetchers/github.cc304
-rw-r--r--src/libfetchers/indirect.cc104
-rw-r--r--src/libfetchers/mercurial.cc224
-rw-r--r--src/libfetchers/path.cc165
-rw-r--r--src/libfetchers/registry.cc212
-rw-r--r--src/libfetchers/registry.hh64
-rw-r--r--src/libfetchers/tarball.cc142
-rw-r--r--src/libfetchers/tree-info.cc14
-rw-r--r--src/libfetchers/tree-info.hh29
14 files changed, 1249 insertions, 688 deletions
diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc
index feb0a6085..1e59faa73 100644
--- a/src/libfetchers/attrs.cc
+++ b/src/libfetchers/attrs.cc
@@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
- if (auto v = std::get_if<int64_t>(&attr.second)) {
+ if (auto v = std::get_if<uint64_t>(&attr.second)) {
json[attr.first] = *v;
} else if (auto v = std::get_if<std::string>(&attr.second)) {
json[attr.first] = *v;
@@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name)
return *s;
}
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
+std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
- if (auto v = std::get_if<int64_t>(&i->second))
+ if (auto v = std::get_if<uint64_t>(&i->second))
return *v;
throw Error("input attribute '%s' is not an integer", name);
}
-int64_t getIntAttr(const Attrs & attrs, const std::string & name)
+uint64_t getIntAttr(const Attrs & attrs, const std::string & name)
{
auto s = maybeGetIntAttr(attrs, name);
if (!s)
@@ -76,8 +76,8 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na
{
auto i = attrs.find(name);
if (i == attrs.end()) return {};
- if (auto v = std::get_if<int64_t>(&i->second))
- return *v;
+ if (auto v = std::get_if<Explicit<bool>>(&i->second))
+ return v->t;
throw Error("input attribute '%s' is not a Boolean", name);
}
@@ -93,7 +93,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
{
std::map<std::string, std::string> query;
for (auto & attr : attrs) {
- if (auto v = std::get_if<int64_t>(&attr.second)) {
+ if (auto v = std::get_if<uint64_t>(&attr.second)) {
query.insert_or_assign(attr.first, fmt("%d", *v));
} else if (auto v = std::get_if<std::string>(&attr.second)) {
query.insert_or_assign(attr.first, *v);
diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index d6e0ae000..4b4630c80 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -13,9 +13,14 @@ namespace nix::fetchers {
template<typename T>
struct Explicit {
T t;
+
+ bool operator ==(const Explicit<T> & other) const
+ {
+ return t == other.t;
+ }
};
-typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
+typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs jsonToAttrs(const nlohmann::json & json);
@@ -26,9 +31,9 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
std::string getStrAttr(const Attrs & attrs, const std::string & name);
-std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
+std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
-int64_t getIntAttr(const Attrs & attrs, const std::string & name);
+uint64_t getIntAttr(const Attrs & attrs, const std::string & name);
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 9174c3de4..c3ee9bf43 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -5,71 +5,265 @@
namespace nix::fetchers {
-std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
+std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr;
-void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
+void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
{
- if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
+ if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>();
inputSchemes->push_back(std::move(inputScheme));
}
-std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
+Input Input::fromURL(const std::string & url)
+{
+ return fromURL(parseURL(url));
+}
+
+static void fixupInput(Input & input)
+{
+ // Check common attributes.
+ input.getType();
+ input.getRef();
+ if (input.getRev())
+ input.immutable = true;
+ input.getRevCount();
+ input.getLastModified();
+ if (input.getNarHash())
+ input.immutable = true;
+}
+
+Input Input::fromURL(const ParsedURL & url)
{
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromURL(url);
- if (res) return res;
+ if (res) {
+ res->scheme = inputScheme;
+ fixupInput(*res);
+ return std::move(*res);
+ }
}
- throw Error("input '%s' is unsupported", url.url);
-}
-std::unique_ptr<Input> inputFromURL(const std::string & url)
-{
- return inputFromURL(parseURL(url));
+ throw Error("input '%s' is unsupported", url.url);
}
-std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
+Input Input::fromAttrs(Attrs && attrs)
{
- auto attrs2(attrs);
- attrs2.erase("narHash");
for (auto & inputScheme : *inputSchemes) {
- auto res = inputScheme->inputFromAttrs(attrs2);
+ auto res = inputScheme->inputFromAttrs(attrs);
if (res) {
- if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
- // FIXME: require SRI hash.
- res->narHash = newHashAllowEmpty(*narHash, {});
- return res;
+ res->scheme = inputScheme;
+ fixupInput(*res);
+ return std::move(*res);
}
}
- throw Error("input '%s' is unsupported", attrsToJson(attrs));
+
+ Input input;
+ input.attrs = attrs;
+ fixupInput(input);
+ return input;
+}
+
+ParsedURL Input::toURL() const
+{
+ if (!scheme)
+ throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
+ return scheme->toURL(*this);
+}
+
+std::string Input::to_string() const
+{
+ return toURL().to_string();
}
Attrs Input::toAttrs() const
{
- auto attrs = toAttrsInternal();
- if (narHash)
- attrs.emplace("narHash", narHash->to_string(SRI, true));
- attrs.emplace("type", type());
return attrs;
}
-std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
+bool Input::hasAllInfo() const
{
- auto [tree, input] = fetchTreeInternal(store);
+ return getNarHash() && scheme && scheme->hasAllInfo(*this);
+}
+
+bool Input::operator ==(const Input & other) const
+{
+ return attrs == other.attrs;
+}
+
+bool Input::contains(const Input & other) const
+{
+ if (*this == other) return true;
+ auto other2(other);
+ other2.attrs.erase("ref");
+ other2.attrs.erase("rev");
+ if (*this == other2) return true;
+ return false;
+}
+
+std::pair<Tree, Input> Input::fetch(ref<Store> store) const
+{
+ if (!scheme)
+ throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
+
+ /* The tree may already be in the Nix store, or it could be
+ substituted (which is often faster than fetching from the
+ original source). So check that. */
+ if (hasAllInfo()) {
+ try {
+ auto storePath = computeStorePath(*store);
+
+ store->ensurePath(storePath);
+
+ debug("using substituted/cached input '%s' in '%s'",
+ to_string(), store->printStorePath(storePath));
+
+ auto actualPath = store->toRealPath(storePath);
+
+ return {fetchers::Tree(std::move(actualPath), std::move(storePath)), *this};
+ } catch (Error & e) {
+ debug("substitution of input '%s' failed: %s", to_string(), e.what());
+ }
+ }
+
+ auto [tree, input] = scheme->fetch(store, *this);
if (tree.actualPath == "")
tree.actualPath = store->toRealPath(tree.storePath);
- if (!tree.info.narHash)
- tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
+ auto narHash = store->queryPathInfo(tree.storePath)->narHash;
+ input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true));
+
+ if (auto prevNarHash = getNarHash()) {
+ if (narHash != *prevNarHash)
+ throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
+ to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
+ }
- if (input->narHash)
- assert(input->narHash == tree.info.narHash);
+ if (auto prevLastModified = getLastModified()) {
+ if (input.getLastModified() != prevLastModified)
+ throw Error("'lastModified' attribute mismatch in input '%s', expected %d",
+ input.to_string(), *prevLastModified);
+ }
- if (narHash && narHash != input->narHash)
- throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
- to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true));
+ if (auto prevRevCount = getRevCount()) {
+ if (input.getRevCount() != prevRevCount)
+ throw Error("'revCount' attribute mismatch in input '%s', expected %d",
+ input.to_string(), *prevRevCount);
+ }
+
+ input.immutable = true;
+
+ assert(input.hasAllInfo());
return {std::move(tree), input};
}
+Input Input::applyOverrides(
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) const
+{
+ if (!scheme) return *this;
+ return scheme->applyOverrides(*this, ref, rev);
+}
+
+void Input::clone(const Path & destDir) const
+{
+ assert(scheme);
+ scheme->clone(*this, destDir);
+}
+
+std::optional<Path> Input::getSourcePath() const
+{
+ assert(scheme);
+ return scheme->getSourcePath(*this);
+}
+
+void Input::markChangedFile(
+ std::string_view file,
+ std::optional<std::string> commitMsg) const
+{
+ assert(scheme);
+ return scheme->markChangedFile(*this, file, commitMsg);
+}
+
+StorePath Input::computeStorePath(Store & store) const
+{
+ auto narHash = getNarHash();
+ if (!narHash)
+ throw Error("cannot compute store path for mutable input '%s'", to_string());
+ return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
+}
+
+std::string Input::getType() const
+{
+ return getStrAttr(attrs, "type");
+}
+
+std::optional<Hash> Input::getNarHash() const
+{
+ if (auto s = maybeGetStrAttr(attrs, "narHash"))
+ // FIXME: require SRI hash.
+ return newHashAllowEmpty(*s, htSHA256);
+ return {};
+}
+
+std::optional<std::string> Input::getRef() const
+{
+ if (auto s = maybeGetStrAttr(attrs, "ref"))
+ return *s;
+ return {};
+}
+
+std::optional<Hash> Input::getRev() const
+{
+ if (auto s = maybeGetStrAttr(attrs, "rev"))
+ return Hash(*s, htSHA1);
+ return {};
+}
+
+std::optional<uint64_t> Input::getRevCount() const
+{
+ if (auto n = maybeGetIntAttr(attrs, "revCount"))
+ return *n;
+ return {};
+}
+
+std::optional<time_t> Input::getLastModified() const
+{
+ if (auto n = maybeGetIntAttr(attrs, "lastModified"))
+ return *n;
+ return {};
+}
+
+ParsedURL InputScheme::toURL(const Input & input)
+{
+ throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
+}
+
+Input InputScheme::applyOverrides(
+ const Input & input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev)
+{
+ if (ref)
+ throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref);
+ if (rev)
+ throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev());
+ return input;
+}
+
+std::optional<Path> InputScheme::getSourcePath(const Input & input)
+{
+ return {};
+}
+
+void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
+{
+ assert(false);
+}
+
+void InputScheme::clone(const Input & input, const Path & destDir)
+{
+ throw Error("do not know how to clone input '%s'", input.to_string());
+}
+
}
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 59a58ae67..be71b786b 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -3,7 +3,6 @@
#include "types.hh"
#include "hash.hh"
#include "path.hh"
-#include "tree-info.hh"
#include "attrs.hh"
#include "url.hh"
@@ -13,73 +12,101 @@ namespace nix { class Store; }
namespace nix::fetchers {
-struct Input;
-
struct Tree
{
Path actualPath;
StorePath storePath;
- TreeInfo info;
+ Tree(Path && actualPath, StorePath && storePath) : actualPath(actualPath), storePath(std::move(storePath)) {}
};
-struct Input : std::enable_shared_from_this<Input>
+struct InputScheme;
+
+struct Input
{
- std::optional<Hash> narHash; // FIXME: implement
+ friend class InputScheme;
+
+ std::shared_ptr<InputScheme> scheme; // note: can be null
+ Attrs attrs;
+ bool immutable = false;
+ bool direct = true;
+
+public:
+ static Input fromURL(const std::string & url);
- virtual std::string type() const = 0;
+ static Input fromURL(const ParsedURL & url);
- virtual ~Input() { }
+ static Input fromAttrs(Attrs && attrs);
- virtual bool operator ==(const Input & other) const { return false; }
+ ParsedURL toURL() const;
+
+ std::string to_string() const;
+
+ Attrs toAttrs() const;
/* Check whether this is a "direct" input, that is, not
one that goes through a registry. */
- virtual bool isDirect() const { return true; }
+ bool isDirect() const { return direct; }
/* Check whether this is an "immutable" input, that is,
one that contains a commit hash or content hash. */
- virtual bool isImmutable() const { return (bool) narHash; }
+ bool isImmutable() const { return immutable; }
- virtual bool contains(const Input & other) const { return false; }
+ bool hasAllInfo() const;
- virtual std::optional<std::string> getRef() const { return {}; }
+ bool operator ==(const Input & other) const;
- virtual std::optional<Hash> getRev() const { return {}; }
+ bool contains(const Input & other) const;
- virtual ParsedURL toURL() const = 0;
+ std::pair<Tree, Input> fetch(ref<Store> store) const;
- std::string to_string() const
- {
- return toURL().to_string();
- }
+ Input applyOverrides(
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) const;
- Attrs toAttrs() const;
+ void clone(const Path & destDir) const;
- std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
+ std::optional<Path> getSourcePath() const;
-private:
+ void markChangedFile(
+ std::string_view file,
+ std::optional<std::string> commitMsg) const;
- virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
+ StorePath computeStorePath(Store & store) const;
- virtual Attrs toAttrsInternal() const = 0;
+ // Convience functions for common attributes.
+ std::string getType() const;
+ std::optional<Hash> getNarHash() const;
+ std::optional<std::string> getRef() const;
+ std::optional<Hash> getRev() const;
+ std::optional<uint64_t> getRevCount() const;
+ std::optional<time_t> getLastModified() const;
};
struct InputScheme
{
- virtual ~InputScheme() { }
+ virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;
- virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
+ virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0;
- virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
-};
+ virtual ParsedURL toURL(const Input & input);
+
+ virtual bool hasAllInfo(const Input & input) = 0;
-std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
+ virtual Input applyOverrides(
+ const Input & input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev);
-std::unique_ptr<Input> inputFromURL(const std::string & url);
+ virtual void clone(const Input & input, const Path & destDir);
-std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
+ virtual std::optional<Path> getSourcePath(const Input & input);
+
+ virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
+
+ virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0;
+};
-void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
+void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
struct DownloadFileResult
{
@@ -94,7 +121,7 @@ DownloadFileResult downloadFile(
const std::string & name,
bool immutable);
-Tree downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 75ce5ee8b..5d38e0c2b 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -22,80 +22,152 @@ static bool isNotDotGitDirectory(const Path & path)
return not std::regex_match(path, gitDirRegex);
}
-struct GitInput : Input
+struct GitInputScheme : InputScheme
{
- ParsedURL url;
- std::optional<std::string> ref;
- std::optional<Hash> rev;
- bool shallow = false;
- bool submodules = false;
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
+ {
+ if (url.scheme != "git" &&
+ url.scheme != "git+http" &&
+ url.scheme != "git+https" &&
+ url.scheme != "git+ssh" &&
+ url.scheme != "git+file") return {};
+
+ auto url2(url);
+ if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
+ url2.query.clear();
+
+ Attrs attrs;
+ attrs.emplace("type", "git");
+
+ for (auto &[name, value] : url.query) {
+ if (name == "rev" || name == "ref")
+ attrs.emplace(name, value);
+ else if (name == "shallow")
+ attrs.emplace(name, Explicit<bool> { value == "1" });
+ else
+ url2.query.emplace(name, value);
+ }
+
+ attrs.emplace("url", url2.to_string());
+
+ return inputFromAttrs(attrs);
+ }
- GitInput(const ParsedURL & url) : url(url)
- { }
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
+ {
+ if (maybeGetStrAttr(attrs, "type") != "git") return {};
- std::string type() const override { return "git"; }
+ for (auto & [name, value] : attrs)
+ if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
+ throw Error("unsupported Git input attribute '%s'", name);
+
+ parseURL(getStrAttr(attrs, "url"));
+ maybeGetBoolAttr(attrs, "shallow");
+ maybeGetBoolAttr(attrs, "submodules");
+
+ if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+ if (std::regex_search(*ref, badGitRefRegex))
+ throw BadURL("invalid Git branch/tag name '%s'", *ref);
+ }
+
+ Input input;
+ input.attrs = attrs;
+ return input;
+ }
- bool operator ==(const Input & other) const override
+ ParsedURL toURL(const Input & input) override
{
- auto other2 = dynamic_cast<const GitInput *>(&other);
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
+ if (url.scheme != "git") url.scheme = "git+" + url.scheme;
+ if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
+ if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
+ if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
+ url.query.insert_or_assign("shallow", "1");
+ return url;
+ }
+
+ bool hasAllInfo(const Input & input) override
+ {
+ bool maybeDirty = !input.getRef();
+ bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
return
- other2
- && url == other2->url
- && rev == other2->rev
- && ref == other2->ref;
+ maybeGetIntAttr(input.attrs, "lastModified")
+ && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount"));
}
- bool isImmutable() const override
+ Input applyOverrides(
+ const Input & input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) override
{
- return (bool) rev || narHash;
+ auto res(input);
+ if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
+ if (ref) res.attrs.insert_or_assign("ref", *ref);
+ if (!res.getRef() && res.getRev())
+ throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string());
+ return res;
}
- std::optional<std::string> getRef() const override { return ref; }
+ void clone(const Input & input, const Path & destDir) override
+ {
+ auto [isLocal, actualUrl] = getActualUrl(input);
+
+ Strings args = {"clone"};
+
+ args.push_back(actualUrl);
- std::optional<Hash> getRev() const override { return rev; }
+ if (auto ref = input.getRef()) {
+ args.push_back("--branch");
+ args.push_back(*ref);
+ }
+
+ if (input.getRev()) throw Error("cloning a specific revision is not implemented");
+
+ args.push_back(destDir);
- ParsedURL toURL() const override
+ runProgram("git", true, args);
+ }
+
+ std::optional<Path> getSourcePath(const Input & input) override
{
- ParsedURL url2(url);
- if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
- if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
- if (ref) url2.query.insert_or_assign("ref", *ref);
- if (shallow) url2.query.insert_or_assign("shallow", "1");
- return url2;
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
+ if (url.scheme == "file" && !input.getRef() && !input.getRev())
+ return url.path;
+ return {};
}
- Attrs toAttrsInternal() const override
+ void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
- Attrs attrs;
- attrs.emplace("url", url.to_string());
- if (ref)
- attrs.emplace("ref", *ref);
- if (rev)
- attrs.emplace("rev", rev->gitRev());
- if (shallow)
- attrs.emplace("shallow", true);
- if (submodules)
- attrs.emplace("submodules", true);
- return attrs;
+ auto sourcePath = getSourcePath(input);
+ assert(sourcePath);
+
+ runProgram("git", true,
+ { "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) });
+
+ if (commitMsg)
+ runProgram("git", true,
+ { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg });
}
- std::pair<bool, std::string> getActualUrl() const
+ std::pair<bool, std::string> getActualUrl(const Input & input) const
{
// Don't clone file:// URIs (but otherwise treat them the
// same as remote URIs, i.e. don't use the working tree or
// HEAD).
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file" && !forceHttp;
return {isLocal, isLocal ? url.path : url.base};
}
- std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
auto name = "source";
- auto input = std::make_shared<GitInput>(*this);
+ Input input(_input);
- assert(!rev || rev->type == htSHA1);
+ bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
+ bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
std::string cacheType = "git";
if (shallow) cacheType += "-shallow";
@@ -106,39 +178,35 @@ struct GitInput : Input
return Attrs({
{"type", cacheType},
{"name", name},
- {"rev", input->rev->gitRev()},
+ {"rev", input.getRev()->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
- -> std::pair<Tree, std::shared_ptr<const Input>>
+ -> std::pair<Tree, Input>
{
- assert(input->rev);
- assert(!rev || rev == input->rev);
+ assert(input.getRev());
+ assert(!_input.getRev() || _input.getRev() == input.getRev());
+ if (!shallow)
+ input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
+ input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
return {
- Tree {
- .actualPath = store->toRealPath(storePath),
- .storePath = std::move(storePath),
- .info = TreeInfo {
- .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
- .lastModified = getIntAttr(infoAttrs, "lastModified"),
- },
- },
+ Tree(store->toRealPath(storePath), std::move(storePath)),
input
};
};
- if (rev) {
+ if (input.getRev()) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
- auto [isLocal, actualUrl_] = getActualUrl();
+ auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
// If this is a local directory and no ref or revision is
// given, then allow the use of an unclean working tree.
- if (!input->ref && !input->rev && isLocal) {
+ if (!input.getRef() && !input.getRev() && isLocal) {
bool clean = false;
/* Check whether this repo has any commits. There are
@@ -197,35 +265,35 @@ struct GitInput : Input
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
- auto tree = Tree {
- .actualPath = store->printStorePath(storePath),
- .storePath = std::move(storePath),
- .info = TreeInfo {
- // FIXME: maybe we should use the timestamp of the last
- // modified dirty file?
- .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
- }
- };
+ // FIXME: maybe we should use the timestamp of the last
+ // modified dirty file?
+ input.attrs.insert_or_assign(
+ "lastModified",
+ haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
- return {std::move(tree), input};
+ return {
+ Tree(store->printStorePath(storePath), std::move(storePath)),
+ input
+ };
}
}
- if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
+ if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
Attrs mutableAttrs({
{"type", cacheType},
{"name", name},
{"url", actualUrl},
- {"ref", *input->ref},
+ {"ref", *input.getRef()},
});
Path repoDir;
if (isLocal) {
- if (!input->rev)
- input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
+ if (!input.getRev())
+ input.attrs.insert_or_assign("rev",
+ Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
repoDir = actualUrl;
@@ -233,8 +301,8 @@ struct GitInput : Input
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
- if (!rev || rev == rev2) {
- input->rev = rev2;
+ if (!input.getRev() || input.getRev() == rev2) {
+ input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
}
}
@@ -248,18 +316,18 @@ struct GitInput : Input
}
Path localRefFile =
- input->ref->compare(0, 5, "refs/") == 0
- ? cacheDir + "/" + *input->ref
- : cacheDir + "/refs/heads/" + *input->ref;
+ input.getRef()->compare(0, 5, "refs/") == 0
+ ? cacheDir + "/" + *input.getRef()
+ : cacheDir + "/refs/heads/" + *input.getRef();
bool doFetch;
time_t now = time(0);
/* If a rev was specified, we need to fetch if it's not in the
repo. */
- if (input->rev) {
+ if (input.getRev()) {
try {
- runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
+ runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() });
doFetch = false;
} catch (ExecError & e) {
if (WIFEXITED(e.status)) {
@@ -282,9 +350,10 @@ struct GitInput : Input
// FIXME: git stderr messes up our progress indicator, so
// we're using --quiet for now. Should process its stderr.
try {
- auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
- ? *input->ref
- : "refs/heads/" + *input->ref;
+ auto ref = input.getRef();
+ auto fetchRef = ref->compare(0, 5, "refs/") == 0
+ ? *ref
+ : "refs/heads/" + *ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
@@ -300,8 +369,8 @@ struct GitInput : Input
utimes(localRefFile.c_str(), times);
}
- if (!input->rev)
- input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
+ if (!input.getRev())
+ input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev());
}
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
@@ -311,7 +380,7 @@ struct GitInput : Input
// FIXME: check whether rev is an ancestor of ref.
- printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
+ printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
/* Now that we know the ref, check again whether we have it in
the store. */
@@ -333,7 +402,7 @@ struct GitInput : Input
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
"--update-head-ok", "--", repoDir, "refs/*:refs/*" });
- runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
+ runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
@@ -342,7 +411,7 @@ struct GitInput : Input
// FIXME: should pipe this, or find some better way to extract a
// revision.
auto source = sinkToSource([&](Sink & sink) {
- RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
+ RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
gitOptions.standardOut = &sink;
runProgram2(gitOptions);
});
@@ -352,18 +421,18 @@ struct GitInput : Input
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
- auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
+ auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
Attrs infoAttrs({
- {"rev", input->rev->gitRev()},
+ {"rev", input.getRev()->gitRev()},
{"lastModified", lastModified},
});
if (!shallow)
infoAttrs.insert_or_assign("revCount",
- std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
+ std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() })));
- if (!this->rev)
+ if (!_input.getRev())
getCache()->add(
store,
mutableAttrs,
@@ -382,60 +451,6 @@ struct GitInput : Input
}
};
-struct GitInputScheme : InputScheme
-{
- std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
- {
- if (url.scheme != "git" &&
- url.scheme != "git+http" &&
- url.scheme != "git+https" &&
- url.scheme != "git+ssh" &&
- url.scheme != "git+file") return nullptr;
-
- auto url2(url);
- if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
- url2.query.clear();
-
- Attrs attrs;
- attrs.emplace("type", "git");
-
- for (auto &[name, value] : url.query) {
- if (name == "rev" || name == "ref")
- attrs.emplace(name, value);
- else
- url2.query.emplace(name, value);
- }
-
- attrs.emplace("url", url2.to_string());
-
- return inputFromAttrs(attrs);
- }
-
- std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
- {
- if (maybeGetStrAttr(attrs, "type") != "git") return {};
-
- for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
- throw Error("unsupported Git input attribute '%s'", name);
-
- auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
- if (auto ref = maybeGetStrAttr(attrs, "ref")) {
- if (std::regex_search(*ref, badGitRefRegex))
- throw BadURL("invalid Git branch/tag name '%s'", *ref);
- input->ref = *ref;
- }
- if (auto rev = maybeGetStrAttr(attrs, "rev"))
- input->rev = Hash(*rev, htSHA1);
-
- input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
-
- input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
-
- return input;
- }
-};
-
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
}
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 0bee1d6b3..8bb7c2c1d 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -8,81 +8,142 @@
namespace nix::fetchers {
-std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
-std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+// A github or gitlab url
+const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
+std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
-struct GitHubInput : Input
+struct GitArchiveInputScheme : InputScheme
{
- std::string owner;
- std::string repo;
- std::optional<std::string> ref;
- std::optional<Hash> rev;
+ virtual std::string type() = 0;
- std::string type() const override { return "github"; }
-
- bool operator ==(const Input & other) const override
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
{
- auto other2 = dynamic_cast<const GitHubInput *>(&other);
- return
- other2
- && owner == other2->owner
- && repo == other2->repo
- && rev == other2->rev
- && ref == other2->ref;
+ if (url.scheme != type()) return {};
+
+ auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+
+ std::optional<Hash> rev;
+ std::optional<std::string> ref;
+ std::optional<std::string> host_url;
+
+ if (path.size() == 2) {
+ } else if (path.size() == 3) {
+ if (std::regex_match(path[2], revRegex))
+ rev = Hash(path[2], htSHA1);
+ else if (std::regex_match(path[2], refRegex))
+ ref = path[2];
+ else
+ throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
+ } else
+ throw BadURL("URL '%s' is invalid", url.url);
+
+ for (auto &[name, value] : url.query) {
+ if (name == "rev") {
+ if (rev)
+ throw BadURL("URL '%s' contains multiple commit hashes", url.url);
+ rev = Hash(value, htSHA1);
+ }
+ else if (name == "ref") {
+ if (!std::regex_match(value, refRegex))
+ throw BadURL("URL '%s' contains an invalid branch/tag name", url.url);
+ if (ref)
+ throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
+ ref = value;
+ }
+ else if (name == "url") {
+ if (!std::regex_match(value, urlRegex))
+ throw BadURL("URL '%s' contains an invalid instance url", url.url);
+ host_url = value;
+ }
+ // FIXME: barf on unsupported attributes
+ }
+
+ if (ref && rev)
+ throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
+
+ Input input;
+ input.attrs.insert_or_assign("type", type());
+ input.attrs.insert_or_assign("owner", path[0]);
+ input.attrs.insert_or_assign("repo", path[1]);
+ if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+ if (ref) input.attrs.insert_or_assign("ref", *ref);
+ if (host_url) input.attrs.insert_or_assign("url", *host_url);
+
+ return input;
}
- bool isImmutable() const override
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
- return (bool) rev || narHash;
- }
+ if (maybeGetStrAttr(attrs, "type") != type()) return {};
+
+ for (auto & [name, value] : attrs)
+ if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
+ throw Error("unsupported input attribute '%s'", name);
- std::optional<std::string> getRef() const override { return ref; }
+ getStrAttr(attrs, "owner");
+ getStrAttr(attrs, "repo");
- std::optional<Hash> getRev() const override { return rev; }
+ Input input;
+ input.attrs = attrs;
+ return input;
+ }
- ParsedURL toURL() const override
+ ParsedURL toURL(const Input & input) override
{
+ auto owner = getStrAttr(input.attrs, "owner");
+ auto repo = getStrAttr(input.attrs, "repo");
+ auto ref = input.getRef();
+ auto rev = input.getRev();
auto path = owner + "/" + repo;
assert(!(ref && rev));
if (ref) path += "/" + *ref;
if (rev) path += "/" + rev->to_string(Base16, false);
return ParsedURL {
- .scheme = "github",
+ .scheme = type(),
.path = path,
};
}
- Attrs toAttrsInternal() const override
+ bool hasAllInfo(const Input & input) override
{
- Attrs attrs;
- attrs.emplace("owner", owner);
- attrs.emplace("repo", repo);
- if (ref)
- attrs.emplace("ref", *ref);
- if (rev)
- attrs.emplace("rev", rev->gitRev());
- return attrs;
+ return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified");
}
- std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+ Input applyOverrides(
+ const Input & _input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) override
{
- auto rev = this->rev;
- auto ref = this->ref.value_or("master");
-
- if (!rev) {
- auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
- owner, repo, ref);
- auto json = nlohmann::json::parse(
- readFile(
- store->toRealPath(
- downloadFile(store, url, "source", false).storePath)));
- rev = Hash(std::string { json["sha"] }, htSHA1);
- debug("HEAD revision for '%s' is %s", url, rev->gitRev());
+ auto input(_input);
+ if (rev && ref)
+ throw BadURL("cannot apply both a commit hash (%s) and a branch/tag name ('%s') to input '%s'",
+ rev->gitRev(), *ref, input.to_string());
+ if (rev) {
+ input.attrs.insert_or_assign("rev", rev->gitRev());
+ input.attrs.erase("ref");
}
+ if (ref) {
+ input.attrs.insert_or_assign("ref", *ref);
+ input.attrs.erase("rev");
+ }
+ return input;
+ }
+
+ virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
+
+ virtual std::string getDownloadUrl(const Input & input) const = 0;
+
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
+ {
+ Input input(_input);
- auto input = std::make_shared<GitHubInput>(*this);
- input->ref = {};
- input->rev = *rev;
+ if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD");
+
+ auto rev = input.getRev();
+ if (!rev) rev = getRevFromRef(store, input);
+
+ input.attrs.erase("ref");
+ input.attrs.insert_or_assign("rev", rev->gitRev());
Attrs immutableAttrs({
{"type", "git-tarball"},
@@ -90,36 +151,25 @@ struct GitHubInput : Input
});
if (auto res = getCache()->lookup(store, immutableAttrs)) {
+ input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
return {
- Tree{
- .actualPath = store->toRealPath(res->second),
- .storePath = std::move(res->second),
- .info = TreeInfo {
- .lastModified = getIntAttr(res->first, "lastModified"),
- },
- },
+ Tree(store->toRealPath(res->second), std::move(res->second)),
input
};
}
- // FIXME: use regular /archive URLs instead? api.github.com
- // might have stricter rate limits.
+ auto url = getDownloadUrl(input);
- auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
- owner, repo, rev->to_string(Base16, false));
+ auto [tree, lastModified] = downloadTarball(store, url, "source", true);
- std::string accessToken = settings.githubAccessToken.get();
- if (accessToken != "")
- url += "?access_token=" + accessToken;
-
- auto tree = downloadTarball(store, url, "source", true);
+ input.attrs.insert_or_assign("lastModified", lastModified);
getCache()->add(
store,
immutableAttrs,
{
{"rev", rev->gitRev()},
- {"lastModified", *tree.info.lastModified}
+ {"lastModified", lastModified}
},
tree.storePath,
true);
@@ -128,68 +178,96 @@ struct GitHubInput : Input
}
};
-struct GitHubInputScheme : InputScheme
+struct GitHubInputScheme : GitArchiveInputScheme
{
- std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+ std::string type() override { return "github"; }
+
+ Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
{
- if (url.scheme != "github") return nullptr;
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+ auto json = nlohmann::json::parse(
+ readFile(
+ store->toRealPath(
+ downloadFile(store, url, "source", false).storePath)));
+ auto rev = Hash(std::string { json["sha"] }, htSHA1);
+ debug("HEAD revision for '%s' is %s", url, rev.gitRev());
+ return rev;
+ }
- auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
- auto input = std::make_unique<GitHubInput>();
+ std::string getDownloadUrl(const Input & input) const override
+ {
+ // FIXME: use regular /archive URLs instead? api.github.com
+ // might have stricter rate limits.
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+ input.getRev()->to_string(Base16, false));
- if (path.size() == 2) {
- } else if (path.size() == 3) {
- if (std::regex_match(path[2], revRegex))
- input->rev = Hash(path[2], htSHA1);
- else if (std::regex_match(path[2], refRegex))
- input->ref = path[2];
- else
- throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
- } else
- throw BadURL("GitHub URL '%s' is invalid", url.url);
+ std::string accessToken = settings.githubAccessToken.get();
+ if (accessToken != "")
+ url += "?access_token=" + accessToken;
- for (auto &[name, value] : url.query) {
- if (name == "rev") {
- if (input->rev)
- throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
- input->rev = Hash(value, htSHA1);
- }
- else if (name == "ref") {
- if (!std::regex_match(value, refRegex))
- throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
- if (input->ref)
- throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
- input->ref = value;
- }
- }
+ return url;
+ }
- if (input->ref && input->rev)
- throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
+ void clone(const Input & input, const Path & destDir) override
+ {
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
+ Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+ .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+ .clone(destDir);
+ }
+};
- input->owner = path[0];
- input->repo = path[1];
+struct GitLabInputScheme : GitArchiveInputScheme
+{
+ std::string type() override { return "gitlab"; }
- return input;
+ Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
+ {
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+ auto json = nlohmann::json::parse(
+ readFile(
+ store->toRealPath(
+ downloadFile(store, url, "source", false).storePath)));
+ auto rev = Hash(std::string(json[0]["id"]), htSHA1);
+ debug("HEAD revision for '%s' is %s", url, rev.gitRev());
+ return rev;
}
- std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+ std::string getDownloadUrl(const Input & input) const override
{
- if (maybeGetStrAttr(attrs, "type") != "github") return {};
+ // FIXME: This endpoint has a rate limit threshold of 5 requests per minute
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
+ input.getRev()->to_string(Base16, false));
- for (auto & [name, value] : attrs)
- if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
- throw Error("unsupported GitHub input attribute '%s'", name);
-
- auto input = std::make_unique<GitHubInput>();
- input->owner = getStrAttr(attrs, "owner");
- input->repo = getStrAttr(attrs, "repo");
- input->ref = maybeGetStrAttr(attrs, "ref");
- if (auto rev = maybeGetStrAttr(attrs, "rev"))
- input->rev = Hash(*rev, htSHA1);
- return input;
+ /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
+ std::string accessToken = settings.githubAccessToken.get();
+ if (accessToken != "")
+ url += "?access_token=" + accessToken;*/
+
+ return url;
+ }
+
+ void clone(const Input & input, const Path & destDir) override
+ {
+ auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+ // FIXME: get username somewhere
+ Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
+ host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
+ .applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
+ .clone(destDir);
}
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
+static auto r2 = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
}
diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc
new file mode 100644
index 000000000..91dc83740
--- /dev/null
+++ b/src/libfetchers/indirect.cc
@@ -0,0 +1,104 @@
+#include "fetchers.hh"
+
+namespace nix::fetchers {
+
+std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
+
+struct IndirectInputScheme : InputScheme
+{
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
+ {
+ if (url.scheme != "flake") return {};
+
+ auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
+
+ std::optional<Hash> rev;
+ std::optional<std::string> ref;
+
+ if (path.size() == 1) {
+ } else if (path.size() == 2) {
+ if (std::regex_match(path[1], revRegex))
+ rev = Hash(path[1], htSHA1);
+ else if (std::regex_match(path[1], refRegex))
+ ref = path[1];
+ else
+ throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
+ } else if (path.size() == 3) {
+ if (!std::regex_match(path[1], refRegex))
+ throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
+ ref = path[1];
+ if (!std::regex_match(path[2], revRegex))
+ throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
+ rev = Hash(path[2], htSHA1);
+ } else
+ throw BadURL("GitHub URL '%s' is invalid", url.url);
+
+ std::string id = path[0];
+ if (!std::regex_match(id, flakeRegex))
+ throw BadURL("'%s' is not a valid flake ID", id);
+
+ // FIXME: forbid query params?
+
+ Input input;
+ input.direct = false;
+ input.attrs.insert_or_assign("type", "indirect");
+ input.attrs.insert_or_assign("id", id);
+ if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+ if (ref) input.attrs.insert_or_assign("ref", *ref);
+
+ return input;
+ }
+
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
+ {
+ if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
+
+ for (auto & [name, value] : attrs)
+ if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
+ throw Error("unsupported indirect input attribute '%s'", name);
+
+ auto id = getStrAttr(attrs, "id");
+ if (!std::regex_match(id, flakeRegex))
+ throw BadURL("'%s' is not a valid flake ID", id);
+
+ Input input;
+ input.direct = false;
+ input.attrs = attrs;
+ return input;
+ }
+
+ ParsedURL toURL(const Input & input) override
+ {
+ ParsedURL url;
+ url.scheme = "flake";
+ url.path = getStrAttr(input.attrs, "id");
+ if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; };
+ if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); };
+ return url;
+ }
+
+ bool hasAllInfo(const Input & input) override
+ {
+ return false;
+ }
+
+ Input applyOverrides(
+ const Input & _input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) override
+ {
+ auto input(_input);
+ if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
+ if (ref) input.attrs.insert_or_assign("ref", *ref);
+ return input;
+ }
+
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+ {
+ throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
+ }
+};
+
+static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
+
+}
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 2e0d4bf4d..c48cb6fd1 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -10,76 +10,124 @@ using namespace std::string_literals;
namespace nix::fetchers {
-struct MercurialInput : Input
+struct MercurialInputScheme : InputScheme
{
- ParsedURL url;
- std::optional<std::string> ref;
- std::optional<Hash> rev;
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
+ {
+ if (url.scheme != "hg+http" &&
+ url.scheme != "hg+https" &&
+ url.scheme != "hg+ssh" &&
+ url.scheme != "hg+file") return {};
- MercurialInput(const ParsedURL & url) : url(url)
- { }
+ auto url2(url);
+ url2.scheme = std::string(url2.scheme, 3);
+ url2.query.clear();
+
+ Attrs attrs;
+ attrs.emplace("type", "hg");
- std::string type() const override { return "hg"; }
+ for (auto &[name, value] : url.query) {
+ if (name == "rev" || name == "ref")
+ attrs.emplace(name, value);
+ else
+ url2.query.emplace(name, value);
+ }
+
+ attrs.emplace("url", url2.to_string());
+
+ return inputFromAttrs(attrs);
+ }
- bool operator ==(const Input & other) const override
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
- auto other2 = dynamic_cast<const MercurialInput *>(&other);
- return
- other2
- && url == other2->url
- && rev == other2->rev
- && ref == other2->ref;
+ if (maybeGetStrAttr(attrs, "type") != "hg") return {};
+
+ for (auto & [name, value] : attrs)
+ if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
+ throw Error("unsupported Mercurial input attribute '%s'", name);
+
+ parseURL(getStrAttr(attrs, "url"));
+
+ if (auto ref = maybeGetStrAttr(attrs, "ref")) {
+ if (!std::regex_match(*ref, refRegex))
+ throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
+ }
+
+ Input input;
+ input.attrs = attrs;
+ return input;
}
- bool isImmutable() const override
+ ParsedURL toURL(const Input & input) override
{
- return (bool) rev || narHash;
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
+ url.scheme = "hg+" + url.scheme;
+ if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
+ if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
+ return url;
}
- std::optional<std::string> getRef() const override { return ref; }
+ bool hasAllInfo(const Input & input) override
+ {
+ // FIXME: ugly, need to distinguish between dirty and clean
+ // default trees.
+ return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount");
+ }
- std::optional<Hash> getRev() const override { return rev; }
+ Input applyOverrides(
+ const Input & input,
+ std::optional<std::string> ref,
+ std::optional<Hash> rev) override
+ {
+ auto res(input);
+ if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
+ if (ref) res.attrs.insert_or_assign("ref", *ref);
+ return res;
+ }
- ParsedURL toURL() const override
+ std::optional<Path> getSourcePath(const Input & input) override
{
- ParsedURL url2(url);
- url2.scheme = "hg+" + url2.scheme;
- if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
- if (ref) url2.query.insert_or_assign("ref", *ref);
- return url;
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
+ if (url.scheme == "file" && !input.getRef() && !input.getRev())
+ return url.path;
+ return {};
}
- Attrs toAttrsInternal() const override
+ void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
- Attrs attrs;
- attrs.emplace("url", url.to_string());
- if (ref)
- attrs.emplace("ref", *ref);
- if (rev)
- attrs.emplace("rev", rev->gitRev());
- return attrs;
+ auto sourcePath = getSourcePath(input);
+ assert(sourcePath);
+
+ // FIXME: shut up if file is already tracked.
+ runProgram("hg", true,
+ { "add", *sourcePath + "/" + std::string(file) });
+
+ if (commitMsg)
+ runProgram("hg", true,
+ { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
}
- std::pair<bool, std::string> getActualUrl() const
+ std::pair<bool, std::string> getActualUrl(const Input & input) const
{
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
bool isLocal = url.scheme == "file";
return {isLocal, isLocal ? url.path : url.base};
}
- std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
auto name = "source";
- auto input = std::make_shared<MercurialInput>(*this);
+ Input input(_input);
- auto [isLocal, actualUrl_] = getActualUrl();
+ auto [isLocal, actualUrl_] = getActualUrl(input);
auto actualUrl = actualUrl_; // work around clang bug
// FIXME: return lastModified.
// FIXME: don't clone local repositories.
- if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
+ if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
@@ -94,7 +142,7 @@ struct MercurialInput : Input
if (settings.warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl);
- input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
+ input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
auto files = tokenizeString<std::set<std::string>>(
runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
@@ -116,60 +164,54 @@ struct MercurialInput : Input
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
- return {Tree {
- .actualPath = store->printStorePath(storePath),
- .storePath = std::move(storePath),
- }, input};
+ return {
+ Tree(store->printStorePath(storePath), std::move(storePath)),
+ input
+ };
}
}
- if (!input->ref) input->ref = "default";
+ if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
auto getImmutableAttrs = [&]()
{
return Attrs({
{"type", "hg"},
{"name", name},
- {"rev", input->rev->gitRev()},
+ {"rev", input.getRev()->gitRev()},
});
};
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
- -> std::pair<Tree, std::shared_ptr<const Input>>
+ -> std::pair<Tree, Input>
{
- assert(input->rev);
- assert(!rev || rev == input->rev);
+ assert(input.getRev());
+ assert(!_input.getRev() || _input.getRev() == input.getRev());
+ input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
return {
- Tree{
- .actualPath = store->toRealPath(storePath),
- .storePath = std::move(storePath),
- .info = TreeInfo {
- .revCount = getIntAttr(infoAttrs, "revCount"),
- },
- },
+ Tree(store->toRealPath(storePath), std::move(storePath)),
input
};
};
- if (input->rev) {
+ if (input.getRev()) {
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
}
- assert(input->rev || input->ref);
- auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
+ auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef();
Attrs mutableAttrs({
{"type", "hg"},
{"name", name},
{"url", actualUrl},
- {"ref", *input->ref},
+ {"ref", *input.getRef()},
});
if (auto res = getCache()->lookup(store, mutableAttrs)) {
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
- if (!rev || rev == rev2) {
- input->rev = rev2;
+ if (!input.getRev() || input.getRev() == rev2) {
+ input.attrs.insert_or_assign("rev", rev2.gitRev());
return makeResult(res->first, std::move(res->second));
}
}
@@ -178,10 +220,10 @@ struct MercurialInput : Input
/* If this is a commit hash that we already have, we don't
have to pull again. */
- if (!(input->rev
+ if (!(input.getRev()
&& pathExists(cacheDir)
&& runProgram(
- RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
+ RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
@@ -210,9 +252,9 @@ struct MercurialInput : Input
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
- input->rev = Hash(tokens[0], htSHA1);
+ input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev());
auto revCount = std::stoull(tokens[1]);
- input->ref = tokens[2];
+ input.attrs.insert_or_assign("ref", tokens[2]);
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
return makeResult(res->first, std::move(res->second));
@@ -220,18 +262,18 @@ struct MercurialInput : Input
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
- runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
+ runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
deletePath(tmpDir + "/.hg_archival.txt");
auto storePath = store->addToStore(name, tmpDir);
Attrs infoAttrs({
- {"rev", input->rev->gitRev()},
+ {"rev", input.getRev()->gitRev()},
{"revCount", (int64_t) revCount},
});
- if (!this->rev)
+ if (!_input.getRev())
getCache()->add(
store,
mutableAttrs,
@@ -250,54 +292,6 @@ struct MercurialInput : Input
}
};
-struct MercurialInputScheme : InputScheme
-{
- std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
- {
- if (url.scheme != "hg+http" &&
- url.scheme != "hg+https" &&
- url.scheme != "hg+ssh" &&
- url.scheme != "hg+file") return nullptr;
-
- auto url2(url);
- url2.scheme = std::string(url2.scheme, 3);
- url2.query.clear();
-
- Attrs attrs;
- attrs.emplace("type", "hg");
-
- for (auto &[name, value] : url.query) {
- if (name == "rev" || name == "ref")
- attrs.emplace(name, value);
- else
- url2.query.emplace(name, value);
- }
-
- attrs.emplace("url", url2.to_string());
-
- return inputFromAttrs(attrs);
- }
-
- std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
- {
- if (maybeGetStrAttr(attrs, "type") != "hg") return {};
-
- for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "ref" && name != "rev")
- throw Error("unsupported Mercurial input attribute '%s'", name);
-
- auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
- if (auto ref = maybeGetStrAttr(attrs, "ref")) {
- if (!std::regex_match(*ref, refRegex))
- throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
- input->ref = *ref;
- }
- if (auto rev = maybeGetStrAttr(attrs, "rev"))
- input->rev = Hash(*rev, htSHA1);
- return input;
- }
-};
-
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
}
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index ba2cc192e..99d4b4e8f 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -3,65 +3,86 @@
namespace nix::fetchers {
-struct PathInput : Input
+struct PathInputScheme : InputScheme
{
- Path path;
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
+ {
+ if (url.scheme != "path") return {};
- /* Allow the user to pass in "fake" tree info attributes. This is
- useful for making a pinned tree work the same as the repository
- from which is exported
- (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
- std::optional<Hash> rev;
- std::optional<uint64_t> revCount;
- std::optional<time_t> lastModified;
+ if (url.authority && *url.authority != "")
+ throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
- std::string type() const override { return "path"; }
+ Input input;
+ input.attrs.insert_or_assign("type", "path");
+ input.attrs.insert_or_assign("path", url.path);
- std::optional<Hash> getRev() const override { return rev; }
+ for (auto & [name, value] : url.query)
+ if (name == "rev" || name == "narHash")
+ input.attrs.insert_or_assign(name, value);
+ else if (name == "revCount" || name == "lastModified") {
+ uint64_t n;
+ if (!string2Int(value, n))
+ throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
+ input.attrs.insert_or_assign(name, n);
+ }
+ else
+ throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
- bool operator ==(const Input & other) const override
- {
- auto other2 = dynamic_cast<const PathInput *>(&other);
- return
- other2
- && path == other2->path
- && rev == other2->rev
- && revCount == other2->revCount
- && lastModified == other2->lastModified;
+ return input;
}
- bool isImmutable() const override
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
- return (bool) narHash;
+ if (maybeGetStrAttr(attrs, "type") != "path") return {};
+
+ getStrAttr(attrs, "path");
+
+ for (auto & [name, value] : attrs)
+ /* Allow the user to pass in "fake" tree info
+ attributes. This is useful for making a pinned tree
+ work the same as the repository from which is exported
+ (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
+ if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
+ // checked in Input::fromAttrs
+ ;
+ else
+ throw Error("unsupported path input attribute '%s'", name);
+
+ Input input;
+ input.attrs = attrs;
+ return input;
}
- ParsedURL toURL() const override
+ ParsedURL toURL(const Input & input) override
{
- auto query = attrsToQuery(toAttrsInternal());
+ auto query = attrsToQuery(input.attrs);
query.erase("path");
+ query.erase("type");
return ParsedURL {
.scheme = "path",
- .path = path,
+ .path = getStrAttr(input.attrs, "path"),
.query = query,
};
}
- Attrs toAttrsInternal() const override
+ bool hasAllInfo(const Input & input) override
+ {
+ return true;
+ }
+
+ std::optional<Path> getSourcePath(const Input & input) override
{
- Attrs attrs;
- attrs.emplace("path", path);
- if (rev)
- attrs.emplace("rev", rev->gitRev());
- if (revCount)
- attrs.emplace("revCount", *revCount);
- if (lastModified)
- attrs.emplace("lastModified", *lastModified);
- return attrs;
+ return getStrAttr(input.attrs, "path");
}
- std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
+ void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
{
- auto input = std::make_shared<PathInput>(*this);
+ // nothing to do
+ }
+
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+ {
+ auto path = getStrAttr(input.attrs, "path");
// FIXME: check whether access to 'path' is allowed.
@@ -74,72 +95,10 @@ struct PathInput : Input
// FIXME: try to substitute storePath.
storePath = store->addToStore("source", path);
- return
- {
- Tree {
- .actualPath = store->toRealPath(*storePath),
- .storePath = std::move(*storePath),
- .info = TreeInfo {
- .revCount = revCount,
- .lastModified = lastModified
- }
- },
- input
- };
- }
-
-};
-
-struct PathInputScheme : InputScheme
-{
- std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
- {
- if (url.scheme != "path") return nullptr;
-
- auto input = std::make_unique<PathInput>();
- input->path = url.path;
-
- for (auto & [name, value] : url.query)
- if (name == "rev")
- input->rev = Hash(value, htSHA1);
- else if (name == "revCount") {
- uint64_t revCount;
- if (!string2Int(value, revCount))
- throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
- input->revCount = revCount;
- }
- else if (name == "lastModified") {
- time_t lastModified;
- if (!string2Int(value, lastModified))
- throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
- input->lastModified = lastModified;
- }
- else
- throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
-
- return input;
- }
-
- std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
- {
- if (maybeGetStrAttr(attrs, "type") != "path") return {};
-
- auto input = std::make_unique<PathInput>();
- input->path = getStrAttr(attrs, "path");
-
- for (auto & [name, value] : attrs)
- if (name == "rev")
- input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
- else if (name == "revCount")
- input->revCount = getIntAttr(attrs, "revCount");
- else if (name == "lastModified")
- input->lastModified = getIntAttr(attrs, "lastModified");
- else if (name == "type" || name == "path")
- ;
- else
- throw Error("unsupported path input attribute '%s'", name);
-
- return input;
+ return {
+ Tree(store->toRealPath(*storePath), std::move(*storePath)),
+ input
+ };
}
};
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
new file mode 100644
index 000000000..d4134ce29
--- /dev/null
+++ b/src/libfetchers/registry.cc
@@ -0,0 +1,212 @@
+#include "registry.hh"
+#include "fetchers.hh"
+#include "util.hh"
+#include "globals.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::fetchers {
+
+std::shared_ptr<Registry> Registry::read(
+ const Path & path, RegistryType type)
+{
+ auto registry = std::make_shared<Registry>(type);
+
+ if (!pathExists(path))
+ return std::make_shared<Registry>(type);
+
+ try {
+
+ auto json = nlohmann::json::parse(readFile(path));
+
+ auto version = json.value("version", 0);
+
+ if (version == 2) {
+ for (auto & i : json["flakes"]) {
+ auto toAttrs = jsonToAttrs(i["to"]);
+ Attrs extraAttrs;
+ auto j = toAttrs.find("dir");
+ if (j != toAttrs.end()) {
+ extraAttrs.insert(*j);
+ toAttrs.erase(j);
+ }
+ auto exact = i.find("exact");
+ registry->entries.push_back(
+ Entry {
+ .from = Input::fromAttrs(jsonToAttrs(i["from"])),
+ .to = Input::fromAttrs(std::move(toAttrs)),
+ .extraAttrs = extraAttrs,
+ .exact = exact != i.end() && exact.value()
+ });
+ }
+ }
+
+ else
+ throw Error("flake registry '%s' has unsupported version %d", path, version);
+
+ } catch (nlohmann::json::exception & e) {
+ warn("cannot parse flake registry '%s': %s", path, e.what());
+ } catch (Error & e) {
+ warn("cannot read flake registry '%s': %s", path, e.what());
+ }
+
+ return registry;
+}
+
+void Registry::write(const Path & path)
+{
+ nlohmann::json arr;
+ for (auto & entry : entries) {
+ nlohmann::json obj;
+ obj["from"] = attrsToJson(entry.from.toAttrs());
+ obj["to"] = attrsToJson(entry.to.toAttrs());
+ if (!entry.extraAttrs.empty())
+ obj["to"].update(attrsToJson(entry.extraAttrs));
+ if (entry.exact)
+ obj["exact"] = true;
+ arr.emplace_back(std::move(obj));
+ }
+
+ nlohmann::json json;
+ json["version"] = 2;
+ json["flakes"] = std::move(arr);
+
+ createDirs(dirOf(path));
+ writeFile(path, json.dump(2));
+}
+
+void Registry::add(
+ const Input & from,
+ const Input & to,
+ const Attrs & extraAttrs)
+{
+ entries.emplace_back(
+ Entry {
+ .from = from,
+ .to = to,
+ .extraAttrs = extraAttrs
+ });
+}
+
+void Registry::remove(const Input & input)
+{
+ // FIXME: use C++20 std::erase.
+ for (auto i = entries.begin(); i != entries.end(); )
+ if (i->from == input)
+ i = entries.erase(i);
+ else
+ ++i;
+}
+
+static Path getSystemRegistryPath()
+{
+ return settings.nixConfDir + "/registry.json";
+}
+
+static std::shared_ptr<Registry> getSystemRegistry()
+{
+ static auto systemRegistry =
+ Registry::read(getSystemRegistryPath(), Registry::System);
+ return systemRegistry;
+}
+
+Path getUserRegistryPath()
+{
+ return getHome() + "/.config/nix/registry.json";
+}
+
+std::shared_ptr<Registry> getUserRegistry()
+{
+ static auto userRegistry =
+ Registry::read(getUserRegistryPath(), Registry::User);
+ return userRegistry;
+}
+
+static std::shared_ptr<Registry> flagRegistry =
+ std::make_shared<Registry>(Registry::Flag);
+
+std::shared_ptr<Registry> getFlagRegistry()
+{
+ return flagRegistry;
+}
+
+void overrideRegistry(
+ const Input & from,
+ const Input & to,
+ const Attrs & extraAttrs)
+{
+ flagRegistry->add(from, to, extraAttrs);
+}
+
+static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
+{
+ static auto reg = [&]() {
+ auto path = settings.flakeRegistry.get();
+
+ if (!hasPrefix(path, "/")) {
+ auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
+ if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
+ store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true);
+ path = store->toRealPath(storePath);
+ }
+
+ return Registry::read(path, Registry::Global);
+ }();
+
+ return reg;
+}
+
+Registries getRegistries(ref<Store> store)
+{
+ Registries registries;
+ registries.push_back(getFlagRegistry());
+ registries.push_back(getUserRegistry());
+ registries.push_back(getSystemRegistry());
+ registries.push_back(getGlobalRegistry(store));
+ return registries;
+}
+
+std::pair<Input, Attrs> lookupInRegistries(
+ ref<Store> store,
+ const Input & _input)
+{
+ Attrs extraAttrs;
+ int n = 0;
+ Input input(_input);
+
+ restart:
+
+ n++;
+ if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string());
+
+ for (auto & registry : getRegistries(store)) {
+ // FIXME: O(n)
+ for (auto & entry : registry->entries) {
+ if (entry.exact) {
+ if (entry.from == input) {
+ input = entry.to;
+ extraAttrs = entry.extraAttrs;
+ goto restart;
+ }
+ } else {
+ if (entry.from.contains(input)) {
+ input = entry.to.applyOverrides(
+ !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(),
+ !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>());
+ extraAttrs = entry.extraAttrs;
+ goto restart;
+ }
+ }
+ }
+ }
+
+ if (!input.isDirect())
+ throw Error("cannot find flake '%s' in the flake registries", input.to_string());
+
+ debug("looked up '%s' -> '%s'", _input.to_string(), input.to_string());
+
+ return {input, extraAttrs};
+}
+
+}
diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh
new file mode 100644
index 000000000..1077af020
--- /dev/null
+++ b/src/libfetchers/registry.hh
@@ -0,0 +1,64 @@
+#pragma once
+
+#include "types.hh"
+#include "fetchers.hh"
+
+namespace nix { class Store; }
+
+namespace nix::fetchers {
+
+struct Registry
+{
+ enum RegistryType {
+ Flag = 0,
+ User = 1,
+ System = 2,
+ Global = 3,
+ };
+
+ RegistryType type;
+
+ struct Entry
+ {
+ Input from, to;
+ Attrs extraAttrs;
+ bool exact = false;
+ };
+
+ std::vector<Entry> entries;
+
+ Registry(RegistryType type)
+ : type(type)
+ { }
+
+ static std::shared_ptr<Registry> read(
+ const Path & path, RegistryType type);
+
+ void write(const Path & path);
+
+ void add(
+ const Input & from,
+ const Input & to,
+ const Attrs & extraAttrs);
+
+ void remove(const Input & input);
+};
+
+typedef std::vector<std::shared_ptr<Registry>> Registries;
+
+std::shared_ptr<Registry> getUserRegistry();
+
+Path getUserRegistryPath();
+
+Registries getRegistries(ref<Store> store);
+
+void overrideRegistry(
+ const Input & from,
+ const Input & to,
+ const Attrs & extraAttrs);
+
+std::pair<Input, Attrs> lookupInRegistries(
+ ref<Store> store,
+ const Input & input);
+
+}
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index f5356f0af..55158cece 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -105,7 +105,7 @@ DownloadFileResult downloadFile(
};
}
-Tree downloadTarball(
+std::pair<Tree, time_t> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
@@ -120,12 +120,9 @@ Tree downloadTarball(
auto cached = getCache()->lookupExpired(store, inAttrs);
if (cached && !cached->expired)
- return Tree {
- .actualPath = store->toRealPath(cached->storePath),
- .storePath = std::move(cached->storePath),
- .info = TreeInfo {
- .lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
- },
+ return {
+ Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
+ getIntAttr(cached->infoAttrs, "lastModified")
};
auto res = downloadFile(store, url, name, immutable);
@@ -160,117 +157,72 @@ Tree downloadTarball(
*unpackedStorePath,
immutable);
- return Tree {
- .actualPath = store->toRealPath(*unpackedStorePath),
- .storePath = std::move(*unpackedStorePath),
- .info = TreeInfo {
- .lastModified = lastModified,
- },
+ return {
+ Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
+ lastModified,
};
}
-struct TarballInput : Input
-{
- ParsedURL url;
- std::optional<Hash> hash;
-
- TarballInput(const ParsedURL & url) : url(url)
- { }
-
- std::string type() const override { return "tarball"; }
-
- bool operator ==(const Input & other) const override
- {
- auto other2 = dynamic_cast<const TarballInput *>(&other);
- return
- other2
- && to_string() == other2->to_string()
- && hash == other2->hash;
- }
-
- bool isImmutable() const override
- {
- return hash || narHash;
- }
-
- ParsedURL toURL() const override
- {
- auto url2(url);
- // NAR hashes are preferred over file hashes since tar/zip files
- // don't have a canonical representation.
- if (narHash)
- url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
- else if (hash)
- url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
- return url2;
- }
-
- Attrs toAttrsInternal() const override
- {
- Attrs attrs;
- attrs.emplace("url", url.to_string());
- if (hash)
- attrs.emplace("hash", hash->to_string(SRI, true));
- return attrs;
- }
-
- std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
- {
- auto tree = downloadTarball(store, url.to_string(), "source", false);
-
- auto input = std::make_shared<TarballInput>(*this);
- input->narHash = store->queryPathInfo(tree.storePath)->narHash;
-
- return {std::move(tree), input};
- }
-};
-
struct TarballInputScheme : InputScheme
{
- std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
+ std::optional<Input> inputFromURL(const ParsedURL & url) override
{
- if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
+ if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {};
if (!hasSuffix(url.path, ".zip")
&& !hasSuffix(url.path, ".tar")
&& !hasSuffix(url.path, ".tar.gz")
&& !hasSuffix(url.path, ".tar.xz")
&& !hasSuffix(url.path, ".tar.bz2"))
- return nullptr;
-
- auto input = std::make_unique<TarballInput>(url);
-
- auto hash = input->url.query.find("hash");
- if (hash != input->url.query.end()) {
- // FIXME: require SRI hash.
- input->hash = Hash(hash->second);
- input->url.query.erase(hash);
- }
-
- auto narHash = input->url.query.find("narHash");
- if (narHash != input->url.query.end()) {
- // FIXME: require SRI hash.
- input->narHash = Hash(narHash->second);
- input->url.query.erase(narHash);
- }
-
+ return {};
+
+ Input input;
+ input.attrs.insert_or_assign("type", "tarball");
+ input.attrs.insert_or_assign("url", url.to_string());
+ auto narHash = url.query.find("narHash");
+ if (narHash != url.query.end())
+ input.attrs.insert_or_assign("narHash", narHash->second);
return input;
}
- std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
+ std::optional<Input> inputFromAttrs(const Attrs & attrs) override
{
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "hash")
+ if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
throw Error("unsupported tarball input attribute '%s'", name);
- auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
- if (auto hash = maybeGetStrAttr(attrs, "hash"))
- input->hash = newHashAllowEmpty(*hash, {});
-
+ Input input;
+ input.attrs = attrs;
+ //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash");
return input;
}
+
+ ParsedURL toURL(const Input & input) override
+ {
+ auto url = parseURL(getStrAttr(input.attrs, "url"));
+ // NAR hashes are preferred over file hashes since tar/zip files
+ // don't have a canonical representation.
+ if (auto narHash = input.getNarHash())
+ url.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
+ /*
+ else if (auto hash = maybeGetStrAttr(input.attrs, "hash"))
+ url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true));
+ */
+ return url;
+ }
+
+ bool hasAllInfo(const Input & input) override
+ {
+ return true;
+ }
+
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+ {
+ auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+ return {std::move(tree), input};
+ }
};
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc
deleted file mode 100644
index 432aa6182..000000000
--- a/src/libfetchers/tree-info.cc
+++ /dev/null
@@ -1,14 +0,0 @@
-#include "tree-info.hh"
-#include "store-api.hh"
-
-#include <nlohmann/json.hpp>
-
-namespace nix::fetchers {
-
-StorePath TreeInfo::computeStorePath(Store & store) const
-{
- assert(narHash);
- return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
-}
-
-}
diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh
deleted file mode 100644
index 9d1872097..000000000
--- a/src/libfetchers/tree-info.hh
+++ /dev/null
@@ -1,29 +0,0 @@
-#pragma once
-
-#include "path.hh"
-#include "hash.hh"
-
-#include <nlohmann/json_fwd.hpp>
-
-namespace nix { class Store; }
-
-namespace nix::fetchers {
-
-struct TreeInfo
-{
- std::optional<Hash> narHash;
- std::optional<uint64_t> revCount;
- std::optional<time_t> lastModified;
-
- bool operator ==(const TreeInfo & other) const
- {
- return
- narHash == other.narHash
- && revCount == other.revCount
- && lastModified == other.lastModified;
- }
-
- StorePath computeStorePath(Store & store) const;
-};
-
-}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 21:07:00 +0000