From e0a0ae0467fa8cdcc542f593b9d94283f04508ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 14:03:28 +0200 Subject: Move fetchers from libstore to libfetchers --- src/libfetchers/attrs.cc | 92 +++++++++ src/libfetchers/attrs.hh | 37 ++++ src/libfetchers/cache.cc | 121 ++++++++++++ src/libfetchers/cache.hh | 34 ++++ src/libfetchers/fetchers.cc | 90 +++++++++ src/libfetchers/fetchers.hh | 119 +++++++++++ src/libfetchers/git.cc | 458 +++++++++++++++++++++++++++++++++++++++++++ src/libfetchers/github.cc | 216 ++++++++++++++++++++ src/libfetchers/indirect.cc | 140 +++++++++++++ src/libfetchers/local.mk | 9 + src/libfetchers/mercurial.cc | 338 +++++++++++++++++++++++++++++++ src/libfetchers/registry.cc | 184 +++++++++++++++++ src/libfetchers/registry.hh | 62 ++++++ src/libfetchers/tarball.cc | 277 ++++++++++++++++++++++++++ src/libfetchers/tree-info.hh | 26 +++ 15 files changed, 2203 insertions(+) create mode 100644 src/libfetchers/attrs.cc create mode 100644 src/libfetchers/attrs.hh create mode 100644 src/libfetchers/cache.cc create mode 100644 src/libfetchers/cache.hh create mode 100644 src/libfetchers/fetchers.cc create mode 100644 src/libfetchers/fetchers.hh create mode 100644 src/libfetchers/git.cc create mode 100644 src/libfetchers/github.cc create mode 100644 src/libfetchers/indirect.cc create mode 100644 src/libfetchers/local.mk create mode 100644 src/libfetchers/mercurial.cc create mode 100644 src/libfetchers/registry.cc create mode 100644 src/libfetchers/registry.hh create mode 100644 src/libfetchers/tarball.cc create mode 100644 src/libfetchers/tree-info.hh (limited to 'src/libfetchers') diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc new file mode 100644 index 000000000..40c02de42 --- /dev/null +++ b/src/libfetchers/attrs.cc @@ -0,0 +1,92 @@ +#include "attrs.hh" +#include "fetchers.hh" + +#include + +namespace nix::fetchers { + +Attrs jsonToAttrs(const nlohmann::json & json) +{ + Attrs attrs; + + for (auto & i : json.items()) { + if (i.value().is_number()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_string()) + attrs.emplace(i.key(), i.value().get()); + else if (i.value().is_boolean()) + attrs.emplace(i.key(), i.value().get()); + else + throw Error("unsupported input attribute type in lock file"); + } + + return attrs; +} + +nlohmann::json attrsToJson(const Attrs & attrs) +{ + nlohmann::json json; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if(&attr.second)) { + json[attr.first] = *v; + } else if (auto v = std::get_if>(&attr.second)) { + json[attr.first] = v->t; + } else abort(); + } + return json; +} + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump()); +} + +std::string getStrAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetStrAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not an integer", name); +} + +int64_t getIntAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetIntAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto i = attrs.find(name); + if (i == attrs.end()) return {}; + if (auto v = std::get_if(&i->second)) + return *v; + throw Error("input attribute '%s' is not a Boolean", name); +} + +bool getBoolAttr(const Attrs & attrs, const std::string & name) +{ + auto s = maybeGetBoolAttr(attrs, name); + if (!s) + throw Error("input attribute '%s' is missing", name); + return *s; +} + +} diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh new file mode 100644 index 000000000..2c9e772d2 --- /dev/null +++ b/src/libfetchers/attrs.hh @@ -0,0 +1,37 @@ +#pragma once + +#include "types.hh" + +#include + +#include + +namespace nix::fetchers { + +/* Wrap bools to prevent string literals (i.e. 'char *') from being + cast to a bool in Attr. */ +template +struct Explicit { + T t; +}; + +typedef std::variant> Attr; +typedef std::map Attrs; + +Attrs jsonToAttrs(const nlohmann::json & json); + +nlohmann::json attrsToJson(const Attrs & attrs); + +std::optional maybeGetStrAttr(const Attrs & attrs, const std::string & name); + +std::string getStrAttr(const Attrs & attrs, const std::string & name); + +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); + +int64_t getIntAttr(const Attrs & attrs, const std::string & name); + +std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name); + +bool getBoolAttr(const Attrs & attrs, const std::string & name); + +} diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc new file mode 100644 index 000000000..e1c7f3dee --- /dev/null +++ b/src/libfetchers/cache.cc @@ -0,0 +1,121 @@ +#include "cache.hh" +#include "sqlite.hh" +#include "sync.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +static const char * schema = R"sql( + +create table if not exists Cache ( + input text not null, + info text not null, + path text not null, + immutable integer not null, + timestamp integer not null, + primary key (input) +); +)sql"; + +struct CacheImpl : Cache +{ + struct State + { + SQLite db; + SQLiteStmt add, lookup; + }; + + Sync _state; + + CacheImpl() + { + auto state(_state.lock()); + + auto dbPath = getCacheDir() + "/nix/fetcher-cache-v1.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->add.create(state->db, + "insert or replace into Cache(input, info, path, immutable, timestamp) values (?, ?, ?, ?, ?)"); + + state->lookup.create(state->db, + "select info, path, immutable, timestamp from Cache where input = ?"); + } + + void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) override + { + _state.lock()->add.use() + (attrsToJson(inAttrs).dump()) + (attrsToJson(infoAttrs).dump()) + (store->printStorePath(storePath)) + (immutable) + (time(0)).exec(); + } + + std::optional> lookup( + ref store, + const Attrs & inAttrs) override + { + if (auto res = lookupExpired(store, inAttrs)) { + if (!res->expired) + return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath)); + debug("ignoring expired cache entry '%s'", + attrsToJson(inAttrs).dump()); + } + return {}; + } + + std::optional lookupExpired( + ref store, + const Attrs & inAttrs) override + { + auto state(_state.lock()); + + auto inAttrsJson = attrsToJson(inAttrs).dump(); + + auto stmt(state->lookup.use()(inAttrsJson)); + if (!stmt.next()) { + debug("did not find cache entry for '%s'", inAttrsJson); + return {}; + } + + auto infoJson = stmt.getStr(0); + auto storePath = store->parseStorePath(stmt.getStr(1)); + auto immutable = stmt.getInt(2) != 0; + auto timestamp = stmt.getInt(3); + + store->addTempRoot(storePath); + if (!store->isValidPath(storePath)) { + // FIXME: we could try to substitute 'storePath'. + debug("ignoring disappeared cache entry '%s'", inAttrsJson); + return {}; + } + + debug("using cache entry '%s' -> '%s', '%s'", + inAttrsJson, infoJson, store->printStorePath(storePath)); + + return Result { + .expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)), + .storePath = std::move(storePath) + }; + } +}; + +ref getCache() +{ + static auto cache = std::make_shared(); + return ref(cache); +} + +} diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh new file mode 100644 index 000000000..d76ab1233 --- /dev/null +++ b/src/libfetchers/cache.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "fetchers.hh" + +namespace nix::fetchers { + +struct Cache +{ + virtual void add( + ref store, + const Attrs & inAttrs, + const Attrs & infoAttrs, + const StorePath & storePath, + bool immutable) = 0; + + virtual std::optional> lookup( + ref store, + const Attrs & inAttrs) = 0; + + struct Result + { + bool expired = false; + Attrs infoAttrs; + StorePath storePath; + }; + + virtual std::optional lookupExpired( + ref store, + const Attrs & inAttrs) = 0; +}; + +ref getCache(); + +} diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc new file mode 100644 index 000000000..5a782a4fe --- /dev/null +++ b/src/libfetchers/fetchers.cc @@ -0,0 +1,90 @@ +#include "fetchers.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::unique_ptr>> inputSchemes = nullptr; + +void registerInputScheme(std::unique_ptr && inputScheme) +{ + if (!inputSchemes) inputSchemes = std::make_unique>>(); + inputSchemes->push_back(std::move(inputScheme)); +} + +std::unique_ptr inputFromURL(const ParsedURL & url) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromURL(url); + if (res) return res; + } + throw Error("input '%s' is unsupported", url.url); +} + +std::unique_ptr inputFromURL(const std::string & url) +{ + return inputFromURL(parseURL(url)); +} + +std::unique_ptr inputFromAttrs(const Attrs & attrs) +{ + for (auto & inputScheme : *inputSchemes) { + auto res = inputScheme->inputFromAttrs(attrs); + if (res) { + if (auto narHash = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + res->narHash = Hash(*narHash); + return res; + } + } + throw Error("input '%s' is unsupported", attrsToJson(attrs)); +} + +Attrs Input::toAttrs() const +{ + auto attrs = toAttrsInternal(); + if (narHash) + attrs.emplace("narHash", narHash->to_string(SRI)); + attrs.emplace("type", type()); + return attrs; +} + +std::pair> Input::fetchTree(ref store) const +{ + auto [tree, input] = fetchTreeInternal(store); + + if (tree.actualPath == "") + tree.actualPath = store->toRealPath(tree.storePath); + + if (!tree.info.narHash) + tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + + if (input->narHash) + assert(input->narHash == tree.info.narHash); + + if (narHash && narHash != input->narHash) + throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); + + return {std::move(tree), input}; +} + +std::shared_ptr Input::applyOverrides( + std::optional ref, + std::optional rev) const +{ + if (ref) + throw Error("don't know how to apply '%s' to '%s'", *ref, to_string()); + if (rev) + throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string()); + return shared_from_this(); +} + +StorePath TreeInfo::computeStorePath(Store & store) const +{ + assert(narHash); + return store.makeFixedOutputPath(true, narHash, "source"); +} + +} diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh new file mode 100644 index 000000000..b75dcffa5 --- /dev/null +++ b/src/libfetchers/fetchers.hh @@ -0,0 +1,119 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "path.hh" +#include "tree-info.hh" +#include "attrs.hh" +#include "url.hh" + +#include + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Input; + +struct Tree +{ + Path actualPath; + StorePath storePath; + TreeInfo info; +}; + +struct Input : std::enable_shared_from_this +{ + std::optional narHash; // FIXME: implement + + virtual std::string type() const = 0; + + virtual ~Input() { } + + virtual bool operator ==(const Input & other) const { return false; } + + /* Check whether this is a "direct" input, that is, not + one that goes through a registry. */ + virtual bool isDirect() const { return true; } + + /* Check whether this is an "immutable" input, that is, + one that contains a commit hash or content hash. */ + virtual bool isImmutable() const { return (bool) narHash; } + + virtual bool contains(const Input & other) const { return false; } + + virtual std::optional getRef() const { return {}; } + + virtual std::optional getRev() const { return {}; } + + virtual ParsedURL toURL() const = 0; + + std::string to_string() const + { + return toURL().to_string(); + } + + Attrs toAttrs() const; + + std::pair> fetchTree(ref store) const; + + virtual std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const; + + virtual std::optional getSourcePath() const { return {}; } + + virtual void markChangedFile( + std::string_view file, + std::optional commitMsg) const + { assert(false); } + + virtual void clone(const Path & destDir) const + { + throw Error("do not know how to clone input '%s'", to_string()); + } + +private: + + virtual std::pair> fetchTreeInternal(ref store) const = 0; + + virtual Attrs toAttrsInternal() const = 0; +}; + +struct InputScheme +{ + virtual ~InputScheme() { } + + virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; + + virtual std::unique_ptr inputFromAttrs(const Attrs & attrs) = 0; +}; + +std::unique_ptr inputFromURL(const ParsedURL & url); + +std::unique_ptr inputFromURL(const std::string & url); + +std::unique_ptr inputFromAttrs(const Attrs & attrs); + +void registerInputScheme(std::unique_ptr && fetcher); + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; +}; + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool immutable); + +Tree downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool immutable); + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc new file mode 100644 index 000000000..e578ed731 --- /dev/null +++ b/src/libfetchers/git.cc @@ -0,0 +1,458 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +static std::string readHead(const Path & path) +{ + return chomp(runProgram("git", true, { "-C", path, "rev-parse", "--abbrev-ref", "HEAD" })); +} + +struct GitInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + bool shallow = false; + + GitInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "git"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + if (shallow) url2.query.insert_or_assign("shallow", "1"); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (shallow) + attrs.emplace("shallow", true); + return attrs; + } + + void clone(const Path & destDir) const override + { + auto [isLocal, actualUrl] = getActualUrl(); + + Strings args = {"clone"}; + + args.push_back(actualUrl); + + if (ref) { + args.push_back("--branch"); + args.push_back(*ref); + } + + if (rev) throw Error("cloning a specific revision is not implemented"); + + args.push_back(destDir); + + runProgram("git", true, args); + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + if (!res->ref && res->rev) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); + + return res; + } + + std::optional getSourcePath() const override + { + if (url.scheme == "file" && !ref && !rev) + return url.path; + return {}; + } + + void markChangedFile(std::string_view file, std::optional commitMsg) const override + { + auto sourcePath = getSourcePath(); + assert(sourcePath); + + runProgram("git", true, + { "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) }); + + if (commitMsg) + runProgram("git", true, + { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); + } + + std::pair getActualUrl() const + { + // Don't clone file:// URIs (but otherwise treat them the + // same as remote URIs, i.e. don't use the working tree or + // HEAD). + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + bool isLocal = url.scheme == "file" && !forceHttp; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + assert(!rev || rev->type == htSHA1); + + auto cacheType = shallow ? "git-shallow" : "git"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", cacheType}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree { + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), + .lastModified = getIntAttr(infoAttrs, "lastModified"), + }, + }, + input + }; + }; + + if (rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input->ref && !input->rev && isLocal) { + bool clean = false; + + /* Check whether this repo has any commits. There are + probably better ways to do this. */ + auto gitDir = actualUrl + "/.git"; + auto commonGitDir = chomp(runProgram( + "git", + true, + { "-C", actualUrl, "rev-parse", "--git-common-dir" } + )); + if (commonGitDir != ".git") + gitDir = commonGitDir; + + bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty(); + + try { + if (haveCommits) { + runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" }); + clean = true; + } + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + } + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked files. */ + + if (!settings.allowDirty) + throw Error("Git tree '%s' is dirty", actualUrl); + + if (settings.warnDirty) + warn("Git tree '%s' is dirty", actualUrl); + + auto files = tokenizeString>( + runProgram("git", true, { "-C", actualUrl, "ls-files", "-z" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + + auto tree = Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, + } + }; + + return {std::move(tree), input}; + } + } + + if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + + Attrs mutableAttrs({ + {"type", cacheType}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + Path repoDir; + + if (isLocal) { + + if (!input->rev) + input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + + repoDir = actualUrl; + + } else { + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); + repoDir = cacheDir; + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", repoDir }); + } + + Path localRefFile = + input->ref->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input->ref + : cacheDir + "/refs/heads/" + *input->ref; + + bool doFetch; + time_t now = time(0); + + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (input->rev) { + try { + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } + + if (doFetch) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + try { + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + } catch (Error & e) { + if (!pathExists(localRefFile)) throw; + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + } + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + if (!input->rev) + input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + } + + bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; + + if (isShallow && !shallow) + throw Error("'%s' is a shallow Git repository, but a non-shallow repository is needed", actualUrl); + + // FIXME: check whether rev is an ancestor of ref. + + printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + + /* Now that we know the ref, check again whether we have it in + the store. */ + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto source = sinkToSource([&](Sink & sink) { + RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + gitOptions.standardOut = &sink; + runProgram2(gitOptions); + }); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + unpackTarfile(*source, tmpDir); + + auto storePath = store->addToStore(name, tmpDir); + + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"lastModified", lastModified}, + }); + + if (!shallow) + infoAttrs.insert_or_assign("revCount", + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct GitInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return nullptr; + + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "git"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "git") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow") + throw Error("unsupported Git input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + + input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc new file mode 100644 index 000000000..505af8af1 --- /dev/null +++ b/src/libfetchers/github.cc @@ -0,0 +1,216 @@ +#include "download.hh" +#include "cache.hh" +#include "fetchers.hh" +#include "globals.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); +std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct GitHubInput : Input +{ + std::string owner; + std::string repo; + std::optional ref; + std::optional rev; + + std::string type() const override { return "github"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && owner == other2->owner + && repo == other2->repo + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + auto path = owner + "/" + repo; + assert(!(ref && rev)); + if (ref) path += "/" + *ref; + if (rev) path += "/" + rev->to_string(Base16, false); + return ParsedURL { + .scheme = "github", + .path = path, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("owner", owner); + attrs.emplace("repo", repo); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + void clone(const Path & destDir) const override + { + std::shared_ptr input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); + input = input->applyOverrides(ref.value_or("master"), rev); + input->clone(destDir); + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto rev = this->rev; + auto ref = this->ref.value_or("master"); + + if (!rev) { + auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", + owner, repo, ref); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false).storePath))); + rev = Hash(json["sha"], htSHA1); + debug("HEAD revision for '%s' is %s", url, rev->gitRev()); + } + + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = *rev; + + Attrs immutableAttrs({ + {"type", "git-tarball"}, + {"rev", rev->gitRev()}, + }); + + if (auto res = getCache()->lookup(store, immutableAttrs)) { + return { + Tree{ + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }, + input + }; + } + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + owner, repo, rev->to_string(Base16, false)); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + auto tree = downloadTarball(store, url, "source", true); + + getCache()->add( + store, + immutableAttrs, + { + {"rev", rev->gitRev()}, + {"lastModified", *tree.info.lastModified} + }, + tree.storePath, + true); + + return {std::move(tree), input}; + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } +}; + +struct GitHubInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "github") return nullptr; + + auto path = tokenizeString>(url.path, "/"); + auto input = std::make_unique(); + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + input->rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + input->ref = path[2]; + else + throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (input->rev) + throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); + if (input->ref) + throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); + input->ref = value; + } + } + + if (input->ref && input->rev) + throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); + + input->owner = path[0]; + input->repo = path[1]; + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "github") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") + throw Error("unsupported GitHub input attribute '%s'", name); + + auto input = std::make_unique(); + input->owner = getStrAttr(attrs, "owner"); + input->repo = getStrAttr(attrs, "repo"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc new file mode 100644 index 000000000..380b69fe0 --- /dev/null +++ b/src/libfetchers/indirect.cc @@ -0,0 +1,140 @@ +#include "fetchers.hh" + +namespace nix::fetchers { + +std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct IndirectInput : Input +{ + std::string id; + std::optional rev; + std::optional ref; + + std::string type() const override { return "indirect"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && id == other2->id + && rev == other2->rev + && ref == other2->ref; + } + + bool isDirect() const override + { + return false; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + bool contains(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && id == other2->id + && (!ref || ref == other2->ref) + && (!rev || rev == other2->rev); + } + + ParsedURL toURL() const override + { + ParsedURL url; + url.scheme = "flake"; + url.path = id; + if (ref) { url.path += '/'; url.path += *ref; }; + if (rev) { url.path += '/'; url.path += rev->gitRev(); }; + return url; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("id", id); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + throw Error("indirect input '%s' cannot be fetched directly", to_string()); + } +}; + +struct IndirectInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "flake") return nullptr; + + auto path = tokenizeString>(url.path, "/"); + auto input = std::make_unique(); + + if (path.size() == 1) { + } else if (path.size() == 2) { + if (std::regex_match(path[1], revRegex)) + input->rev = Hash(path[1], htSHA1); + else if (std::regex_match(path[1], refRegex)) + input->ref = path[1]; + else + throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); + } else if (path.size() == 3) { + if (!std::regex_match(path[1], refRegex)) + throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); + input->ref = path[1]; + if (!std::regex_match(path[2], revRegex)) + throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); + input->rev = Hash(path[2], htSHA1); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + // FIXME: forbid query params? + + input->id = path[0]; + if (!std::regex_match(input->id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", input->id); + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "id" && name != "ref" && name != "rev") + throw Error("unsupported indirect input attribute '%s'", name); + + auto input = std::make_unique(); + input->id = getStrAttr(attrs, "id"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk new file mode 100644 index 000000000..4f3d4e85a --- /dev/null +++ b/src/libfetchers/local.mk @@ -0,0 +1,9 @@ +libraries += libfetchers + +libfetchers_NAME = libnixfetchers + +libfetchers_DIR := $(d) + +libfetchers_SOURCES := $(wildcard $(d)/*.cc) + +libfetchers_LIBS = libutil libstore libnixrust diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc new file mode 100644 index 000000000..4b73fbcbe --- /dev/null +++ b/src/libfetchers/mercurial.cc @@ -0,0 +1,338 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "globals.hh" +#include "tarfile.hh" +#include "store-api.hh" + +#include + +using namespace std::string_literals; + +namespace nix::fetchers { + +struct MercurialInput : Input +{ + ParsedURL url; + std::optional ref; + std::optional rev; + + MercurialInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "hg"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && url == other2->url + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + ParsedURL url2(url); + url2.scheme = "hg+" + url2.scheme; + if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); + if (ref) url2.query.insert_or_assign("ref", *ref); + return url; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } + + std::optional getSourcePath() const + { + if (url.scheme == "file" && !ref && !rev) + return url.path; + return {}; + } + + void markChangedFile(std::string_view file, std::optional commitMsg) const override + { + auto sourcePath = getSourcePath(); + assert(sourcePath); + + // FIXME: shut up if file is already tracked. + runProgram("hg", true, + { "add", *sourcePath + "/" + std::string(file) }); + + if (commitMsg) + runProgram("hg", true, + { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); + } + + std::pair getActualUrl() const + { + bool isLocal = url.scheme == "file"; + return {isLocal, isLocal ? url.path : url.base}; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto name = "source"; + + auto input = std::make_shared(*this); + + auto [isLocal, actualUrl_] = getActualUrl(); + auto actualUrl = actualUrl_; // work around clang bug + + // FIXME: return lastModified. + + // FIXME: don't clone local repositories. + + if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + + bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked + files. */ + + if (!settings.allowDirty) + throw Error("Mercurial tree '%s' is unclean", actualUrl); + + if (settings.warnDirty) + warn("Mercurial tree '%s' is unclean", actualUrl); + + input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + + auto files = tokenizeString>( + runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, actualUrl)); + std::string file(p, actualUrl.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); + + return {Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input}; + } + } + + if (!input->ref) input->ref = "default"; + + auto getImmutableAttrs = [&]() + { + return Attrs({ + {"type", "hg"}, + {"name", name}, + {"rev", input->rev->gitRev()}, + }); + }; + + auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) + -> std::pair> + { + assert(input->rev); + assert(!rev || rev == input->rev); + return { + Tree{ + .actualPath = store->toRealPath(storePath), + .storePath = std::move(storePath), + .info = TreeInfo { + .revCount = getIntAttr(infoAttrs, "revCount"), + }, + }, + input + }; + }; + + if (input->rev) { + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + } + + assert(input->rev || input->ref); + auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + + Attrs mutableAttrs({ + {"type", "hg"}, + {"name", name}, + {"url", actualUrl}, + {"ref", *input->ref}, + }); + + if (auto res = getCache()->lookup(store, mutableAttrs)) { + auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); + if (!rev || rev == rev2) { + input->rev = rev2; + return makeResult(res->first, std::move(res->second)); + } + } + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + + /* If this is a commit hash that we already have, we don't + have to pull again. */ + if (!(input->rev + && pathExists(cacheDir) + && runProgram( + RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + .killStderr(true)).second == "1")) + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } + catch (ExecError & e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, { "recover", "-R", cacheDir }); + runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl }); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + } + } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir }); + } + } + + auto tokens = tokenizeString>( + runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); + assert(tokens.size() == 3); + + input->rev = Hash(tokens[0], htSHA1); + auto revCount = std::stoull(tokens[1]); + input->ref = tokens[2]; + + if (auto res = getCache()->lookup(store, getImmutableAttrs())) + return makeResult(res->first, std::move(res->second)); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + + deletePath(tmpDir + "/.hg_archival.txt"); + + auto storePath = store->addToStore(name, tmpDir); + + Attrs infoAttrs({ + {"rev", input->rev->gitRev()}, + {"revCount", (int64_t) revCount}, + }); + + if (!this->rev) + getCache()->add( + store, + mutableAttrs, + infoAttrs, + storePath, + false); + + getCache()->add( + store, + getImmutableAttrs(), + infoAttrs, + storePath, + true); + + return makeResult(infoAttrs, std::move(storePath)); + } +}; + +struct MercurialInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return nullptr; + + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); + + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev") + throw Error("unsupported Mercurial input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + input->ref = *ref; + } + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc new file mode 100644 index 000000000..34e63180c --- /dev/null +++ b/src/libfetchers/registry.cc @@ -0,0 +1,184 @@ +#include "registry.hh" +#include "fetchers.hh" +#include "util.hh" +#include "globals.hh" +#include "download.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +std::shared_ptr Registry::read( + const Path & path, RegistryType type) +{ + auto registry = std::make_shared(type); + + if (!pathExists(path)) + return std::make_shared(type); + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + + // FIXME: remove soon + if (version == 1) { + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + auto url = i->value("url", i->value("uri", "")); + if (url.empty()) + throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", + path, i.key()); + registry->entries.push_back( + {inputFromURL(i.key()), inputFromURL(url), {}}); + } + } + + else if (version == 2) { + for (auto & i : json["flakes"]) { + auto toAttrs = jsonToAttrs(i["to"]); + Attrs extraAttrs; + auto j = toAttrs.find("dir"); + if (j != toAttrs.end()) { + extraAttrs.insert(*j); + toAttrs.erase(j); + } + registry->entries.push_back( + { inputFromAttrs(jsonToAttrs(i["from"])) + , inputFromAttrs(toAttrs) + , extraAttrs + }); + } + } + + else + throw Error("flake registry '%s' has unsupported version %d", path, version); + + + return registry; +} + +void Registry::write(const Path & path) +{ + nlohmann::json arr; + for (auto & elem : entries) { + nlohmann::json obj; + obj["from"] = attrsToJson(std::get<0>(elem)->toAttrs()); + obj["to"] = attrsToJson(std::get<1>(elem)->toAttrs()); + if (!std::get<2>(elem).empty()) + obj["to"].update(attrsToJson(std::get<2>(elem))); + arr.emplace_back(std::move(obj)); + } + + nlohmann::json json; + json["version"] = 2; + json["flakes"] = std::move(arr); + + createDirs(dirOf(path)); + writeFile(path, json.dump(2)); +} + +void Registry::add( + const std::shared_ptr & from, + const std::shared_ptr & to, + const Attrs & extraAttrs) +{ + entries.emplace_back(from, to, extraAttrs); +} + +void Registry::remove(const std::shared_ptr & input) +{ + // FIXME: use C++20 std::erase. + for (auto i = entries.begin(); i != entries.end(); ) + if (*std::get<0>(*i) == *input) + i = entries.erase(i); + else + ++i; +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr getUserRegistry() +{ + return Registry::read(getUserRegistryPath(), Registry::User); +} + +static std::shared_ptr flagRegistry = + std::make_shared(Registry::Flag); + +std::shared_ptr getFlagRegistry() +{ + return flagRegistry; +} + +void overrideRegistry( + const std::shared_ptr & from, + const std::shared_ptr & to, + const Attrs & extraAttrs) +{ + flagRegistry->add(from, to, extraAttrs); +} + +static std::shared_ptr getGlobalRegistry(ref store) +{ + static auto reg = [&]() { + auto path = settings.flakeRegistry; + + if (!hasPrefix(path, "/")) { + auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + if (auto store2 = store.dynamic_pointer_cast()) + store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true); + path = store->toRealPath(storePath); + } + + return Registry::read(path, Registry::Global); + }(); + + return reg; +} + +Registries getRegistries(ref store) +{ + Registries registries; + registries.push_back(getFlagRegistry()); + registries.push_back(getUserRegistry()); + registries.push_back(getGlobalRegistry(store)); + return registries; +} + +std::pair, Attrs> lookupInRegistries( + ref store, + std::shared_ptr input) +{ + Attrs extraAttrs; + int n = 0; + + restart: + + n++; + if (n > 100) throw Error("cycle detected in flake registr for '%s'", input); + + for (auto & registry : getRegistries(store)) { + // FIXME: O(n) + for (auto & entry : registry->entries) { + auto from = std::get<0>(entry); + if (from->contains(*input)) { + input = std::get<1>(entry)->applyOverrides( + !from->getRef() && input->getRef() ? input->getRef() : std::optional(), + !from->getRev() && input->getRev() ? input->getRev() : std::optional()); + extraAttrs = std::get<2>(entry); + goto restart; + } + } + } + + if (!input->isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input->to_string()); + + return {input, extraAttrs}; +} + +} diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh new file mode 100644 index 000000000..d2eb7749b --- /dev/null +++ b/src/libfetchers/registry.hh @@ -0,0 +1,62 @@ +#pragma once + +#include "types.hh" +#include "fetchers.hh" + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Registry +{ + enum RegistryType { + Flag = 0, + User = 1, + Global = 2, + }; + + RegistryType type; + + std::vector< + std::tuple< + std::shared_ptr, // from + std::shared_ptr, // to + Attrs // extra attributes + > + > entries; + + Registry(RegistryType type) + : type(type) + { } + + static std::shared_ptr read( + const Path & path, RegistryType type); + + void write(const Path & path); + + void add( + const std::shared_ptr & from, + const std::shared_ptr & to, + const Attrs & extraAttrs); + + void remove(const std::shared_ptr & input); +}; + +typedef std::vector> Registries; + +std::shared_ptr getUserRegistry(); + +Path getUserRegistryPath(); + +Registries getRegistries(ref store); + +void overrideRegistry( + const std::shared_ptr & from, + const std::shared_ptr & to, + const Attrs & extraAttrs); + +std::pair, Attrs> lookupInRegistries( + ref store, + std::shared_ptr input); + +} diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc new file mode 100644 index 000000000..b0a83001e --- /dev/null +++ b/src/libfetchers/tarball.cc @@ -0,0 +1,277 @@ +#include "fetchers.hh" +#include "cache.hh" +#include "download.hh" +#include "globals.hh" +#include "store-api.hh" +#include "archive.hh" +#include "tarfile.hh" + +namespace nix::fetchers { + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool immutable) +{ + // FIXME: check store + + Attrs inAttrs({ + {"type", "file"}, + {"url", url}, + {"name", name}, + }); + + auto cached = getCache()->lookupExpired(store, inAttrs); + + auto useCached = [&]() -> DownloadFileResult + { + return { + .storePath = std::move(cached->storePath), + .etag = getStrAttr(cached->infoAttrs, "etag"), + .effectiveUrl = getStrAttr(cached->infoAttrs, "url") + }; + }; + + if (cached && !cached->expired) + return useCached(); + + DownloadRequest request(url); + if (cached) + request.expectedETag = getStrAttr(cached->infoAttrs, "etag"); + DownloadResult res; + try { + res = getDownloader()->download(request); + } catch (DownloadError & e) { + if (cached) { + warn("%s; using cached version", e.msg()); + return useCached(); + } else + throw; + } + + // FIXME: write to temporary file. + + Attrs infoAttrs({ + {"etag", res.etag}, + {"url", res.effectiveUri}, + }); + + std::optional storePath; + + if (res.cached) { + assert(cached); + assert(request.expectedETag == res.etag); + storePath = std::move(cached->storePath); + } else { + StringSink sink; + dumpString(*res.data, sink); + auto hash = hashString(htSHA256, *res.data); + ValidPathInfo info(store->makeFixedOutputPath(false, hash, name)); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(false, hash); + store->addToStore(info, sink.s, NoRepair, NoCheckSigs); + storePath = std::move(info.path); + } + + getCache()->add( + store, + inAttrs, + infoAttrs, + *storePath, + immutable); + + if (url != res.effectiveUri) + getCache()->add( + store, + { + {"type", "file"}, + {"url", res.effectiveUri}, + {"name", name}, + }, + infoAttrs, + *storePath, + immutable); + + return { + .storePath = std::move(*storePath), + .etag = res.etag, + .effectiveUrl = res.effectiveUri, + }; +} + +Tree downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool immutable) +{ + Attrs inAttrs({ + {"type", "tarball"}, + {"url", url}, + {"name", name}, + }); + + auto cached = getCache()->lookupExpired(store, inAttrs); + + if (cached && !cached->expired) + return Tree { + .actualPath = store->toRealPath(cached->storePath), + .storePath = std::move(cached->storePath), + .info = TreeInfo { + .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), + }, + }; + + auto res = downloadFile(store, url, name, immutable); + + std::optional unpackedStorePath; + time_t lastModified; + + if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { + unpackedStorePath = std::move(cached->storePath); + lastModified = getIntAttr(cached->infoAttrs, "lastModified"); + } else { + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + unpackTarfile(store->toRealPath(res.storePath), tmpDir); + auto members = readDirectory(tmpDir); + if (members.size() != 1) + throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); + auto topDir = tmpDir + "/" + members.begin()->name; + lastModified = lstat(topDir).st_mtime; + unpackedStorePath = store->addToStore(name, topDir, true, htSHA256, defaultPathFilter, NoRepair); + } + + Attrs infoAttrs({ + {"lastModified", lastModified}, + {"etag", res.etag}, + }); + + getCache()->add( + store, + inAttrs, + infoAttrs, + *unpackedStorePath, + immutable); + + return Tree { + .actualPath = store->toRealPath(*unpackedStorePath), + .storePath = std::move(*unpackedStorePath), + .info = TreeInfo { + .lastModified = lastModified, + }, + }; +} + +struct TarballInput : Input +{ + ParsedURL url; + std::optional hash; + + TarballInput(const ParsedURL & url) : url(url) + { } + + std::string type() const override { return "tarball"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && to_string() == other2->to_string() + && hash == other2->hash; + } + + bool isImmutable() const override + { + return hash || narHash; + } + + ParsedURL toURL() const override + { + auto url2(url); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (narHash) + url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); + else if (hash) + url2.query.insert_or_assign("hash", hash->to_string(SRI)); + return url2; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("url", url.to_string()); + if (narHash) + attrs.emplace("narHash", narHash->to_string(SRI)); + else if (hash) + attrs.emplace("hash", hash->to_string(SRI)); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto tree = downloadTarball(store, url.to_string(), "source", false); + + auto input = std::make_shared(*this); + input->narHash = store->queryPathInfo(tree.storePath)->narHash; + + return {std::move(tree), input}; + } +}; + +struct TarballInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + + if (!hasSuffix(url.path, ".zip") + && !hasSuffix(url.path, ".tar") + && !hasSuffix(url.path, ".tar.gz") + && !hasSuffix(url.path, ".tar.xz") + && !hasSuffix(url.path, ".tar.bz2")) + return nullptr; + + auto input = std::make_unique(url); + + auto hash = input->url.query.find("hash"); + if (hash != input->url.query.end()) { + // FIXME: require SRI hash. + input->hash = Hash(hash->second); + input->url.query.erase(hash); + } + + auto narHash = input->url.query.find("narHash"); + if (narHash != input->url.query.end()) { + // FIXME: require SRI hash. + input->narHash = Hash(narHash->second); + input->url.query.erase(narHash); + } + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "hash" && name != "narHash") + throw Error("unsupported tarball input attribute '%s'", name); + + auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); + if (auto hash = maybeGetStrAttr(attrs, "hash")) + // FIXME: require SRI hash. + input->hash = Hash(*hash); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh new file mode 100644 index 000000000..02e92759b --- /dev/null +++ b/src/libfetchers/tree-info.hh @@ -0,0 +1,26 @@ +#pragma once + +#include "path.hh" + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct TreeInfo +{ + Hash narHash; + std::optional revCount; + std::optional lastModified; + + bool operator ==(const TreeInfo & other) const + { + return + narHash == other.narHash + && revCount == other.revCount + && lastModified == other.lastModified; + } + + StorePath computeStorePath(Store & store) const; +}; + +} -- cgit v1.2.3 From 2c692a3b144523bca68dd6de618124ba6c9bb332 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Mar 2020 14:29:29 +0200 Subject: Remove global -I flags --- src/libfetchers/local.mk | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libfetchers') diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index 4f3d4e85a..d7143d8a6 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -6,4 +6,6 @@ libfetchers_DIR := $(d) libfetchers_SOURCES := $(wildcard $(d)/*.cc) +libfetchers_CXXFLAGS += -I src/libutil -I src/libstore + libfetchers_LIBS = libutil libstore libnixrust -- cgit v1.2.3 From 77ffaea4fa12f0525758d6c4a3a3bd074906978d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Apr 2020 22:56:50 +0200 Subject: Add a system-wide flake registry /etc/nix/registry.json One application for this is pinning the 'nixpkgs' flake to the exact revision used to build the NixOS system, e.g. { "flakes": [ { "from": { "id": "nixpkgs", "type": "indirect" }, "to": { "owner": "NixOS", "repo": "nixpkgs", "type": "github", "rev": "b0c285807d6a9f1b7562ec417c24fa1a30ecc31a" } } ], "version": 2 } --- src/libfetchers/registry.cc | 17 ++++++++++++++++- src/libfetchers/registry.hh | 3 ++- 2 files changed, 18 insertions(+), 2 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 34e63180c..08dbe797e 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -96,6 +96,18 @@ void Registry::remove(const std::shared_ptr & input) ++i; } +static Path getSystemRegistryPath() +{ + return settings.nixConfDir + "/registry.json"; +} + +static std::shared_ptr getSystemRegistry() +{ + static auto systemRegistry = + Registry::read(getSystemRegistryPath(), Registry::System); + return systemRegistry; +} + Path getUserRegistryPath() { return getHome() + "/.config/nix/registry.json"; @@ -103,7 +115,9 @@ Path getUserRegistryPath() std::shared_ptr getUserRegistry() { - return Registry::read(getUserRegistryPath(), Registry::User); + static auto userRegistry = + Registry::read(getUserRegistryPath(), Registry::User); + return userRegistry; } static std::shared_ptr flagRegistry = @@ -145,6 +159,7 @@ Registries getRegistries(ref store) Registries registries; registries.push_back(getFlagRegistry()); registries.push_back(getUserRegistry()); + registries.push_back(getSystemRegistry()); registries.push_back(getGlobalRegistry(store)); return registries; } diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index d2eb7749b..5e8857850 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -12,7 +12,8 @@ struct Registry enum RegistryType { Flag = 0, User = 1, - Global = 2, + System = 2, + Global = 3, }; RegistryType type; -- cgit v1.2.3 From bd10a07d17161fb4a4e1af5aa365b23d405a5216 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Apr 2020 23:03:27 +0200 Subject: Registry: Use a struct instead of a tuple for entries --- src/libfetchers/registry.cc | 37 +++++++++++++++++++++---------------- src/libfetchers/registry.hh | 15 ++++++++------- 2 files changed, 29 insertions(+), 23 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 08dbe797e..6627d3725 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -44,9 +44,10 @@ std::shared_ptr Registry::read( toAttrs.erase(j); } registry->entries.push_back( - { inputFromAttrs(jsonToAttrs(i["from"])) - , inputFromAttrs(toAttrs) - , extraAttrs + Entry { + .from = inputFromAttrs(jsonToAttrs(i["from"])), + .to = inputFromAttrs(toAttrs), + .extraAttrs = extraAttrs, }); } } @@ -61,12 +62,12 @@ std::shared_ptr Registry::read( void Registry::write(const Path & path) { nlohmann::json arr; - for (auto & elem : entries) { + for (auto & entry : entries) { nlohmann::json obj; - obj["from"] = attrsToJson(std::get<0>(elem)->toAttrs()); - obj["to"] = attrsToJson(std::get<1>(elem)->toAttrs()); - if (!std::get<2>(elem).empty()) - obj["to"].update(attrsToJson(std::get<2>(elem))); + obj["from"] = attrsToJson(entry.from->toAttrs()); + obj["to"] = attrsToJson(entry.to->toAttrs()); + if (!entry.extraAttrs.empty()) + obj["to"].update(attrsToJson(entry.extraAttrs)); arr.emplace_back(std::move(obj)); } @@ -83,14 +84,19 @@ void Registry::add( const std::shared_ptr & to, const Attrs & extraAttrs) { - entries.emplace_back(from, to, extraAttrs); + entries.emplace_back( + Entry { + .from = from, + .to = to, + .extraAttrs = extraAttrs + }); } void Registry::remove(const std::shared_ptr & input) { // FIXME: use C++20 std::erase. for (auto i = entries.begin(); i != entries.end(); ) - if (*std::get<0>(*i) == *input) + if (*i->from == *input) i = entries.erase(i); else ++i; @@ -179,12 +185,11 @@ std::pair, Attrs> lookupInRegistries( for (auto & registry : getRegistries(store)) { // FIXME: O(n) for (auto & entry : registry->entries) { - auto from = std::get<0>(entry); - if (from->contains(*input)) { - input = std::get<1>(entry)->applyOverrides( - !from->getRef() && input->getRef() ? input->getRef() : std::optional(), - !from->getRev() && input->getRev() ? input->getRev() : std::optional()); - extraAttrs = std::get<2>(entry); + if (entry.from->contains(*input)) { + input = entry.to->applyOverrides( + !entry.from->getRef() && input->getRef() ? input->getRef() : std::optional(), + !entry.from->getRev() && input->getRev() ? input->getRev() : std::optional()); + extraAttrs = entry.extraAttrs; goto restart; } } diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 5e8857850..722c41b10 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -18,13 +18,14 @@ struct Registry RegistryType type; - std::vector< - std::tuple< - std::shared_ptr, // from - std::shared_ptr, // to - Attrs // extra attributes - > - > entries; + struct Entry + { + std::shared_ptr from; + std::shared_ptr to; + Attrs extraAttrs; + }; + + std::vector entries; Registry(RegistryType type) : type(type) -- cgit v1.2.3 From 74024515a306a382f11058d4320b80897ebe09a0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Apr 2020 23:12:45 +0200 Subject: Support registry entries that must match exactly An example use is for pinning the "nixpkgs" entry the system-wide registry to a particular store path. Inexact matches (e.g. "nixpkgs/master") should still use the global registry. --- src/libfetchers/registry.cc | 24 ++++++++++++++++++------ src/libfetchers/registry.hh | 1 + 2 files changed, 19 insertions(+), 6 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 6627d3725..e452e8497 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -43,11 +43,13 @@ std::shared_ptr Registry::read( extraAttrs.insert(*j); toAttrs.erase(j); } + auto exact = i.find("exact"); registry->entries.push_back( Entry { .from = inputFromAttrs(jsonToAttrs(i["from"])), .to = inputFromAttrs(toAttrs), .extraAttrs = extraAttrs, + .exact = exact != i.end() && exact.value() }); } } @@ -68,6 +70,8 @@ void Registry::write(const Path & path) obj["to"] = attrsToJson(entry.to->toAttrs()); if (!entry.extraAttrs.empty()) obj["to"].update(attrsToJson(entry.extraAttrs)); + if (entry.exact) + obj["exact"] = true; arr.emplace_back(std::move(obj)); } @@ -185,12 +189,20 @@ std::pair, Attrs> lookupInRegistries( for (auto & registry : getRegistries(store)) { // FIXME: O(n) for (auto & entry : registry->entries) { - if (entry.from->contains(*input)) { - input = entry.to->applyOverrides( - !entry.from->getRef() && input->getRef() ? input->getRef() : std::optional(), - !entry.from->getRev() && input->getRev() ? input->getRev() : std::optional()); - extraAttrs = entry.extraAttrs; - goto restart; + if (entry.exact) { + if (*entry.from == *input) { + input = entry.to; + extraAttrs = entry.extraAttrs; + goto restart; + } + } else { + if (entry.from->contains(*input)) { + input = entry.to->applyOverrides( + !entry.from->getRef() && input->getRef() ? input->getRef() : std::optional(), + !entry.from->getRev() && input->getRev() ? input->getRev() : std::optional()); + extraAttrs = entry.extraAttrs; + goto restart; + } } } } diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 722c41b10..c3ce948a8 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -23,6 +23,7 @@ struct Registry std::shared_ptr from; std::shared_ptr to; Attrs extraAttrs; + bool exact = false; }; std::vector entries; -- cgit v1.2.3 From ed13457dbf1a78f47f760e349985b23dfd5f16c4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 11:51:34 +0200 Subject: nix flake info --json: Show TreeInfo --- src/libfetchers/fetchers.cc | 6 ------ src/libfetchers/tree-info.hh | 5 +++++ 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5a782a4fe..765a6585f 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -81,10 +81,4 @@ std::shared_ptr Input::applyOverrides( return shared_from_this(); } -StorePath TreeInfo::computeStorePath(Store & store) const -{ - assert(narHash); - return store.makeFixedOutputPath(true, narHash, "source"); -} - } diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh index 02e92759b..25cee445e 100644 --- a/src/libfetchers/tree-info.hh +++ b/src/libfetchers/tree-info.hh @@ -1,6 +1,9 @@ #pragma once #include "path.hh" +#include "hash.hh" + +#include namespace nix { class Store; } @@ -21,6 +24,8 @@ struct TreeInfo } StorePath computeStorePath(Store & store) const; + + nlohmann::json toJson() const; }; } -- cgit v1.2.3 From 00e1400eb741d4f5329a6bcfdc8946617967dcb1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 11:55:41 +0200 Subject: Doh --- src/libfetchers/tree-info.cc | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/libfetchers/tree-info.cc (limited to 'src/libfetchers') diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc new file mode 100644 index 000000000..50795e162 --- /dev/null +++ b/src/libfetchers/tree-info.cc @@ -0,0 +1,26 @@ +#include "tree-info.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +StorePath TreeInfo::computeStorePath(Store & store) const +{ + assert(narHash); + return store.makeFixedOutputPath(true, narHash, "source"); +} + +nlohmann::json TreeInfo::toJson() const +{ + nlohmann::json json; + assert(narHash); + json["narHash"] = narHash.to_string(SRI); + if (revCount) + json["revCount"] = *revCount; + if (lastModified) + json["lastModified"] = *lastModified; + return json; +} + +} -- cgit v1.2.3 From 12f9379123eba828f2ae06f7978a37b7045c2b23 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 14:56:20 +0200 Subject: Add 'path' fetcher This fetchers copies a plain directory (i.e. not a Git/Mercurial repository) to the store (or does nothing if the path is already a store path). One use case is to pin the 'nixpkgs' flake used to build the current NixOS system, and prevent it from being garbage-collected, via a system registry entry like this: { "from": { "id": "nixpkgs", "type": "indirect" }, "to": { "type": "path", "path": "/nix/store/rralhl3wj4rdwzjn16g7d93mibvlr521-source", "lastModified": 1585388205, "rev": "b0c285807d6a9f1b7562ec417c24fa1a30ecc31a" }, "exact": true } Note the fake "lastModified" and "rev" attributes that ensure that the flake gives the same evaluation results as the corresponding Git/GitHub inputs. --- src/libfetchers/attrs.cc | 15 ++++++ src/libfetchers/attrs.hh | 2 + src/libfetchers/path.cc | 130 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 src/libfetchers/path.cc (limited to 'src/libfetchers') diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 40c02de42..feb0a6085 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -89,4 +89,19 @@ bool getBoolAttr(const Attrs & attrs, const std::string & name) return *s; } +std::map attrsToQuery(const Attrs & attrs) +{ + std::map query; + for (auto & attr : attrs) { + if (auto v = std::get_if(&attr.second)) { + query.insert_or_assign(attr.first, fmt("%d", *v)); + } else if (auto v = std::get_if(&attr.second)) { + query.insert_or_assign(attr.first, *v); + } else if (auto v = std::get_if>(&attr.second)) { + query.insert_or_assign(attr.first, v->t ? "1" : "0"); + } else abort(); + } + return query; +} + } diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 2c9e772d2..d6e0ae000 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -34,4 +34,6 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na bool getBoolAttr(const Attrs & attrs, const std::string & name); +std::map attrsToQuery(const Attrs & attrs); + } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc new file mode 100644 index 000000000..9801d9b86 --- /dev/null +++ b/src/libfetchers/path.cc @@ -0,0 +1,130 @@ +#include "fetchers.hh" +#include "store-api.hh" + +namespace nix::fetchers { + +struct PathInput : Input +{ + Path path; + + /* Allow the user to pass in "fake" tree info attributes. This is + useful for making a pinned tree work the same as the repository + from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + std::optional rev; + std::optional revCount; + std::optional lastModified; + + std::string type() const override { return "path"; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + auto query = attrsToQuery(toAttrsInternal()); + query.erase("path"); + return ParsedURL { + .scheme = "path", + .path = path, + .query = query, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("path", path); + if (rev) + attrs.emplace("rev", rev->gitRev()); + if (revCount) + attrs.emplace("revCount", *revCount); + if (lastModified) + attrs.emplace("lastModified", *lastModified); + return attrs; + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto input = std::make_shared(*this); + + auto storePath = store->maybeParseStorePath(path); + + if (storePath) + store->addTempRoot(*storePath); + + if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) + // FIXME: try to substitute storePath. + storePath = store->addToStore("name", path); + + return + { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + .info = TreeInfo { + .revCount = revCount, + .lastModified = lastModified + } + }, + input + }; + } + +}; + +struct PathInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "path") return nullptr; + + auto input = std::make_unique(); + input->path = url.path; + + for (auto & [name, value] : url.query) + if (name == "rev") + input->rev = Hash(value, htSHA1); + else if (name == "revCount") { + uint64_t revCount; + if (!string2Int(value, revCount)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input->revCount = revCount; + } + else if (name == "lastModified") { + time_t lastModified; + if (!string2Int(value, lastModified)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input->lastModified = lastModified; + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + auto input = std::make_unique(); + input->path = getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + if (name == "rev") + input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); + else if (name == "revCount") + input->revCount = getIntAttr(attrs, "revCount"); + else if (name == "lastModified") + input->lastModified = getIntAttr(attrs, "lastModified"); + else if (name == "type" || name == "path") + ; + else + throw Error("unsupported path input attribute '%s'", name); + + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} -- cgit v1.2.3 From 6d6467d376ff8ccc758c9fce1fe6d9b658956598 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 18:26:39 +0200 Subject: Move parseTreeInfo() --- src/libfetchers/tree-info.cc | 34 ++++++++++++++++++++++++++++++++++ src/libfetchers/tree-info.hh | 2 ++ 2 files changed, 36 insertions(+) (limited to 'src/libfetchers') diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index 50795e162..42a19cbc8 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -11,6 +11,40 @@ StorePath TreeInfo::computeStorePath(Store & store) const return store.makeFixedOutputPath(true, narHash, "source"); } +TreeInfo TreeInfo::fromJson(const nlohmann::json & json) +{ + TreeInfo info; + + auto i = json.find("info"); + if (i != json.end()) { + const nlohmann::json & i2(*i); + + auto j = i2.find("narHash"); + if (j != i2.end()) + info.narHash = Hash((std::string) *j); + else + throw Error("attribute 'narHash' missing in lock file"); + + j = i2.find("revCount"); + if (j != i2.end()) + info.revCount = *j; + + j = i2.find("lastModified"); + if (j != i2.end()) + info.lastModified = *j; + + return info; + } + + i = json.find("narHash"); + if (i != json.end()) { + info.narHash = Hash((std::string) *i); + return info; + } + + throw Error("attribute 'info' missing in lock file"); +} + nlohmann::json TreeInfo::toJson() const { nlohmann::json json; diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh index 25cee445e..3b62151c6 100644 --- a/src/libfetchers/tree-info.hh +++ b/src/libfetchers/tree-info.hh @@ -25,6 +25,8 @@ struct TreeInfo StorePath computeStorePath(Store & store) const; + static TreeInfo fromJson(const nlohmann::json & json); + nlohmann::json toJson() const; }; -- cgit v1.2.3 From 78ad5b3d91507427fa563f3474dc52da608ad224 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 19:04:27 +0200 Subject: PathInput: Add some methods --- src/libfetchers/path.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src/libfetchers') diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 9801d9b86..037404726 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -19,6 +19,22 @@ struct PathInput : Input std::optional getRev() const override { return rev; } + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && path == other2->path + && rev == other2->rev + && revCount == other2->revCount + && lastModified == other2->lastModified; + } + + bool isImmutable() const override + { + return (bool) narHash; + } + ParsedURL toURL() const override { auto query = attrsToQuery(toAttrsInternal()); -- cgit v1.2.3 From a6ff66b658b61aef80d936f0183447fe4cb46000 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 19:04:33 +0200 Subject: Respect the narHash attribute in more input types call-flake.nix now passes node.info.narHash to fetchTree. This ensures that dirty Git trees work even in pure mode. --- src/libfetchers/fetchers.cc | 4 +++- src/libfetchers/git.cc | 2 +- src/libfetchers/github.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libfetchers/tarball.cc | 6 ++---- 5 files changed, 8 insertions(+), 8 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 765a6585f..83268b4bf 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -29,8 +29,10 @@ std::unique_ptr inputFromURL(const std::string & url) std::unique_ptr inputFromAttrs(const Attrs & attrs) { + auto attrs2(attrs); + attrs2.erase("narHash"); for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs); + auto res = inputScheme->inputFromAttrs(attrs2); if (res) { if (auto narHash = maybeGetStrAttr(attrs, "narHash")) // FIXME: require SRI hash. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index e578ed731..dbf57aa33 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -39,7 +39,7 @@ struct GitInput : Input bool isImmutable() const override { - return (bool) rev; + return (bool) rev || narHash; } std::optional getRef() const override { return ref; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 505af8af1..f50c927eb 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -33,7 +33,7 @@ struct GitHubInput : Input bool isImmutable() const override { - return (bool) rev; + return (bool) rev || narHash; } std::optional getRef() const override { return ref; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 4b73fbcbe..5abb00172 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -33,7 +33,7 @@ struct MercurialInput : Input bool isImmutable() const override { - return (bool) rev; + return (bool) rev || narHash; } std::optional getRef() const override { return ref; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b0a83001e..4c4e5828e 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -205,9 +205,7 @@ struct TarballInput : Input { Attrs attrs; attrs.emplace("url", url.to_string()); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - else if (hash) + if (hash) attrs.emplace("hash", hash->to_string(SRI)); return attrs; } @@ -260,7 +258,7 @@ struct TarballInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "hash" && name != "narHash") + if (name != "type" && name != "url" && name != "hash") throw Error("unsupported tarball input attribute '%s'", name); auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); -- cgit v1.2.3 From 485a87f22f96498727deab044fc74931164a1acd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 2 Apr 2020 19:09:17 +0200 Subject: Don't barf on registry parse errors --- src/libfetchers/registry.cc | 73 ++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 34 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index e452e8497..77e5da741 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -17,46 +17,51 @@ std::shared_ptr Registry::read( if (!pathExists(path)) return std::make_shared(type); - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - - // FIXME: remove soon - if (version == 1) { - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - auto url = i->value("url", i->value("uri", "")); - if (url.empty()) - throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", - path, i.key()); - registry->entries.push_back( - {inputFromURL(i.key()), inputFromURL(url), {}}); + try { + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + + // FIXME: remove soon + if (version == 1) { + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + auto url = i->value("url", i->value("uri", "")); + if (url.empty()) + throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", + path, i.key()); + registry->entries.push_back( + {inputFromURL(i.key()), inputFromURL(url), {}}); + } } - } - else if (version == 2) { - for (auto & i : json["flakes"]) { - auto toAttrs = jsonToAttrs(i["to"]); - Attrs extraAttrs; - auto j = toAttrs.find("dir"); - if (j != toAttrs.end()) { - extraAttrs.insert(*j); - toAttrs.erase(j); + else if (version == 2) { + for (auto & i : json["flakes"]) { + auto toAttrs = jsonToAttrs(i["to"]); + Attrs extraAttrs; + auto j = toAttrs.find("dir"); + if (j != toAttrs.end()) { + extraAttrs.insert(*j); + toAttrs.erase(j); + } + auto exact = i.find("exact"); + registry->entries.push_back( + Entry { + .from = inputFromAttrs(jsonToAttrs(i["from"])), + .to = inputFromAttrs(toAttrs), + .extraAttrs = extraAttrs, + .exact = exact != i.end() && exact.value() + }); } - auto exact = i.find("exact"); - registry->entries.push_back( - Entry { - .from = inputFromAttrs(jsonToAttrs(i["from"])), - .to = inputFromAttrs(toAttrs), - .extraAttrs = extraAttrs, - .exact = exact != i.end() && exact.value() - }); } - } - else - throw Error("flake registry '%s' has unsupported version %d", path, version); + else + throw Error("flake registry '%s' has unsupported version %d", path, version); + } catch (nlohmann::json::exception & e) { + warn("cannot parse flake registry '%s': %s", path, e.what()); + } return registry; } -- cgit v1.2.3 From 3aaceeb7e2d3fb8a07a1aa5a21df1dca6bbaa0ef Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Apr 2020 16:00:43 +0200 Subject: Fix build --- src/libfetchers/registry.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 77e5da741..84f0fe4a2 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -2,7 +2,6 @@ #include "fetchers.hh" #include "util.hh" #include "globals.hh" -#include "download.hh" #include "store-api.hh" #include -- cgit v1.2.3 From 3729df34da524e9f42ce86b83a762dd3da6f7384 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Apr 2020 17:25:39 +0200 Subject: Make Registry::read() more robust --- src/libfetchers/registry.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libfetchers') diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 84f0fe4a2..77d3b3378 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -60,6 +60,8 @@ std::shared_ptr Registry::read( } catch (nlohmann::json::exception & e) { warn("cannot parse flake registry '%s': %s", path, e.what()); + } catch (Error & e) { + warn("cannot read flake registry '%s': %s", path, e.what()); } return registry; -- cgit v1.2.3 From c7af247beacd418e6f2c4d33dffc35299101cd12 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Apr 2020 10:15:32 +0200 Subject: Path fetcher: Fix store path name --- src/libfetchers/path.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 7c7e20f4e..ba2cc192e 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -72,7 +72,7 @@ struct PathInput : Input if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) // FIXME: try to substitute storePath. - storePath = store->addToStore("name", path); + storePath = store->addToStore("source", path); return { -- cgit v1.2.3 From 6521c92ce8289a5f9e959c6789ab24dacdad082e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 27 Apr 2020 22:53:11 +0200 Subject: Improve path:// handling In particular, doing 'nix build /path/to/dir' now works if /path/to/dir is not a Git tree (it only has to contain a flake.nix file). Also, 'nix flake init' no longer requires a Git tree (but it will do a 'git add flake.nix' if it's a Git tree) --- src/libfetchers/path.cc | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index ba2cc192e..77fe87d59 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -32,7 +32,7 @@ struct PathInput : Input bool isImmutable() const override { - return (bool) narHash; + return narHash || rev; } ParsedURL toURL() const override @@ -56,9 +56,20 @@ struct PathInput : Input attrs.emplace("revCount", *revCount); if (lastModified) attrs.emplace("lastModified", *lastModified); + if (!rev && narHash) + attrs.emplace("narHash", narHash->to_string(SRI)); return attrs; } + std::optional getSourcePath() const override + { + return path; + } + + void markChangedFile(std::string_view file, std::optional commitMsg) const override + { + } + std::pair> fetchTreeInternal(nix::ref store) const override { auto input = std::make_shared(*this); @@ -74,6 +85,8 @@ struct PathInput : Input // FIXME: try to substitute storePath. storePath = store->addToStore("source", path); + input->narHash = store->queryPathInfo(*storePath)->narHash; + return { Tree { @@ -99,6 +112,9 @@ struct PathInputScheme : InputScheme auto input = std::make_unique(); input->path = url.path; + if (url.authority && *url.authority != "") + throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); + for (auto & [name, value] : url.query) if (name == "rev") input->rev = Hash(value, htSHA1); @@ -114,6 +130,9 @@ struct PathInputScheme : InputScheme throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); input->lastModified = lastModified; } + else if (name == "narHash") + // FIXME: require SRI hash. + input->narHash = Hash(value); else throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); @@ -134,6 +153,9 @@ struct PathInputScheme : InputScheme input->revCount = getIntAttr(attrs, "revCount"); else if (name == "lastModified") input->lastModified = getIntAttr(attrs, "lastModified"); + else if (name == "narHash") + // FIXME: require SRI hash. + input->narHash = Hash(getStrAttr(attrs, "narHash")); else if (name == "type" || name == "path") ; else -- cgit v1.2.3 From 0884f180f5ca8a864e6db5256eaa10646e87d671 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 10 May 2020 21:50:32 +0200 Subject: Simplify --- src/libfetchers/registry.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 77d3b3378..f6760d2d0 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -155,7 +155,7 @@ void overrideRegistry( static std::shared_ptr getGlobalRegistry(ref store) { static auto reg = [&]() { - auto path = settings.flakeRegistry; + auto path = settings.flakeRegistry.get(); if (!hasPrefix(path, "/")) { auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; -- cgit v1.2.3 From 849d3968dbe8c18651e1a7fbf071fc3304257517 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 May 2020 10:41:21 +0200 Subject: Update src/libfetchers/git.cc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörg Thalheim --- src/libfetchers/git.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 210e29193..c9f9a4b23 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -129,7 +129,7 @@ struct GitInput : Input assert(sourcePath); runProgram("git", true, - { "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) }); + { "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) }); if (commitMsg) runProgram("git", true, -- cgit v1.2.3 From 5256bc77cafeee2b0551195fa443afc4fca3196d Mon Sep 17 00:00:00 2001 From: Finn Behrens Date: Thu, 28 May 2020 20:34:52 +0200 Subject: add gitlab libfetcher --- src/libfetchers/gitlab.cc | 213 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 src/libfetchers/gitlab.cc (limited to 'src/libfetchers') diff --git a/src/libfetchers/gitlab.cc b/src/libfetchers/gitlab.cc new file mode 100644 index 000000000..9f47e69fe --- /dev/null +++ b/src/libfetchers/gitlab.cc @@ -0,0 +1,213 @@ +#include "filetransfer.hh" +#include "cache.hh" +#include "fetchers.hh" +#include "globals.hh" +#include "store-api.hh" + +#include + +namespace nix::fetchers { + +struct GitLabInput : Input +{ + std::string owner; + std::string repo; + std::optional ref; + std::optional rev; + + std::string type() const override { return "gitlab"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast(&other); + return + other2 + && owner == other2->owner + && repo == other2->repo + && rev == other2->rev + && ref == other2->ref; + } + + bool isImmutable() const override + { + return (bool) rev || narHash; + } + + std::optional getRef() const override { return ref; } + + std::optional getRev() const override { return rev; } + + ParsedURL toURL() const override + { + auto path = owner + "/" + repo; + assert(!(ref && rev)); + if (ref) path += "/" + *ref; + if (rev) path += "/" + rev->to_string(Base16, false); + return ParsedURL { + .scheme = "gitlab", + .path = path, + }; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("owner", owner); + attrs.emplace("repo", repo); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + void clone(const Path & destDir) const override + { + std::shared_ptr input = inputFromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", owner, repo)); + input = input->applyOverrides(ref.value_or("master"), rev); + input->clone(destDir); + } + + std::pair> fetchTreeInternal(nix::ref store) const override + { + auto rev = this->rev; + auto ref = this->ref.value_or("master"); + + if (!rev) { + auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/branches/%s", + owner, repo, ref); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false).storePath))); + rev = Hash(json["commit"]["id"], htSHA1); + debug("HEAD revision for '%s' is %s", url, rev->gitRev()); + } + + auto input = std::make_shared(*this); + input->ref = {}; + input->rev = *rev; + + Attrs immutableAttrs({ + {"type", "git-tarball"}, + {"rev", rev->gitRev()}, + }); + + if (auto res = getCache()->lookup(store, immutableAttrs)) { + return { + Tree{ + .actualPath = store->toRealPath(res->second), + .storePath = std::move(res->second), + .info = TreeInfo { + .lastModified = getIntAttr(res->first, "lastModified"), + }, + }, + input + }; + } + + // FIXME: This endpoint has a rate limit threshold of 5 requests per minute. + + auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", + owner, repo, rev->to_string(Base16, false)); + + /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: "`) + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken;*/ + + auto tree = downloadTarball(store, url, "source", true); + + getCache()->add( + store, + immutableAttrs, + { + {"rev", rev->gitRev()}, + {"lastModified", *tree.info.lastModified} + }, + tree.storePath, + true); + + return {std::move(tree), input}; + } + + std::shared_ptr applyOverrides( + std::optional ref, + std::optional rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } +}; + +struct GitLabInputScheme : InputScheme +{ + std::unique_ptr inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "gitlab") return nullptr; + + auto path = tokenizeString>(url.path, "/"); + auto input = std::make_unique(); + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + input->rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + input->ref = path[2]; + else + throw BadURL("in GitLab URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("GitLab URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (input->rev) + throw BadURL("GitLab URL '%s' contains multiple commit hashes", url.url); + input->rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("GitLab URL '%s' contains an invalid branch/tag name", url.url); + if (input->ref) + throw BadURL("GitLab URL '%s' contains multiple branch/tag names", url.url); + input->ref = value; + } + } + + if (input->ref && input->rev) + throw BadURL("GitLab URL '%s' contains both a commit hash and a branch/tag name", url.url); + + input->owner = path[0]; + input->repo = path[1]; + + return input; + } + + std::unique_ptr inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "gitlab") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") + throw Error("unsupported GitLab input attribute '%s'", name); + + auto input = std::make_unique(); + input->owner = getStrAttr(attrs, "owner"); + input->repo = getStrAttr(attrs, "repo"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); + +} -- cgit v1.2.3 From 5633c0975b9dec6bceaa85003223a346d7d1bd0b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 29 May 2020 14:23:32 +0200 Subject: Factor out GitHub / GitLab commonality --- src/libfetchers/github.cc | 182 +++++++++++++++++++++++++++++---------- src/libfetchers/gitlab.cc | 213 ---------------------------------------------- 2 files changed, 137 insertions(+), 258 deletions(-) delete mode 100644 src/libfetchers/gitlab.cc (limited to 'src/libfetchers') diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index c01917dcc..e59c83be4 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,21 +8,18 @@ namespace nix::fetchers { -std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); -std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); - -struct GitHubInput : Input +struct GitArchiveInput : Input { std::string owner; std::string repo; std::optional ref; std::optional rev; - std::string type() const override { return "github"; } + virtual std::shared_ptr _clone() const = 0; bool operator ==(const Input & other) const override { - auto other2 = dynamic_cast(&other); + auto other2 = dynamic_cast(&other); return other2 && owner == other2->owner @@ -47,7 +44,7 @@ struct GitHubInput : Input if (ref) path += "/" + *ref; if (rev) path += "/" + rev->to_string(Base16, false); return ParsedURL { - .scheme = "github", + .scheme = type(), .path = path, }; } @@ -64,30 +61,18 @@ struct GitHubInput : Input return attrs; } - void clone(const Path & destDir) const override - { - std::shared_ptr input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); - } + virtual Hash getRevFromRef(nix::ref store, std::string_view ref) const = 0; + + virtual std::string getDownloadUrl() const = 0; std::pair> fetchTreeInternal(nix::ref store) const override { auto rev = this->rev; auto ref = this->ref.value_or("master"); - if (!rev) { - auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - owner, repo, ref); - auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false).storePath))); - rev = Hash(json["sha"], htSHA1); - debug("HEAD revision for '%s' is %s", url, rev->gitRev()); - } + if (!rev) rev = getRevFromRef(store, ref); - auto input = std::make_shared(*this); + auto input = _clone(); input->ref = {}; input->rev = *rev; @@ -109,15 +94,8 @@ struct GitHubInput : Input }; } - // FIXME: use regular /archive URLs instead? api.github.com - // might have stricter rate limits. + auto url = input->getDownloadUrl(); - auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - owner, repo, rev->to_string(Base16, false)); - - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - url += "?access_token=" + accessToken; auto tree = downloadTarball(store, url, "source", true); @@ -140,7 +118,7 @@ struct GitHubInput : Input { if (!ref && !rev) return shared_from_this(); - auto res = std::make_shared(*this); + auto res = _clone(); if (ref) res->ref = ref; if (rev) res->rev = rev; @@ -149,14 +127,21 @@ struct GitHubInput : Input } }; -struct GitHubInputScheme : InputScheme +struct GitArchiveInputScheme : InputScheme { + std::string type; + + GitArchiveInputScheme(std::string && type) : type(type) + { } + + virtual std::unique_ptr create() = 0; + std::unique_ptr inputFromURL(const ParsedURL & url) override { - if (url.scheme != "github") return nullptr; + if (url.scheme != type) return nullptr; auto path = tokenizeString>(url.path, "/"); - auto input = std::make_unique(); + auto input = create(); if (path.size() == 2) { } else if (path.size() == 3) { @@ -165,27 +150,27 @@ struct GitHubInputScheme : InputScheme else if (std::regex_match(path[2], refRegex)) input->ref = path[2]; else - throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); } else - throw BadURL("GitHub URL '%s' is invalid", url.url); + throw BadURL("URL '%s' is invalid", url.url); for (auto &[name, value] : url.query) { if (name == "rev") { if (input->rev) - throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url); + throw BadURL("URL '%s' contains multiple commit hashes", url.url); input->rev = Hash(value, htSHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex)) - throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url); + throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); if (input->ref) - throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url); + throw BadURL("URL '%s' contains multiple branch/tag names", url.url); input->ref = value; } } if (input->ref && input->rev) - throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url); + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url); input->owner = path[0]; input->repo = path[1]; @@ -195,13 +180,13 @@ struct GitHubInputScheme : InputScheme std::unique_ptr inputFromAttrs(const Attrs & attrs) override { - if (maybeGetStrAttr(attrs, "type") != "github") return {}; + if (maybeGetStrAttr(attrs, "type") != type) return {}; for (auto & [name, value] : attrs) if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") - throw Error("unsupported GitHub input attribute '%s'", name); + throw Error("unsupported input attribute '%s'", name); - auto input = std::make_unique(); + auto input = create(); input->owner = getStrAttr(attrs, "owner"); input->repo = getStrAttr(attrs, "repo"); input->ref = maybeGetStrAttr(attrs, "ref"); @@ -211,6 +196,113 @@ struct GitHubInputScheme : InputScheme } }; +struct GitHubInput : GitArchiveInput +{ + std::string type() const override { return "github"; } + + std::shared_ptr _clone() const override + { return std::make_shared(*this); } + + Hash getRevFromRef(nix::ref store, std::string_view ref) const override + { + auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", + owner, repo, ref); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false).storePath))); + auto rev = Hash(json["sha"], htSHA1); + debug("HEAD revision for '%s' is %s", url, rev.gitRev()); + return rev; + } + + std::string getDownloadUrl() const override + { + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + owner, repo, rev->to_string(Base16, false)); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + return url; + } + + void clone(const Path & destDir) const override + { + std::shared_ptr input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); + input = input->applyOverrides(ref.value_or("master"), rev); + input->clone(destDir); + } +}; + +struct GitHubInputScheme : GitArchiveInputScheme +{ + GitHubInputScheme() : GitArchiveInputScheme("github") { } + + std::unique_ptr create() override + { + return std::make_unique(); + } +}; + +struct GitLabInput : GitArchiveInput +{ + std::string type() const override { return "gitlab"; } + + std::shared_ptr _clone() const override + { return std::make_shared(*this); } + + Hash getRevFromRef(nix::ref store, std::string_view ref) const override + { + auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/branches/%s", + owner, repo, ref); + auto json = nlohmann::json::parse( + readFile( + store->toRealPath( + downloadFile(store, url, "source", false).storePath))); + auto rev = Hash(json["commit"]["id"], htSHA1); + debug("HEAD revision for '%s' is %s", url, rev.gitRev()); + return rev; + } + + std::string getDownloadUrl() const override + { + // FIXME: This endpoint has a rate limit threshold of 5 requests per minute. + + auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", + owner, repo, rev->to_string(Base16, false)); + + /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: "`) + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken;*/ + + return url; + } + + void clone(const Path & destDir) const override + { + std::shared_ptr input = inputFromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", owner, repo)); + input = input->applyOverrides(ref.value_or("master"), rev); + input->clone(destDir); + } +}; + +struct GitLabInputScheme : GitArchiveInputScheme +{ + GitLabInputScheme() : GitArchiveInputScheme("gitlab") { } + + std::unique_ptr create() override + { + return std::make_unique(); + } +}; + static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); +static auto r2 = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/gitlab.cc b/src/libfetchers/gitlab.cc deleted file mode 100644 index 9f47e69fe..000000000 --- a/src/libfetchers/gitlab.cc +++ /dev/null @@ -1,213 +0,0 @@ -#include "filetransfer.hh" -#include "cache.hh" -#include "fetchers.hh" -#include "globals.hh" -#include "store-api.hh" - -#include - -namespace nix::fetchers { - -struct GitLabInput : Input -{ - std::string owner; - std::string repo; - std::optional ref; - std::optional rev; - - std::string type() const override { return "gitlab"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && owner == other2->owner - && repo == other2->repo - && rev == other2->rev - && ref == other2->ref; - } - - bool isImmutable() const override - { - return (bool) rev || narHash; - } - - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - ParsedURL toURL() const override - { - auto path = owner + "/" + repo; - assert(!(ref && rev)); - if (ref) path += "/" + *ref; - if (rev) path += "/" + rev->to_string(Base16, false); - return ParsedURL { - .scheme = "gitlab", - .path = path, - }; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("owner", owner); - attrs.emplace("repo", repo); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; - } - - void clone(const Path & destDir) const override - { - std::shared_ptr input = inputFromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - auto rev = this->rev; - auto ref = this->ref.value_or("master"); - - if (!rev) { - auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/branches/%s", - owner, repo, ref); - auto json = nlohmann::json::parse( - readFile( - store->toRealPath( - downloadFile(store, url, "source", false).storePath))); - rev = Hash(json["commit"]["id"], htSHA1); - debug("HEAD revision for '%s' is %s", url, rev->gitRev()); - } - - auto input = std::make_shared(*this); - input->ref = {}; - input->rev = *rev; - - Attrs immutableAttrs({ - {"type", "git-tarball"}, - {"rev", rev->gitRev()}, - }); - - if (auto res = getCache()->lookup(store, immutableAttrs)) { - return { - Tree{ - .actualPath = store->toRealPath(res->second), - .storePath = std::move(res->second), - .info = TreeInfo { - .lastModified = getIntAttr(res->first, "lastModified"), - }, - }, - input - }; - } - - // FIXME: This endpoint has a rate limit threshold of 5 requests per minute. - - auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", - owner, repo, rev->to_string(Base16, false)); - - /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: "`) - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - url += "?access_token=" + accessToken;*/ - - auto tree = downloadTarball(store, url, "source", true); - - getCache()->add( - store, - immutableAttrs, - { - {"rev", rev->gitRev()}, - {"lastModified", *tree.info.lastModified} - }, - tree.storePath, - true); - - return {std::move(tree), input}; - } - - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } -}; - -struct GitLabInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "gitlab") return nullptr; - - auto path = tokenizeString>(url.path, "/"); - auto input = std::make_unique(); - - if (path.size() == 2) { - } else if (path.size() == 3) { - if (std::regex_match(path[2], revRegex)) - input->rev = Hash(path[2], htSHA1); - else if (std::regex_match(path[2], refRegex)) - input->ref = path[2]; - else - throw BadURL("in GitLab URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else - throw BadURL("GitLab URL '%s' is invalid", url.url); - - for (auto &[name, value] : url.query) { - if (name == "rev") { - if (input->rev) - throw BadURL("GitLab URL '%s' contains multiple commit hashes", url.url); - input->rev = Hash(value, htSHA1); - } - else if (name == "ref") { - if (!std::regex_match(value, refRegex)) - throw BadURL("GitLab URL '%s' contains an invalid branch/tag name", url.url); - if (input->ref) - throw BadURL("GitLab URL '%s' contains multiple branch/tag names", url.url); - input->ref = value; - } - } - - if (input->ref && input->rev) - throw BadURL("GitLab URL '%s' contains both a commit hash and a branch/tag name", url.url); - - input->owner = path[0]; - input->repo = path[1]; - - return input; - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "gitlab") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") - throw Error("unsupported GitLab input attribute '%s'", name); - - auto input = std::make_unique(); - input->owner = getStrAttr(attrs, "owner"); - input->repo = getStrAttr(attrs, "repo"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - -static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); - -} -- cgit v1.2.3 From 950b46821f644eb3f92725460584a3102f356179 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 30 May 2020 00:44:11 +0200 Subject: Remove TreeInfo The attributes previously stored in TreeInfo (narHash, revCount, lastModified) are now stored in Input. This makes it less arbitrary what attributes are stored where. As a result, the lock file format has changed. An entry like "info": { "lastModified": 1585405475, "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE=" }, "locked": { "owner": "NixOS", "repo": "nixpkgs", "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be", "type": "github" }, is now stored as "locked": { "owner": "NixOS", "repo": "nixpkgs", "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be", "type": "github", "lastModified": 1585405475, "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE=" }, The 'Input' class is now a dumb set of attributes. All the fetcher implementations subclass InputScheme, not Input. This simplifies the API. Also, fix substitution of flake inputs. This was broken since lazy flake fetching started using fetchTree internally. --- src/libfetchers/attrs.cc | 14 +- src/libfetchers/attrs.hh | 11 +- src/libfetchers/fetchers.cc | 224 ++++++++++++++++++++++++++------ src/libfetchers/fetchers.hh | 104 ++++++++------- src/libfetchers/git.cc | 292 ++++++++++++++++++------------------------ src/libfetchers/github.cc | 297 ++++++++++++++++++------------------------- src/libfetchers/indirect.cc | 154 +++++++++------------- src/libfetchers/mercurial.cc | 204 ++++++++++++----------------- src/libfetchers/path.cc | 178 +++++++++----------------- src/libfetchers/registry.cc | 56 ++++---- src/libfetchers/registry.hh | 17 ++- src/libfetchers/tarball.cc | 145 ++++++++------------- src/libfetchers/tree-info.cc | 60 --------- src/libfetchers/tree-info.hh | 33 ----- 14 files changed, 791 insertions(+), 998 deletions(-) delete mode 100644 src/libfetchers/tree-info.cc delete mode 100644 src/libfetchers/tree-info.hh (limited to 'src/libfetchers') diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index feb0a6085..1e59faa73 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + if (auto v = std::get_if(&attr.second)) { json[attr.first] = *v; } else if (auto v = std::get_if(&attr.second)) { json[attr.first] = *v; @@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name) return *s; } -std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) + if (auto v = std::get_if(&i->second)) return *v; throw Error("input attribute '%s' is not an integer", name); } -int64_t getIntAttr(const Attrs & attrs, const std::string & name) +uint64_t getIntAttr(const Attrs & attrs, const std::string & name) { auto s = maybeGetIntAttr(attrs, name); if (!s) @@ -76,8 +76,8 @@ std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & na { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if(&i->second)) - return *v; + if (auto v = std::get_if>(&i->second)) + return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -93,7 +93,7 @@ std::map attrsToQuery(const Attrs & attrs) { std::map query; for (auto & attr : attrs) { - if (auto v = std::get_if(&attr.second)) { + if (auto v = std::get_if(&attr.second)) { query.insert_or_assign(attr.first, fmt("%d", *v)); } else if (auto v = std::get_if(&attr.second)) { query.insert_or_assign(attr.first, *v); diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index d6e0ae000..4b4630c80 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,9 +13,14 @@ namespace nix::fetchers { template struct Explicit { T t; + + bool operator ==(const Explicit & other) const + { + return t == other.t; + } }; -typedef std::variant> Attr; +typedef std::variant> Attr; typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); @@ -26,9 +31,9 @@ std::optional maybeGetStrAttr(const Attrs & attrs, const std::strin std::string getStrAttr(const Attrs & attrs, const std::string & name); -std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); +std::optional maybeGetIntAttr(const Attrs & attrs, const std::string & name); -int64_t getIntAttr(const Attrs & attrs, const std::string & name); +uint64_t getIntAttr(const Attrs & attrs, const std::string & name); std::optional maybeGetBoolAttr(const Attrs & attrs, const std::string & name); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 83268b4bf..e4852d662 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -5,82 +5,234 @@ namespace nix::fetchers { -std::unique_ptr>> inputSchemes = nullptr; +std::unique_ptr>> inputSchemes = nullptr; -void registerInputScheme(std::unique_ptr && inputScheme) +void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique>>(); + if (!inputSchemes) inputSchemes = std::make_unique>>(); inputSchemes->push_back(std::move(inputScheme)); } -std::unique_ptr inputFromURL(const ParsedURL & url) +Input Input::fromURL(const std::string & url) +{ + return fromURL(parseURL(url)); +} + +static void fixupInput(Input & input) +{ + // Check common attributes. + input.getType(); + input.getRef(); + if (input.getRev()) + input.immutable = true; + input.getRevCount(); + input.getLastModified(); + if (input.getNarHash()) + input.immutable = true; +} + +Input Input::fromURL(const ParsedURL & url) { for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromURL(url); - if (res) return res; + if (res) { + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); + } } - throw Error("input '%s' is unsupported", url.url); -} -std::unique_ptr inputFromURL(const std::string & url) -{ - return inputFromURL(parseURL(url)); + throw Error("input '%s' is unsupported", url.url); } -std::unique_ptr inputFromAttrs(const Attrs & attrs) +Input Input::fromAttrs(Attrs && attrs) { - auto attrs2(attrs); - attrs2.erase("narHash"); for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs2); + auto res = inputScheme->inputFromAttrs(attrs); if (res) { - if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - // FIXME: require SRI hash. - res->narHash = Hash(*narHash); - return res; + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); } } - throw Error("input '%s' is unsupported", attrsToJson(attrs)); + + Input input; + input.attrs = attrs; + fixupInput(input); + return input; +} + +ParsedURL Input::toURL() const +{ + if (!scheme) + throw Error("cannot show unsupported input '%s'", attrsToJson(attrs)); + return scheme->toURL(*this); +} + +std::string Input::to_string() const +{ + return toURL().to_string(); } Attrs Input::toAttrs() const { - auto attrs = toAttrsInternal(); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - attrs.emplace("type", type()); return attrs; } -std::pair> Input::fetchTree(ref store) const +bool Input::hasAllInfo() const { - auto [tree, input] = fetchTreeInternal(store); + return getNarHash() && scheme && scheme->hasAllInfo(*this); +} + +bool Input::operator ==(const Input & other) const +{ + return attrs == other.attrs; +} + +bool Input::contains(const Input & other) const +{ + auto other2(other); + other2.attrs.erase("ref"); + other2.attrs.erase("rev"); + if (*this == other2) return true; + return false; +} + +std::pair Input::fetch(ref store) const +{ + if (!scheme) + throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs())); + + auto [tree, input] = scheme->fetch(store, *this); if (tree.actualPath == "") tree.actualPath = store->toRealPath(tree.storePath); - if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + auto narHash = store->queryPathInfo(tree.storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(SRI)); - if (input->narHash) - assert(input->narHash == tree.info.narHash); + if (auto narHash2 = getNarHash()) { + if (narHash != *narHash2) + throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, narHash2->to_string(SRI), narHash.to_string(SRI)); + } - if (narHash && narHash != input->narHash) - throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); + // FIXME: check lastModified, revCount + + input.immutable = true; + + assert(input.hasAllInfo()); return {std::move(tree), input}; } -std::shared_ptr Input::applyOverrides( +Input Input::applyOverrides( std::optional ref, std::optional rev) const +{ + if (!scheme) return *this; + return scheme->applyOverrides(*this, ref, rev); +} + +void Input::clone(const Path & destDir) const +{ + assert(scheme); + scheme->clone(*this, destDir); +} + +std::optional Input::getSourcePath() const +{ + assert(scheme); + return scheme->getSourcePath(*this); +} + +void Input::markChangedFile( + std::string_view file, + std::optional commitMsg) const +{ + assert(scheme); + return scheme->markChangedFile(*this, file, commitMsg); +} + +StorePath Input::computeStorePath(Store & store) const +{ + auto narHash = getNarHash(); + if (!narHash) + throw Error("cannot compute store path for mutable input '%s'", to_string()); + return store.makeFixedOutputPath(true, *narHash, "source"); +} + +std::string Input::getType() const +{ + return getStrAttr(attrs, "type"); +} + +std::optional Input::getNarHash() const +{ + if (auto s = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + return Hash(*s, htSHA256); + return {}; +} + +std::optional Input::getRef() const +{ + if (auto s = maybeGetStrAttr(attrs, "ref")) + return *s; + return {}; +} + +std::optional Input::getRev() const +{ + if (auto s = maybeGetStrAttr(attrs, "rev")) + return Hash(*s, htSHA1); + return {}; +} + +std::optional Input::getRevCount() const +{ + if (auto n = maybeGetIntAttr(attrs, "revCount")) + return *n; + return {}; +} + +std::optional Input::getLastModified() const +{ + if (auto n = maybeGetIntAttr(attrs, "lastModified")) + return *n; + return {}; +} + +ParsedURL InputScheme::toURL(const Input & input) +{ + throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs)); +} + +Input InputScheme::applyOverrides( + const Input & input, + std::optional ref, + std::optional rev) { if (ref) - throw Error("don't know how to apply '%s' to '%s'", *ref, to_string()); + throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref); if (rev) - throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string()); - return shared_from_this(); + throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev()); + return input; +} + +std::optional InputScheme::getSourcePath(const Input & input) +{ + return {}; +} + +void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) +{ + assert(false); +} + +void InputScheme::clone(const Input & input, const Path & destDir) +{ + throw Error("do not know how to clone input '%s'", input.to_string()); } } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b75dcffa5..c43cfe50c 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "hash.hh" #include "path.hh" -#include "tree-info.hh" #include "attrs.hh" #include "url.hh" @@ -13,89 +12,100 @@ namespace nix { class Store; } namespace nix::fetchers { -struct Input; - struct Tree { Path actualPath; StorePath storePath; - TreeInfo info; }; -struct Input : std::enable_shared_from_this +struct InputScheme; + +struct Input { - std::optional narHash; // FIXME: implement + friend class InputScheme; + + std::shared_ptr scheme; // note: can be null + Attrs attrs; + bool immutable = false; + bool direct = true; + +public: + static Input fromURL(const std::string & url); + + static Input fromURL(const ParsedURL & url); - virtual std::string type() const = 0; + static Input fromAttrs(Attrs && attrs); - virtual ~Input() { } + ParsedURL toURL() const; - virtual bool operator ==(const Input & other) const { return false; } + std::string to_string() const; + + Attrs toAttrs() const; /* Check whether this is a "direct" input, that is, not one that goes through a registry. */ - virtual bool isDirect() const { return true; } + bool isDirect() const { return direct; } /* Check whether this is an "immutable" input, that is, one that contains a commit hash or content hash. */ - virtual bool isImmutable() const { return (bool) narHash; } - - virtual bool contains(const Input & other) const { return false; } - - virtual std::optional getRef() const { return {}; } - - virtual std::optional getRev() const { return {}; } + bool isImmutable() const { return immutable; } - virtual ParsedURL toURL() const = 0; + bool hasAllInfo() const; - std::string to_string() const - { - return toURL().to_string(); - } + bool operator ==(const Input & other) const; - Attrs toAttrs() const; + bool contains(const Input & other) const; - std::pair> fetchTree(ref store) const; + std::pair fetch(ref store) const; - virtual std::shared_ptr applyOverrides( + Input applyOverrides( std::optional ref, std::optional rev) const; - virtual std::optional getSourcePath() const { return {}; } - - virtual void markChangedFile( - std::string_view file, - std::optional commitMsg) const - { assert(false); } + void clone(const Path & destDir) const; - virtual void clone(const Path & destDir) const - { - throw Error("do not know how to clone input '%s'", to_string()); - } + std::optional getSourcePath() const; -private: + void markChangedFile( + std::string_view file, + std::optional commitMsg) const; - virtual std::pair> fetchTreeInternal(ref store) const = 0; + StorePath computeStorePath(Store & store) const; - virtual Attrs toAttrsInternal() const = 0; + // Convience functions for common attributes. + std::string getType() const; + std::optional getNarHash() const; + std::optional getRef() const; + std::optional getRev() const; + std::optional getRevCount() const; + std::optional getLastModified() const; }; struct InputScheme { - virtual ~InputScheme() { } + virtual std::optional inputFromURL(const ParsedURL & url) = 0; - virtual std::unique_ptr inputFromURL(const ParsedURL & url) = 0; + virtual std::optional inputFromAttrs(const Attrs & attrs) = 0; - virtual std::unique_ptr inputFromAttrs(const Attrs & attrs) = 0; -}; + virtual ParsedURL toURL(const Input & input); -std::unique_ptr inputFromURL(const ParsedURL & url); + virtual bool hasAllInfo(const Input & input) = 0; -std::unique_ptr inputFromURL(const std::string & url); + virtual Input applyOverrides( + const Input & input, + std::optional ref, + std::optional rev); + + virtual void clone(const Input & input, const Path & destDir); + + virtual std::optional getSourcePath(const Input & input); -std::unique_ptr inputFromAttrs(const Attrs & attrs); + virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); + + virtual std::pair fetch(ref store, const Input & input) = 0; +}; -void registerInputScheme(std::unique_ptr && fetcher); +void registerInputScheme(std::shared_ptr && fetcher); struct DownloadFileResult { @@ -110,7 +120,7 @@ DownloadFileResult downloadFile( const std::string & name, bool immutable); -Tree downloadTarball( +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c9f9a4b23..4fcf3f542 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -22,110 +22,121 @@ static bool isNotDotGitDirectory(const Path & path) return not std::regex_match(path, gitDirRegex); } -struct GitInput : Input +struct GitInputScheme : InputScheme { - ParsedURL url; - std::optional ref; - std::optional rev; - bool shallow = false; - bool submodules = false; + std::optional inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return {}; - GitInput(const ParsedURL & url) : url(url) - { } + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); - std::string type() const override { return "git"; } + Attrs attrs; + attrs.emplace("type", "git"); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != "git") return {}; - std::optional getRef() const override { return ref; } + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash") + throw Error("unsupported Git input attribute '%s'", name); + + parseURL(getStrAttr(attrs, "url")); + maybeGetBoolAttr(attrs, "shallow"); + maybeGetBoolAttr(attrs, "submodules"); + + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + } - std::optional getRev() const override { return rev; } + Input input; + input.attrs = attrs; + return input; + } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - ParsedURL url2(url); - if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); - if (shallow) url2.query.insert_or_assign("shallow", "1"); - return url2; + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme != "git") url.scheme = "git+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); + if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + url.query.insert_or_assign("shallow", "1"); + return url; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (shallow) - attrs.emplace("shallow", true); - if (submodules) - attrs.emplace("submodules", true); - return attrs; + bool maybeDirty = !input.getRef(); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + return + maybeGetIntAttr(input.attrs, "lastModified") + && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); } - void clone(const Path & destDir) const override + Input applyOverrides( + const Input & input, + std::optional ref, + std::optional rev) override + { + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); + if (!res.getRef() && res.getRev()) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string()); + return res; + } + + void clone(const Input & input, const Path & destDir) override { - auto [isLocal, actualUrl] = getActualUrl(); + auto [isLocal, actualUrl] = getActualUrl(input); Strings args = {"clone"}; args.push_back(actualUrl); - if (ref) { + if (auto ref = input.getRef()) { args.push_back("--branch"); args.push_back(*ref); } - if (rev) throw Error("cloning a specific revision is not implemented"); + if (input.getRev()) throw Error("cloning a specific revision is not implemented"); args.push_back(destDir); runProgram("git", true, args); } - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - if (!res->ref && res->rev) - throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); - - return res; - } - - std::optional getSourcePath() const override + std::optional getSourcePath(const Input & input) override { - if (url.scheme == "file" && !ref && !rev) + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) return url.path; return {}; } - void markChangedFile(std::string_view file, std::optional commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override { - auto sourcePath = getSourcePath(); + auto sourcePath = getSourcePath(input); assert(sourcePath); runProgram("git", true, @@ -136,23 +147,25 @@ struct GitInput : Input { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); } - std::pair getActualUrl() const + std::pair getActualUrl(const Input & input) const { // Don't clone file:// URIs (but otherwise treat them the // same as remote URIs, i.e. don't use the working tree or // HEAD). static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file" && !forceHttp; return {isLocal, isLocal ? url.path : url.base}; } - std::pair> fetchTreeInternal(nix::ref store) const override + std::pair fetch(ref store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared(*this); + Input input(_input); - assert(!rev || rev->type == htSHA1); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -163,39 +176,38 @@ struct GitInput : Input return Attrs({ {"type", cacheType}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair> + -> std::pair { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + if (!shallow) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return { Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), - .lastModified = getIntAttr(infoAttrs, "lastModified"), - }, }, input }; }; - if (rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. - if (!input->ref && !input->rev && isLocal) { + if (!input.getRef() && !input.getRev() && isLocal) { bool clean = false; /* Check whether this repo has any commits. There are @@ -254,35 +266,37 @@ struct GitInput : Input auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); - auto tree = Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, - } + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + input.attrs.insert_or_assign( + "lastModified", + haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0); + + return { + Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input }; - - return {std::move(tree), input}; } } - if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master"); Attrs mutableAttrs({ {"type", cacheType}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); Path repoDir; if (isLocal) { - if (!input->rev) - input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", + Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; @@ -290,8 +304,8 @@ struct GitInput : Input if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -305,18 +319,18 @@ struct GitInput : Input } Path localRefFile = - input->ref->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input->ref - : cacheDir + "/refs/heads/" + *input->ref; + input.getRef()->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input.getRef() + : cacheDir + "/refs/heads/" + *input.getRef(); bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input->rev) { + if (input.getRev()) { try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -339,7 +353,7 @@ struct GitInput : Input // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. try { - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input.getRef(), *input.getRef()) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); @@ -354,8 +368,8 @@ struct GitInput : Input utimes(localRefFile.c_str(), times); } - if (!input->rev) - input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev()); } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; @@ -365,7 +379,7 @@ struct GitInput : Input // FIXME: check whether rev is an ancestor of ref. - printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); /* Now that we know the ref, check again whether we have it in the store. */ @@ -387,7 +401,7 @@ struct GitInput : Input runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); @@ -396,7 +410,7 @@ struct GitInput : Input // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() }); gitOptions.standardOut = &sink; runProgram2(gitOptions); }); @@ -406,18 +420,18 @@ struct GitInput : Input auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() })); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"lastModified", lastModified}, }); if (!shallow) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -436,60 +450,6 @@ struct GitInput : Input } }; -struct GitInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "git" && - url.scheme != "git+http" && - url.scheme != "git+https" && - url.scheme != "git+ssh" && - url.scheme != "git+file") return nullptr; - - auto url2(url); - if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "git"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules") - throw Error("unsupported Git input attribute '%s'", name); - - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Git branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - - input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); - - input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false); - - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index e59c83be4..8d113967e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,37 +8,80 @@ namespace nix::fetchers { -struct GitArchiveInput : Input +struct GitArchiveInputScheme : InputScheme { - std::string owner; - std::string repo; - std::optional ref; - std::optional rev; - - virtual std::shared_ptr _clone() const = 0; + virtual std::string type() = 0; - bool operator ==(const Input & other) const override + std::optional inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast(&other); - return - other2 - && owner == other2->owner - && repo == other2->repo - && rev == other2->rev - && ref == other2->ref; + if (url.scheme != type()) return {}; + + auto path = tokenizeString>(url.path, "/"); + + std::optional rev; + std::optional ref; + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + ref = path[2]; + else + throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (rev) + throw BadURL("URL '%s' contains multiple commit hashes", url.url); + rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); + if (ref) + throw BadURL("URL '%s' contains multiple branch/tag names", url.url); + ref = value; + } + } + + if (ref && rev) + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url); + + Input input; + input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("owner", path[0]); + input.attrs.insert_or_assign("repo", path[1]); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + + return input; } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != type()) return {}; - std::optional getRef() const override { return ref; } + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified") + throw Error("unsupported input attribute '%s'", name); - std::optional getRev() const override { return rev; } + getStrAttr(attrs, "owner"); + getStrAttr(attrs, "repo"); + + Input input; + input.attrs = attrs; + return input; + } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { + auto owner = getStrAttr(input.attrs, "owner"); + auto repo = getStrAttr(input.attrs, "repo"); + auto ref = input.getRef(); + auto rev = input.getRev(); auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; @@ -49,32 +92,44 @@ struct GitArchiveInput : Input }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("owner", owner); - attrs.emplace("repo", repo); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); + } + + Input applyOverrides( + const Input & _input, + std::optional ref, + std::optional rev) override + { + auto input(_input); + if (rev) { + input.attrs.insert_or_assign("rev", rev->gitRev()); + input.attrs.erase("ref"); + } + if (ref) { + if (input.getRev()) + throw BadURL("input '%s' contains both a commit hash and a branch/tag name", input.to_string()); + input.attrs.insert_or_assign("ref", *ref); + } + return input; } - virtual Hash getRevFromRef(nix::ref store, std::string_view ref) const = 0; + virtual Hash getRevFromRef(nix::ref store, const Input & input) const = 0; - virtual std::string getDownloadUrl() const = 0; + virtual std::string getDownloadUrl(const Input & input) const = 0; - std::pair> fetchTreeInternal(nix::ref store) const override + std::pair fetch(ref store, const Input & _input) override { - auto rev = this->rev; - auto ref = this->ref.value_or("master"); + Input input(_input); - if (!rev) rev = getRevFromRef(store, ref); + if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "master"); - auto input = _clone(); - input->ref = {}; - input->rev = *rev; + auto rev = input.getRev(); + if (!rev) rev = getRevFromRef(store, input); + + input.attrs.erase("ref"); + input.attrs.insert_or_assign("rev", rev->gitRev()); Attrs immutableAttrs({ {"type", "git-tarball"}, @@ -82,131 +137,44 @@ struct GitArchiveInput : Input }); if (auto res = getCache()->lookup(store, immutableAttrs)) { + input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); return { Tree{ .actualPath = store->toRealPath(res->second), .storePath = std::move(res->second), - .info = TreeInfo { - .lastModified = getIntAttr(res->first, "lastModified"), - }, }, input }; } - auto url = input->getDownloadUrl(); + auto url = getDownloadUrl(input); + auto [tree, lastModified] = downloadTarball(store, url, "source", true); - auto tree = downloadTarball(store, url, "source", true); + input.attrs.insert_or_assign("lastModified", lastModified); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", *tree.info.lastModified} + {"lastModified", lastModified} }, tree.storePath, true); return {std::move(tree), input}; } - - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = _clone(); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } }; -struct GitArchiveInputScheme : InputScheme -{ - std::string type; - - GitArchiveInputScheme(std::string && type) : type(type) - { } - - virtual std::unique_ptr create() = 0; - - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != type) return nullptr; - - auto path = tokenizeString>(url.path, "/"); - auto input = create(); - - if (path.size() == 2) { - } else if (path.size() == 3) { - if (std::regex_match(path[2], revRegex)) - input->rev = Hash(path[2], htSHA1); - else if (std::regex_match(path[2], refRegex)) - input->ref = path[2]; - else - throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else - throw BadURL("URL '%s' is invalid", url.url); - - for (auto &[name, value] : url.query) { - if (name == "rev") { - if (input->rev) - throw BadURL("URL '%s' contains multiple commit hashes", url.url); - input->rev = Hash(value, htSHA1); - } - else if (name == "ref") { - if (!std::regex_match(value, refRegex)) - throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); - if (input->ref) - throw BadURL("URL '%s' contains multiple branch/tag names", url.url); - input->ref = value; - } - } - - if (input->ref && input->rev) - throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url); - - input->owner = path[0]; - input->repo = path[1]; - - return input; - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != type) return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") - throw Error("unsupported input attribute '%s'", name); - - auto input = create(); - input->owner = getStrAttr(attrs, "owner"); - input->repo = getStrAttr(attrs, "repo"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - -struct GitHubInput : GitArchiveInput +struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() const override { return "github"; } + std::string type() override { return "github"; } - std::shared_ptr _clone() const override - { return std::make_shared(*this); } - - Hash getRevFromRef(nix::ref store, std::string_view ref) const override + Hash getRevFromRef(nix::ref store, const Input & input) const override { auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - owner, repo, ref); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( @@ -216,13 +184,14 @@ struct GitHubInput : GitArchiveInput return rev; } - std::string getDownloadUrl() const override + std::string getDownloadUrl(const Input & input) const override { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - owner, repo, rev->to_string(Base16, false)); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); std::string accessToken = settings.githubAccessToken.get(); if (accessToken != "") @@ -231,35 +200,23 @@ struct GitHubInput : GitArchiveInput return url; } - void clone(const Path & destDir) const override - { - std::shared_ptr input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); - } -}; - -struct GitHubInputScheme : GitArchiveInputScheme -{ - GitHubInputScheme() : GitArchiveInputScheme("github") { } - - std::unique_ptr create() override + void clone(const Input & input, const Path & destDir) override { - return std::make_unique(); + Input::fromURL(fmt("git+ssh://git@github.com/%s/%s.git", + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("master"), input.getRev()) + .clone(destDir); } }; -struct GitLabInput : GitArchiveInput +struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() const override { return "gitlab"; } - - std::shared_ptr _clone() const override - { return std::make_shared(*this); } + std::string type() override { return "gitlab"; } - Hash getRevFromRef(nix::ref store, std::string_view ref) const override + Hash getRevFromRef(nix::ref store, const Input & input) const override { auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/branches/%s", - owner, repo, ref); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( @@ -269,12 +226,13 @@ struct GitLabInput : GitArchiveInput return rev; } - std::string getDownloadUrl() const override + std::string getDownloadUrl(const Input & input) const override { // FIXME: This endpoint has a rate limit threshold of 5 requests per minute. auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", - owner, repo, rev->to_string(Base16, false)); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: "`) std::string accessToken = settings.githubAccessToken.get(); @@ -284,21 +242,12 @@ struct GitLabInput : GitArchiveInput return url; } - void clone(const Path & destDir) const override - { - std::shared_ptr input = inputFromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); - } -}; - -struct GitLabInputScheme : GitArchiveInputScheme -{ - GitLabInputScheme() : GitArchiveInputScheme("gitlab") { } - - std::unique_ptr create() override + void clone(const Input & input, const Path & destDir) override { - return std::make_unique(); + Input::fromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("master"), input.getRev()) + .clone(destDir); } }; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 380b69fe0..91dc83740 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -4,135 +4,99 @@ namespace nix::fetchers { std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); -struct IndirectInput : Input -{ - std::string id; - std::optional rev; - std::optional ref; - - std::string type() const override { return "indirect"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && id == other2->id - && rev == other2->rev - && ref == other2->ref; - } - - bool isDirect() const override - { - return false; - } - - std::optional getRef() const override { return ref; } - - std::optional getRev() const override { return rev; } - - bool contains(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && id == other2->id - && (!ref || ref == other2->ref) - && (!rev || rev == other2->rev); - } - - ParsedURL toURL() const override - { - ParsedURL url; - url.scheme = "flake"; - url.path = id; - if (ref) { url.path += '/'; url.path += *ref; }; - if (rev) { url.path += '/'; url.path += rev->gitRev(); }; - return url; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("id", id); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; - } - - std::shared_ptr applyOverrides( - std::optional ref, - std::optional rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - throw Error("indirect input '%s' cannot be fetched directly", to_string()); - } -}; - struct IndirectInputScheme : InputScheme { - std::unique_ptr inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) override { - if (url.scheme != "flake") return nullptr; + if (url.scheme != "flake") return {}; auto path = tokenizeString>(url.path, "/"); - auto input = std::make_unique(); + + std::optional rev; + std::optional ref; if (path.size() == 1) { } else if (path.size() == 2) { if (std::regex_match(path[1], revRegex)) - input->rev = Hash(path[1], htSHA1); + rev = Hash(path[1], htSHA1); else if (std::regex_match(path[1], refRegex)) - input->ref = path[1]; + ref = path[1]; else throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); } else if (path.size() == 3) { if (!std::regex_match(path[1], refRegex)) throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); - input->ref = path[1]; + ref = path[1]; if (!std::regex_match(path[2], revRegex)) throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); - input->rev = Hash(path[2], htSHA1); + rev = Hash(path[2], htSHA1); } else throw BadURL("GitHub URL '%s' is invalid", url.url); + std::string id = path[0]; + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + // FIXME: forbid query params? - input->id = path[0]; - if (!std::regex_match(input->id, flakeRegex)) - throw BadURL("'%s' is not a valid flake ID", input->id); + Input input; + input.direct = false; + input.attrs.insert_or_assign("type", "indirect"); + input.attrs.insert_or_assign("id", id); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); return input; } - std::unique_ptr inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev") + if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") throw Error("unsupported indirect input attribute '%s'", name); - auto input = std::make_unique(); - input->id = getStrAttr(attrs, "id"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); + auto id = getStrAttr(attrs, "id"); + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + + Input input; + input.direct = false; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override + { + ParsedURL url; + url.scheme = "flake"; + url.path = getStrAttr(input.attrs, "id"); + if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; }; + if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); }; + return url; + } + + bool hasAllInfo(const Input & input) override + { + return false; + } + + Input applyOverrides( + const Input & _input, + std::optional ref, + std::optional rev) override + { + auto input(_input); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); return input; } + + std::pair fetch(ref store, const Input & input) override + { + throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 5abb00172..49ed63243 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -10,80 +10,92 @@ using namespace std::string_literals; namespace nix::fetchers { -struct MercurialInput : Input +struct MercurialInputScheme : InputScheme { - ParsedURL url; - std::optional ref; - std::optional rev; + std::optional inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return {}; + + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); - MercurialInput(const ParsedURL & url) : url(url) - { } + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } - std::string type() const override { return "hg"; } + attrs.emplace("url", url2.to_string()); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + return inputFromAttrs(attrs); } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash") + throw Error("unsupported Mercurial input attribute '%s'", name); - std::optional getRef() const override { return ref; } + parseURL(getStrAttr(attrs, "url")); - std::optional getRev() const override { return rev; } + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + } - ParsedURL toURL() const override + Input input; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override { - ParsedURL url2(url); - url2.scheme = "hg+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); + auto url = parseURL(getStrAttr(input.attrs, "url")); + url.scheme = "hg+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); return url; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + // FIXME: ugly, need to distinguish between dirty and clean + // default trees. + return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); } - std::shared_ptr applyOverrides( + Input applyOverrides( + const Input & input, std::optional ref, - std::optional rev) const override + std::optional rev) override { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); return res; } - std::optional getSourcePath() const + std::optional getSourcePath(const Input & input) override { - if (url.scheme == "file" && !ref && !rev) + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) return url.path; return {}; } - void markChangedFile(std::string_view file, std::optional commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override { - auto sourcePath = getSourcePath(); + auto sourcePath = getSourcePath(input); assert(sourcePath); // FIXME: shut up if file is already tracked. @@ -95,26 +107,27 @@ struct MercurialInput : Input { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); } - std::pair getActualUrl() const + std::pair getActualUrl(const Input & input) const { + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file"; return {isLocal, isLocal ? url.path : url.base}; } - std::pair> fetchTreeInternal(nix::ref store) const override + std::pair fetch(ref store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared(*this); + Input input(_input); - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // FIXME: return lastModified. // FIXME: don't clone local repositories. - if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) { bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; @@ -129,7 +142,7 @@ struct MercurialInput : Input if (settings.warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); - input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl }))); auto files = tokenizeString>( runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); @@ -158,53 +171,50 @@ struct MercurialInput : Input } } - if (!input->ref) input->ref = "default"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); auto getImmutableAttrs = [&]() { return Attrs({ {"type", "hg"}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair> + -> std::pair { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); return { Tree{ .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = getIntAttr(infoAttrs, "revCount"), - }, }, input }; }; - if (input->rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - assert(input->rev || input->ref); - auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef(); Attrs mutableAttrs({ {"type", "hg"}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -213,10 +223,10 @@ struct MercurialInput : Input /* If this is a commit hash that we already have, we don't have to pull again. */ - if (!(input->rev + if (!(input.getRev() && pathExists(cacheDir) && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) .killStderr(true)).second == "1")) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); @@ -245,9 +255,9 @@ struct MercurialInput : Input runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input->rev = Hash(tokens[0], htSHA1); + input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev()); auto revCount = std::stoull(tokens[1]); - input->ref = tokens[2]; + input.attrs.insert_or_assign("ref", tokens[2]); if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); @@ -255,18 +265,18 @@ struct MercurialInput : Input Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); deletePath(tmpDir + "/.hg_archival.txt"); auto storePath = store->addToStore(name, tmpDir); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"revCount", (int64_t) revCount}, }); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -285,54 +295,6 @@ struct MercurialInput : Input } }; -struct MercurialInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "hg+http" && - url.scheme != "hg+https" && - url.scheme != "hg+ssh" && - url.scheme != "hg+file") return nullptr; - - auto url2(url); - url2.scheme = std::string(url2.scheme, 3); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "hg"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev") - throw Error("unsupported Mercurial input attribute '%s'", name); - - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 77fe87d59..cbbb0fa02 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -3,76 +3,86 @@ namespace nix::fetchers { -struct PathInput : Input +struct PathInputScheme : InputScheme { - Path path; + std::optional inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "path") return {}; - /* Allow the user to pass in "fake" tree info attributes. This is - useful for making a pinned tree work the same as the repository - from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - std::optional rev; - std::optional revCount; - std::optional lastModified; + if (url.authority && *url.authority != "") + throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - std::string type() const override { return "path"; } + Input input; + input.attrs.insert_or_assign("type", "path"); + input.attrs.insert_or_assign("path", url.path); - std::optional getRev() const override { return rev; } + for (auto & [name, value] : url.query) + if (name == "rev" || name == "narHash") + input.attrs.insert_or_assign(name, value); + else if (name == "revCount" || name == "lastModified") { + uint64_t n; + if (!string2Int(value, n)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input.attrs.insert_or_assign(name, n); + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && path == other2->path - && rev == other2->rev - && revCount == other2->revCount - && lastModified == other2->lastModified; + return input; } - bool isImmutable() const override + std::optional inputFromAttrs(const Attrs & attrs) override { - return narHash || rev; + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + /* Allow the user to pass in "fake" tree info + attributes. This is useful for making a pinned tree + work the same as the repository from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") + // checked in Input::fromAttrs + ; + else + throw Error("unsupported path input attribute '%s'", name); + + Input input; + input.attrs = attrs; + return input; } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - auto query = attrsToQuery(toAttrsInternal()); + auto query = attrsToQuery(input.attrs); query.erase("path"); + query.erase("type"); return ParsedURL { .scheme = "path", - .path = path, + .path = getStrAttr(input.attrs, "path"), .query = query, }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("path", path); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (revCount) - attrs.emplace("revCount", *revCount); - if (lastModified) - attrs.emplace("lastModified", *lastModified); - if (!rev && narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - return attrs; + return true; } - std::optional getSourcePath() const override + std::optional getSourcePath(const Input & input) override { - return path; + return getStrAttr(input.attrs, "path"); } - void markChangedFile(std::string_view file, std::optional commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override { + // nothing to do } - std::pair> fetchTreeInternal(nix::ref store) const override + std::pair fetch(ref store, const Input & input) override { - auto input = std::make_shared(*this); + auto path = getStrAttr(input.attrs, "path"); // FIXME: check whether access to 'path' is allowed. @@ -85,83 +95,13 @@ struct PathInput : Input // FIXME: try to substitute storePath. storePath = store->addToStore("source", path); - input->narHash = store->queryPathInfo(*storePath)->narHash; - - return - { - Tree { - .actualPath = store->toRealPath(*storePath), - .storePath = std::move(*storePath), - .info = TreeInfo { - .revCount = revCount, - .lastModified = lastModified - } - }, - input - }; - } - -}; - -struct PathInputScheme : InputScheme -{ - std::unique_ptr inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "path") return nullptr; - - auto input = std::make_unique(); - input->path = url.path; - - if (url.authority && *url.authority != "") - throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - - for (auto & [name, value] : url.query) - if (name == "rev") - input->rev = Hash(value, htSHA1); - else if (name == "revCount") { - uint64_t revCount; - if (!string2Int(value, revCount)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->revCount = revCount; - } - else if (name == "lastModified") { - time_t lastModified; - if (!string2Int(value, lastModified)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->lastModified = lastModified; - } - else if (name == "narHash") - // FIXME: require SRI hash. - input->narHash = Hash(value); - else - throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); - - return input; - } - - std::unique_ptr inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - - auto input = std::make_unique(); - input->path = getStrAttr(attrs, "path"); - - for (auto & [name, value] : attrs) - if (name == "rev") - input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); - else if (name == "revCount") - input->revCount = getIntAttr(attrs, "revCount"); - else if (name == "lastModified") - input->lastModified = getIntAttr(attrs, "lastModified"); - else if (name == "narHash") - // FIXME: require SRI hash. - input->narHash = Hash(getStrAttr(attrs, "narHash")); - else if (name == "type" || name == "path") - ; - else - throw Error("unsupported path input attribute '%s'", name); - - return input; + return { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + }, + input + }; } }; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index f6760d2d0..914a0e1e8 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -22,20 +22,7 @@ std::shared_ptr Registry::read( auto version = json.value("version", 0); - // FIXME: remove soon - if (version == 1) { - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - auto url = i->value("url", i->value("uri", "")); - if (url.empty()) - throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", - path, i.key()); - registry->entries.push_back( - {inputFromURL(i.key()), inputFromURL(url), {}}); - } - } - - else if (version == 2) { + if (version == 2) { for (auto & i : json["flakes"]) { auto toAttrs = jsonToAttrs(i["to"]); Attrs extraAttrs; @@ -47,8 +34,8 @@ std::shared_ptr Registry::read( auto exact = i.find("exact"); registry->entries.push_back( Entry { - .from = inputFromAttrs(jsonToAttrs(i["from"])), - .to = inputFromAttrs(toAttrs), + .from = Input::fromAttrs(jsonToAttrs(i["from"])), + .to = Input::fromAttrs(std::move(toAttrs)), .extraAttrs = extraAttrs, .exact = exact != i.end() && exact.value() }); @@ -72,8 +59,8 @@ void Registry::write(const Path & path) nlohmann::json arr; for (auto & entry : entries) { nlohmann::json obj; - obj["from"] = attrsToJson(entry.from->toAttrs()); - obj["to"] = attrsToJson(entry.to->toAttrs()); + obj["from"] = attrsToJson(entry.from.toAttrs()); + obj["to"] = attrsToJson(entry.to.toAttrs()); if (!entry.extraAttrs.empty()) obj["to"].update(attrsToJson(entry.extraAttrs)); if (entry.exact) @@ -90,8 +77,8 @@ void Registry::write(const Path & path) } void Registry::add( - const std::shared_ptr & from, - const std::shared_ptr & to, + const Input & from, + const Input & to, const Attrs & extraAttrs) { entries.emplace_back( @@ -102,11 +89,11 @@ void Registry::add( }); } -void Registry::remove(const std::shared_ptr & input) +void Registry::remove(const Input & input) { // FIXME: use C++20 std::erase. for (auto i = entries.begin(); i != entries.end(); ) - if (*i->from == *input) + if (i->from == input) i = entries.erase(i); else ++i; @@ -145,8 +132,8 @@ std::shared_ptr getFlagRegistry() } void overrideRegistry( - const std::shared_ptr & from, - const std::shared_ptr & to, + const Input & from, + const Input & to, const Attrs & extraAttrs) { flagRegistry->add(from, to, extraAttrs); @@ -180,32 +167,33 @@ Registries getRegistries(ref store) return registries; } -std::pair, Attrs> lookupInRegistries( +std::pair lookupInRegistries( ref store, - std::shared_ptr input) + const Input & _input) { Attrs extraAttrs; int n = 0; + Input input(_input); restart: n++; - if (n > 100) throw Error("cycle detected in flake registr for '%s'", input); + if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); for (auto & registry : getRegistries(store)) { // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { - if (*entry.from == *input) { + if (entry.from == input) { input = entry.to; extraAttrs = entry.extraAttrs; goto restart; } } else { - if (entry.from->contains(*input)) { - input = entry.to->applyOverrides( - !entry.from->getRef() && input->getRef() ? input->getRef() : std::optional(), - !entry.from->getRev() && input->getRev() ? input->getRev() : std::optional()); + if (entry.from.contains(input)) { + input = entry.to.applyOverrides( + !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional(), + !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional()); extraAttrs = entry.extraAttrs; goto restart; } @@ -213,8 +201,8 @@ std::pair, Attrs> lookupInRegistries( } } - if (!input->isDirect()) - throw Error("cannot find flake '%s' in the flake registries", input->to_string()); + if (!input.isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input.to_string()); return {input, extraAttrs}; } diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index c3ce948a8..1077af020 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -20,8 +20,7 @@ struct Registry struct Entry { - std::shared_ptr from; - std::shared_ptr to; + Input from, to; Attrs extraAttrs; bool exact = false; }; @@ -38,11 +37,11 @@ struct Registry void write(const Path & path); void add( - const std::shared_ptr & from, - const std::shared_ptr & to, + const Input & from, + const Input & to, const Attrs & extraAttrs); - void remove(const std::shared_ptr & input); + void remove(const Input & input); }; typedef std::vector> Registries; @@ -54,12 +53,12 @@ Path getUserRegistryPath(); Registries getRegistries(ref store); void overrideRegistry( - const std::shared_ptr & from, - const std::shared_ptr & to, + const Input & from, + const Input & to, const Attrs & extraAttrs); -std::pair, Attrs> lookupInRegistries( +std::pair lookupInRegistries( ref store, - std::shared_ptr input); + const Input & input); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 695525b31..624f8b3fb 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -101,7 +101,7 @@ DownloadFileResult downloadFile( }; } -Tree downloadTarball( +std::pair downloadTarball( ref store, const std::string & url, const std::string & name, @@ -116,12 +116,12 @@ Tree downloadTarball( auto cached = getCache()->lookupExpired(store, inAttrs); if (cached && !cached->expired) - return Tree { - .actualPath = store->toRealPath(cached->storePath), - .storePath = std::move(cached->storePath), - .info = TreeInfo { - .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), + return { + Tree { + .actualPath = store->toRealPath(cached->storePath), + .storePath = std::move(cached->storePath), }, + getIntAttr(cached->infoAttrs, "lastModified") }; auto res = downloadFile(store, url, name, immutable); @@ -156,118 +156,75 @@ Tree downloadTarball( *unpackedStorePath, immutable); - return Tree { - .actualPath = store->toRealPath(*unpackedStorePath), - .storePath = std::move(*unpackedStorePath), - .info = TreeInfo { - .lastModified = lastModified, + return { + Tree { + .actualPath = store->toRealPath(*unpackedStorePath), + .storePath = std::move(*unpackedStorePath), }, + lastModified, }; } -struct TarballInput : Input -{ - ParsedURL url; - std::optional hash; - - TarballInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "tarball"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast(&other); - return - other2 - && to_string() == other2->to_string() - && hash == other2->hash; - } - - bool isImmutable() const override - { - return hash || narHash; - } - - ParsedURL toURL() const override - { - auto url2(url); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. - if (narHash) - url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); - else if (hash) - url2.query.insert_or_assign("hash", hash->to_string(SRI)); - return url2; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (hash) - attrs.emplace("hash", hash->to_string(SRI)); - return attrs; - } - - std::pair> fetchTreeInternal(nix::ref store) const override - { - auto tree = downloadTarball(store, url.to_string(), "source", false); - - auto input = std::make_shared(*this); - input->narHash = store->queryPathInfo(tree.storePath)->narHash; - - return {std::move(tree), input}; - } -}; - struct TarballInputScheme : InputScheme { - std::unique_ptr inputFromURL(const ParsedURL & url) override + std::optional inputFromURL(const ParsedURL & url) override { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {}; if (!hasSuffix(url.path, ".zip") && !hasSuffix(url.path, ".tar") && !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.xz") && !hasSuffix(url.path, ".tar.bz2")) - return nullptr; - - auto input = std::make_unique(url); - - auto hash = input->url.query.find("hash"); - if (hash != input->url.query.end()) { - // FIXME: require SRI hash. - input->hash = Hash(hash->second); - input->url.query.erase(hash); - } - - auto narHash = input->url.query.find("narHash"); - if (narHash != input->url.query.end()) { - // FIXME: require SRI hash. - input->narHash = Hash(narHash->second); - input->url.query.erase(narHash); - } - + return {}; + + Input input; + input.attrs.insert_or_assign("type", "tarball"); + input.attrs.insert_or_assign("url", url.to_string()); + auto narHash = url.query.find("narHash"); + if (narHash != url.query.end()) + input.attrs.insert_or_assign("narHash", narHash->second); return input; } - std::unique_ptr inputFromAttrs(const Attrs & attrs) override + std::optional inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "hash") + if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash") throw Error("unsupported tarball input attribute '%s'", name); - auto input = std::make_unique(parseURL(getStrAttr(attrs, "url"))); - if (auto hash = maybeGetStrAttr(attrs, "hash")) - // FIXME: require SRI hash. - input->hash = Hash(*hash); - + Input input; + input.attrs = attrs; + //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash"); return input; } + + ParsedURL toURL(const Input & input) override + { + auto url = parseURL(getStrAttr(input.attrs, "url")); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (auto narHash = input.getNarHash()) + url.query.insert_or_assign("narHash", narHash->to_string(SRI)); + /* + else if (auto hash = maybeGetStrAttr(input.attrs, "hash")) + url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI)); + */ + return url; + } + + bool hasAllInfo(const Input & input) override + { + return true; + } + + std::pair fetch(ref store, const Input & input) override + { + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + return {std::move(tree), input}; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc deleted file mode 100644 index 42a19cbc8..000000000 --- a/src/libfetchers/tree-info.cc +++ /dev/null @@ -1,60 +0,0 @@ -#include "tree-info.hh" -#include "store-api.hh" - -#include - -namespace nix::fetchers { - -StorePath TreeInfo::computeStorePath(Store & store) const -{ - assert(narHash); - return store.makeFixedOutputPath(true, narHash, "source"); -} - -TreeInfo TreeInfo::fromJson(const nlohmann::json & json) -{ - TreeInfo info; - - auto i = json.find("info"); - if (i != json.end()) { - const nlohmann::json & i2(*i); - - auto j = i2.find("narHash"); - if (j != i2.end()) - info.narHash = Hash((std::string) *j); - else - throw Error("attribute 'narHash' missing in lock file"); - - j = i2.find("revCount"); - if (j != i2.end()) - info.revCount = *j; - - j = i2.find("lastModified"); - if (j != i2.end()) - info.lastModified = *j; - - return info; - } - - i = json.find("narHash"); - if (i != json.end()) { - info.narHash = Hash((std::string) *i); - return info; - } - - throw Error("attribute 'info' missing in lock file"); -} - -nlohmann::json TreeInfo::toJson() const -{ - nlohmann::json json; - assert(narHash); - json["narHash"] = narHash.to_string(SRI); - if (revCount) - json["revCount"] = *revCount; - if (lastModified) - json["lastModified"] = *lastModified; - return json; -} - -} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh deleted file mode 100644 index 3b62151c6..000000000 --- a/src/libfetchers/tree-info.hh +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "path.hh" -#include "hash.hh" - -#include - -namespace nix { class Store; } - -namespace nix::fetchers { - -struct TreeInfo -{ - Hash narHash; - std::optional revCount; - std::optional lastModified; - - bool operator ==(const TreeInfo & other) const - { - return - narHash == other.narHash - && revCount == other.revCount - && lastModified == other.lastModified; - } - - StorePath computeStorePath(Store & store) const; - - static TreeInfo fromJson(const nlohmann::json & json); - - nlohmann::json toJson() const; -}; - -} -- cgit v1.2.3 From 0e7f77a59a90d8cdb9560feeff4f3a48ab888843 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 30 May 2020 00:59:13 +0200 Subject: Check revCount / lastModified input attributes if specified --- src/libfetchers/fetchers.cc | 18 ++++++++++++++---- src/libfetchers/github.cc | 1 + 2 files changed, 15 insertions(+), 4 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e4852d662..aac8aa8c5 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -111,13 +111,23 @@ std::pair Input::fetch(ref store) const auto narHash = store->queryPathInfo(tree.storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(SRI)); - if (auto narHash2 = getNarHash()) { - if (narHash != *narHash2) + if (auto prevNarHash = getNarHash()) { + if (narHash != *prevNarHash) throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash2->to_string(SRI), narHash.to_string(SRI)); + to_string(), tree.actualPath, prevNarHash->to_string(SRI), narHash.to_string(SRI)); } - // FIXME: check lastModified, revCount + if (auto prevLastModified = getLastModified()) { + if (input.getLastModified() != prevLastModified) + throw Error("'lastModified' attribute mismatch in input '%s', expected %d", + input.to_string(), *prevLastModified); + } + + if (auto prevRevCount = getRevCount()) { + if (input.getRevCount() != prevRevCount) + throw Error("'revCount' attribute mismatch in input '%s', expected %d", + input.to_string(), *prevRevCount); + } input.immutable = true; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8d113967e..d20b5d00c 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -45,6 +45,7 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains multiple branch/tag names", url.url); ref = value; } + // FIXME: barf on unsupported attributes } if (ref && rev) -- cgit v1.2.3 From 89e0b3e2d62f72e48bdda63b77a086c69b314113 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 30 May 2020 01:16:53 +0200 Subject: Move substitution into Input::fetch() Closes #3520. --- src/libfetchers/fetchers.cc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index aac8aa8c5..f0d8f72c8 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -103,6 +103,26 @@ std::pair Input::fetch(ref store) const if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs())); + /* The tree may already be in the Nix store, or it could be + substituted (which is often faster than fetching from the + original source). So check that. */ + if (hasAllInfo()) { + try { + auto storePath = computeStorePath(*store); + + store->ensurePath(storePath); + + debug("using substituted/cached input '%s' in '%s'", + to_string(), store->printStorePath(storePath)); + + auto actualPath = store->toRealPath(storePath); + + return {fetchers::Tree { .actualPath = actualPath, .storePath = std::move(storePath) }, *this}; + } catch (Error & e) { + debug("substitution of input '%s' failed: %s", to_string(), e.what()); + } + } + auto [tree, input] = scheme->fetch(store, *this); if (tree.actualPath == "") -- cgit v1.2.3 From 768099350666f103131c59853cc7d70c0a6e19cd Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Mon, 1 Jun 2020 09:01:37 -0600 Subject: Tree ctors --- src/libfetchers/fetchers.hh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index c43cfe50c..2e8c534b0 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -16,6 +16,8 @@ struct Tree { Path actualPath; StorePath storePath; + Tree(Path && actualPath, StorePath && storePath) : actualPath(actualPath), storePath(std::move(storePath)) {} + Tree (const Tree & rhs) : actualPath(rhs.actualPath), storePath(rhs.storePath.clone()) {} }; struct InputScheme; -- cgit v1.2.3 From c254254a8088ca303e97b9430484cf25cf8f7806 Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Mon, 1 Jun 2020 08:59:26 -0600 Subject: use Tree ctor --- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/git.cc | 11 +++-------- src/libfetchers/github.cc | 5 +---- src/libfetchers/mercurial.cc | 13 +++++-------- src/libfetchers/path.cc | 5 +---- src/libfetchers/tarball.cc | 10 ++-------- 6 files changed, 13 insertions(+), 33 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index f0d8f72c8..dae8b9fb2 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -117,7 +117,7 @@ std::pair Input::fetch(ref store) const auto actualPath = store->toRealPath(storePath); - return {fetchers::Tree { .actualPath = actualPath, .storePath = std::move(storePath) }, *this}; + return {fetchers::Tree(std::move(actualPath), std::move(storePath)), *this}; } catch (Error & e) { debug("substitution of input '%s' failed: %s", to_string(), e.what()); } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 4fcf3f542..af4efc9b1 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -189,10 +189,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return { - Tree { - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - }, + Tree(store->toRealPath(storePath), std::move(storePath)), input }; }; @@ -273,10 +270,8 @@ struct GitInputScheme : InputScheme haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0); return { - Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - }, input + Tree(store->printStorePath(storePath), std::move(storePath)), + input }; } } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index d20b5d00c..cf2554a50 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -140,10 +140,7 @@ struct GitArchiveInputScheme : InputScheme if (auto res = getCache()->lookup(store, immutableAttrs)) { input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); return { - Tree{ - .actualPath = store->toRealPath(res->second), - .storePath = std::move(res->second), - }, + Tree(store->toRealPath(res->second), std::move(res->second)), input }; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 49ed63243..9fae32b83 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -164,10 +164,10 @@ struct MercurialInputScheme : InputScheme auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); - return {Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - }, input}; + return { + Tree(store->printStorePath(storePath), std::move(storePath)), + input + }; } } @@ -189,10 +189,7 @@ struct MercurialInputScheme : InputScheme assert(!_input.getRev() || _input.getRev() == input.getRev()); input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); return { - Tree{ - .actualPath = store->toRealPath(storePath), - .storePath = std::move(storePath), - }, + Tree(store->toRealPath(storePath), std::move(storePath)), input }; }; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index cbbb0fa02..99d4b4e8f 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -96,10 +96,7 @@ struct PathInputScheme : InputScheme storePath = store->addToStore("source", path); return { - Tree { - .actualPath = store->toRealPath(*storePath), - .storePath = std::move(*storePath), - }, + Tree(store->toRealPath(*storePath), std::move(*storePath)), input }; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 624f8b3fb..9479bb1b3 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -117,10 +117,7 @@ std::pair downloadTarball( if (cached && !cached->expired) return { - Tree { - .actualPath = store->toRealPath(cached->storePath), - .storePath = std::move(cached->storePath), - }, + Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)), getIntAttr(cached->infoAttrs, "lastModified") }; @@ -157,10 +154,7 @@ std::pair downloadTarball( immutable); return { - Tree { - .actualPath = store->toRealPath(*unpackedStorePath), - .storePath = std::move(*unpackedStorePath), - }, + Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)), lastModified, }; } -- cgit v1.2.3 From 108debef6f703aa3ca1f6d6c45428880624ec6f8 Mon Sep 17 00:00:00 2001 From: Finn Behrens Date: Thu, 4 Jun 2020 13:36:30 +0200 Subject: add support for selfhosted gitlab/github --- src/libfetchers/github.cc | 56 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 15 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index cf2554a50..09a8289c4 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,6 +8,20 @@ namespace nix::fetchers { +// get the default url attribute +std::string getDefaultUrl(const Attrs & attrs, const std::string & url) { + auto s = maybeGetStrAttr(attrs, "url"); + if (!s) + return url; + return *s; +} + + +// A github or gitlab url +const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check +std::regex urlRegex(urlRegexS, std::regex::ECMAScript); + + struct GitArchiveInputScheme : InputScheme { virtual std::string type() = 0; @@ -20,6 +34,7 @@ struct GitArchiveInputScheme : InputScheme std::optional rev; std::optional ref; + std::optional host_url; if (path.size() == 2) { } else if (path.size() == 3) { @@ -45,6 +60,11 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains multiple branch/tag names", url.url); ref = value; } + else if (name == "url") { + if (!std::regex_match(value, urlRegex)) + throw BadURL("URL '%s' contains an invalid instance url", url.url); + host_url = value; + } // FIXME: barf on unsupported attributes } @@ -57,6 +77,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); if (ref) input.attrs.insert_or_assign("ref", *ref); + if (host_url) input.attrs.insert_or_assign("url", *host_url); return input; } @@ -171,8 +192,9 @@ struct GitHubInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + auto host_url = getDefaultUrl(input.attrs, "github.com"); + auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( @@ -186,9 +208,9 @@ struct GitHubInputScheme : GitArchiveInputScheme { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. - - auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + auto host_url = getDefaultUrl(input.attrs, "github.com"); + auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); std::string accessToken = settings.githubAccessToken.get(); @@ -200,8 +222,9 @@ struct GitHubInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) override { - Input::fromURL(fmt("git+ssh://git@github.com/%s/%s.git", - getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + auto host_url = getDefaultUrl(input.attrs, "github.com"); + Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef().value_or("master"), input.getRev()) .clone(destDir); } @@ -213,8 +236,9 @@ struct GitLabInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/branches/%s", - getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); + auto host_url = getDefaultUrl(input.attrs, "gitlab.com"); + auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/branches/%s", + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( @@ -226,10 +250,10 @@ struct GitLabInputScheme : GitArchiveInputScheme std::string getDownloadUrl(const Input & input) const override { - // FIXME: This endpoint has a rate limit threshold of 5 requests per minute. - - auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", - getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + // FIXME: This endpoint has a rate limit threshold of 5 requests per minute + auto host_url = getDefaultUrl(input.attrs, "gitlab.com"); + auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: "`) @@ -242,8 +266,10 @@ struct GitLabInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) override { - Input::fromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", - getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + auto host_url = getDefaultUrl(input.attrs, "gitlab.com"); + // FIXME: get username somewhere + Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", + host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef().value_or("master"), input.getRev()) .clone(destDir); } -- cgit v1.2.3 From ab54031e044af73e38f1610ac825c424ae058aa5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Jun 2020 20:24:28 +0200 Subject: getDefaultUrl() -> value_or() --- src/libfetchers/github.cc | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 09a8289c4..e704592f7 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,20 +8,10 @@ namespace nix::fetchers { -// get the default url attribute -std::string getDefaultUrl(const Attrs & attrs, const std::string & url) { - auto s = maybeGetStrAttr(attrs, "url"); - if (!s) - return url; - return *s; -} - - // A github or gitlab url const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check std::regex urlRegex(urlRegexS, std::regex::ECMAScript); - struct GitArchiveInputScheme : InputScheme { virtual std::string type() = 0; @@ -192,7 +182,7 @@ struct GitHubInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host_url = getDefaultUrl(input.attrs, "github.com"); + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( @@ -208,7 +198,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. - auto host_url = getDefaultUrl(input.attrs, "github.com"); + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); @@ -222,7 +212,7 @@ struct GitHubInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) override { - auto host_url = getDefaultUrl(input.attrs, "github.com"); + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef().value_or("master"), input.getRev()) @@ -236,7 +226,7 @@ struct GitLabInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { - auto host_url = getDefaultUrl(input.attrs, "gitlab.com"); + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/branches/%s", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( @@ -251,7 +241,7 @@ struct GitLabInputScheme : GitArchiveInputScheme std::string getDownloadUrl(const Input & input) const override { // FIXME: This endpoint has a rate limit threshold of 5 requests per minute - auto host_url = getDefaultUrl(input.attrs, "gitlab.com"); + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); @@ -266,7 +256,7 @@ struct GitLabInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) override { - auto host_url = getDefaultUrl(input.attrs, "gitlab.com"); + auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); // FIXME: get username somewhere Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) -- cgit v1.2.3 From 6cfc2db49424c63b7ca6b837e8a4cb7bf373c01b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 9 Jun 2020 13:45:07 +0200 Subject: Fix applyOverride() for github --- src/libfetchers/fetchers.cc | 1 + src/libfetchers/github.cc | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 067db5e2e..d808aadd6 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -91,6 +91,7 @@ bool Input::operator ==(const Input & other) const bool Input::contains(const Input & other) const { + if (*this == other) return true; auto other2(other); other2.attrs.erase("ref"); other2.attrs.erase("rev"); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index e704592f7..8dc9cb4bf 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -59,7 +59,7 @@ struct GitArchiveInputScheme : InputScheme } if (ref && rev) - throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url); + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); Input input; input.attrs.insert_or_assign("type", type()); @@ -115,14 +115,16 @@ struct GitArchiveInputScheme : InputScheme std::optional rev) override { auto input(_input); + if (rev && ref) + throw BadURL("cannot apply both a commit hash (%s) and a branch/tag name ('%s') to input '%s'", + rev->gitRev(), *ref, input.to_string()); if (rev) { input.attrs.insert_or_assign("rev", rev->gitRev()); input.attrs.erase("ref"); } if (ref) { - if (input.getRev()) - throw BadURL("input '%s' contains both a commit hash and a branch/tag name", input.to_string()); input.attrs.insert_or_assign("ref", *ref); + input.attrs.erase("rev"); } return input; } -- cgit v1.2.3 From 3d492199bb9a4c78fa6e474bb5a67df72db189c7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Jun 2020 13:25:08 +0200 Subject: github: Respect default branch --- src/libfetchers/github.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 12d8d346c..4c1a140ff 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -137,7 +137,7 @@ struct GitArchiveInputScheme : InputScheme { Input input(_input); - if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "master"); + if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD"); auto rev = input.getRev(); if (!rev) rev = getRevFromRef(store, input); @@ -217,7 +217,7 @@ struct GitHubInputScheme : GitArchiveInputScheme auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com"); Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) - .applyOverrides(input.getRef().value_or("master"), input.getRev()) + .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) .clone(destDir); } }; @@ -262,7 +262,7 @@ struct GitLabInputScheme : GitArchiveInputScheme // FIXME: get username somewhere Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) - .applyOverrides(input.getRef().value_or("master"), input.getRev()) + .applyOverrides(input.getRef().value_or("HEAD"), input.getRev()) .clone(destDir); } }; -- cgit v1.2.3 From 507aa48739f23f9a16c8b7079bfa6fc1806be78e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Jun 2020 18:41:33 +0000 Subject: WIP: Make Hash always store a valid hash type --- src/libfetchers/tree-info.cc | 2 +- src/libfetchers/tree-info.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index b2d8cfc8d..432aa6182 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -8,7 +8,7 @@ namespace nix::fetchers { StorePath TreeInfo::computeStorePath(Store & store) const { assert(narHash); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source"); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); } } diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh index 2c7347281..9d1872097 100644 --- a/src/libfetchers/tree-info.hh +++ b/src/libfetchers/tree-info.hh @@ -11,7 +11,7 @@ namespace nix::fetchers { struct TreeInfo { - Hash narHash; + std::optional narHash; std::optional revCount; std::optional lastModified; -- cgit v1.2.3 From 7d554f295ca0a8947042f367adb4f198730696be Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Jul 2020 14:57:59 +0200 Subject: Support building flakes from a shallow Git repo Fixes #3756. --- src/libfetchers/git.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libfetchers') diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 0b6155e71..5d38e0c2b 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -42,6 +42,8 @@ struct GitInputScheme : InputScheme for (auto &[name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); + else if (name == "shallow") + attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); } -- cgit v1.2.3 From d746503e5c6fafd0fb6b2a4e6527c12cfc626637 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Jul 2020 20:23:39 +0200 Subject: Add --inputs-from to use flake inputs as registry entries This allows you to refer to an input from another flake. For example, $ nix run --inputs-from /path/to/hydra nixpkgs#hello runs 'hello' from the 'nixpkgs' inputs of the 'hydra' flake. Fixes #3769. --- src/libfetchers/registry.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libfetchers') diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 914a0e1e8..d4134ce29 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -204,6 +204,8 @@ std::pair lookupInRegistries( if (!input.isDirect()) throw Error("cannot find flake '%s' in the flake registries", input.to_string()); + debug("looked up '%s' -> '%s'", _input.to_string(), input.to_string()); + return {input, extraAttrs}; } -- cgit v1.2.3 From 6f8fd3a3f22de24223bba05635aaa62107d92767 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 3 Jul 2020 14:50:07 +0200 Subject: Shut up a clang warning --- src/libfetchers/cache.hh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/libfetchers') diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index d76ab1233..3db4f081c 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -6,6 +6,8 @@ namespace nix::fetchers { struct Cache { + virtual ~Cache() { } + virtual void add( ref store, const Attrs & inAttrs, -- cgit v1.2.3 From cf9f33995bb0e992b84ade84b3ffa659eab1a9d8 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Mon, 13 Jul 2020 19:22:59 +0200 Subject: Fix gitlab-fetcher to obtain tags and branches Until now, the `gitlab`-fetcher determined the source's rev by checking the latest commit of the given `ref` using the `/repository/branches`-API. This breaks however when trying to fetch a gitlab-repo by its tag: ``` $ nix repl nix-repl> builtins.fetchTree gitlab:Ma27/nvim.nix/0.2.0 error: --- Error ------------------------------------------------------------------------------------- nix unable to download 'https://gitlab.com/api/v4/projects/Ma27%2Fnvim.nix/repository/branches/0.2.0': HTTP error 404 ('') ``` When using the `/commits?ref_name`-endpoint[1] you can pass any kind of valid ref to the `gitlab`-fetcher. Please note that this fetches the only first 20 commits on a ref, unfortunately there's currently no endpoint which only retrieves the latest commit of any kind of `ref`. [1] https://docs.gitlab.com/ee/api/commits.html#list-repository-commits --- src/libfetchers/github.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 4c1a140ff..8bb7c2c1d 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -229,13 +229,13 @@ struct GitLabInputScheme : GitArchiveInputScheme Hash getRevFromRef(nix::ref store, const Input & input) const override { auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com"); - auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/branches/%s", + auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( downloadFile(store, url, "source", false).storePath))); - auto rev = Hash(std::string(json["commit"]["id"]), htSHA1); + auto rev = Hash(std::string(json[0]["id"]), htSHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } -- cgit v1.2.3 From f74243846512ffabf082985bca395890c97643e0 Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Wed, 29 Apr 2020 22:39:58 +0200 Subject: Merge legacy `fetchGit`-builtin with the generic `fetchTree`-function The original idea was to implement a git-fetcher in Nix's core that supports content hashes[1]. In #3549[2] it has been suggested to actually use `fetchTree` for this since it's a fairly generic wrapper over the new fetcher-API[3] and already supports content-hashes. This patch implements a new git-fetcher based on `fetchTree` by incorporating the following changes: * Removed the original `fetchGit`-implementation and replaced it with an alias on the `fetchTree` implementation. * Ensured that the `git`-fetcher from `libfetchers` always computes a content-hash and returns an "empty" revision on dirty trees (the latter one is needed to retain backwards-compatibility). * The hash-mismatch error in the fetcher-API exits with code 102 as it usually happens whenever a hash-mismatch is detected by Nix. * Removed the `flakes`-feature-flag: I didn't see a reason why this API is so tightly coupled to the flakes-API and at least `fetchGit` should remain usable without any feature-flags. * It's only possible to specify a `narHash` for a `git`-tree if either a `ref` or a `rev` is given[4]. * It's now possible to specify an URL without a protocol. If it's missing, `file://` is automatically added as it was the case in the original `fetchGit`-implementation. [1] https://github.com/NixOS/nix/pull/3216 [2] https://github.com/NixOS/nix/pull/3549#issuecomment-625194383 [3] https://github.com/NixOS/nix/pull/3459 [4] https://github.com/NixOS/nix/pull/3216#issuecomment-553956703 --- src/libfetchers/fetchers.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/libfetchers') diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index c3ee9bf43..28db8aa9c 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -134,7 +134,7 @@ std::pair Input::fetch(ref store) const if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) - throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true)); } -- cgit v1.2.3