diff options
author | Eelco Dolstra <edolstra@gmail.com> | 2020-05-30 00:44:11 +0200 |
---|---|---|
committer | Eelco Dolstra <edolstra@gmail.com> | 2020-05-30 00:44:11 +0200 |
commit | 950b46821f644eb3f92725460584a3102f356179 (patch) | |
tree | 0916b068dad6c03dfb81a80dd51b199464376bf4 | |
parent | 5633c0975b9dec6bceaa85003223a346d7d1bd0b (diff) |
Remove TreeInfo
The attributes previously stored in TreeInfo (narHash, revCount,
lastModified) are now stored in Input. This makes it less arbitrary
what attributes are stored where.
As a result, the lock file format has changed. An entry like
"info": {
"lastModified": 1585405475,
"narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE="
},
"locked": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be",
"type": "github"
},
is now stored as
"locked": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be",
"type": "github",
"lastModified": 1585405475,
"narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE="
},
The 'Input' class is now a dumb set of attributes. All the fetcher
implementations subclass InputScheme, not Input. This simplifies the
API.
Also, fix substitution of flake inputs. This was broken since lazy
flake fetching started using fetchTree internally.
30 files changed, 939 insertions, 1121 deletions
diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 8665abe8c..6b48ead1f 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -76,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s) if (isUri(s)) { return state.store->toRealPath( fetchers::downloadTarball( - state.store, resolveUri(s), "source", false).storePath); + state.store, resolveUri(s), "source", false).first.storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index 8ee17b8f4..2084e3fb3 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -11,7 +11,7 @@ let sourceInfo = if key == lockFile.root then rootSrc - else fetchTree ({ inherit (node.info) narHash; } // removeAttrs node.locked ["dir"]); + else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); inputs = builtins.mapAttrs (inputName: key: allNodes.${key}) (node.inputs or {}); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 78b58cdfa..9741d98c5 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -19,7 +19,7 @@ static FlakeRef maybeLookupFlake( const FlakeRef & flakeRef, bool allowLookup) { - if (!flakeRef.input->isDirect()) { + if (!flakeRef.input.isDirect()) { if (allowLookup) return flakeRef.resolve(store); else @@ -49,16 +49,15 @@ static FlakeRef lookupInFlakeCache( static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, - std::optional<TreeInfo> treeInfo, bool allowLookup, FlakeCache & flakeCache) { /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. */ - if (treeInfo && originalRef.input->isDirect() && originalRef.input->isImmutable()) { + if (originalRef.input.isDirect() && originalRef.input.isImmutable() && originalRef.input.hasAllInfo()) { try { - auto storePath = treeInfo->computeStorePath(*state.store); + auto storePath = originalRef.input.computeStorePath(*state.store); state.store->ensurePath(storePath); @@ -74,7 +73,6 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( Tree { .actualPath = actualPath, .storePath = std::move(storePath), - .info = *treeInfo, }, originalRef, originalRef @@ -99,8 +97,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); - if (treeInfo) - assert(tree.storePath == treeInfo->computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); return {std::move(tree), resolvedRef, lockedRef}; } @@ -202,12 +199,11 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs( static Flake getFlake( EvalState & state, const FlakeRef & originalRef, - std::optional<TreeInfo> treeInfo, bool allowLookup, FlakeCache & flakeCache) { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, originalRef, treeInfo, allowLookup, flakeCache); + state, originalRef, allowLookup, flakeCache); // Guard against symlink attacks. auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix"); @@ -278,7 +274,7 @@ static Flake getFlake( Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) { FlakeCache flakeCache; - return getFlake(state, originalRef, {}, allowLookup, flakeCache); + return getFlake(state, originalRef, allowLookup, flakeCache); } /* Compute an in-memory lock file for the specified top-level flake, @@ -292,7 +288,7 @@ LockedFlake lockFlake( FlakeCache flakeCache; - auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache); + auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); // FIXME: symlink attack auto oldLockFile = LockFile::read( @@ -393,7 +389,7 @@ LockedFlake lockFlake( didn't change and there is no override from a higher level flake. */ auto childNode = std::make_shared<LockedNode>( - oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake); + oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); node->inputs.insert_or_assign(id, childNode); @@ -409,7 +405,7 @@ LockedFlake lockFlake( if (hasChildUpdate) { auto inputFlake = getFlake( - state, oldLock->lockedRef, oldLock->info, false, flakeCache); + state, oldLock->lockedRef, false, flakeCache); computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); } else { /* No need to fetch this flake, we can be @@ -440,11 +436,11 @@ LockedFlake lockFlake( /* We need to create a new lock file entry. So fetch this input. */ - if (!lockFlags.allowMutable && !input.ref.input->isImmutable()) + if (!lockFlags.allowMutable && !input.ref.input.isImmutable()) throw Error("cannot update flake input '%s' in pure mode", inputPathS); if (input.isFlake) { - auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache); + auto inputFlake = getFlake(state, input.ref, lockFlags.useRegistries, flakeCache); /* Note: in case of an --override-input, we use the *original* ref (input2.ref) for the @@ -454,7 +450,7 @@ LockedFlake lockFlake( file. That is, overrides are sticky unless you use --no-write-lock-file. */ auto childNode = std::make_shared<LockedNode>( - inputFlake.lockedRef, input2.ref, inputFlake.sourceInfo->info); + inputFlake.lockedRef, input2.ref); node->inputs.insert_or_assign(id, childNode); @@ -479,9 +475,9 @@ LockedFlake lockFlake( else { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, input.ref, {}, lockFlags.useRegistries, flakeCache); + state, input.ref, lockFlags.useRegistries, flakeCache); node->inputs.insert_or_assign(id, - std::make_shared<LockedNode>(lockedRef, input.ref, sourceInfo.info, false)); + std::make_shared<LockedNode>(lockedRef, input.ref, false)); } } } @@ -534,7 +530,7 @@ LockedFlake lockFlake( printInfo("inputs of flake '%s' changed:\n%s", topRef, chomp(diff)); if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input->getSourcePath()) { + if (auto sourcePath = topRef.input.getSourcePath()) { if (!newLockFile.isImmutable()) { if (settings.warnDirty) warn("will not write lock file of flake '%s' because it has a mutable input", topRef); @@ -555,7 +551,7 @@ LockedFlake lockFlake( newLockFile.write(path); - topRef.input->markChangedFile( + topRef.input.markChangedFile( (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", lockFlags.commitLockFile ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s", @@ -567,19 +563,19 @@ LockedFlake lockFlake( also just clear the 'rev' field... */ auto prevLockedRef = flake.lockedRef; FlakeCache dummyCache; - flake = getFlake(state, topRef, {}, lockFlags.useRegistries, dummyCache); + flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache); if (lockFlags.commitLockFile && - flake.lockedRef.input->getRev() && - prevLockedRef.input->getRev() != flake.lockedRef.input->getRev()) - warn("committed new revision '%s'", flake.lockedRef.input->getRev()->gitRev()); + flake.lockedRef.input.getRev() && + prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) + warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); /* Make sure that we picked up the change, i.e. the tree should usually be dirty now. Corner case: we could have reverted from a dirty to a clean tree! */ if (flake.lockedRef.input == prevLockedRef.input - && !flake.lockedRef.input->isImmutable()) + && !flake.lockedRef.input.isImmutable()) throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); } } else @@ -625,7 +621,7 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va { auto flakeRefS = state.forceStringNoCtx(*args[0], pos); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (evalSettings.pureEval && !flakeRef.input->isImmutable()) + if (evalSettings.pureEval && !flakeRef.input.isImmutable()) throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); callFlake(state, @@ -650,8 +646,8 @@ Fingerprint LockedFlake::getFingerprint() const return hashString(htSHA256, fmt("%s;%d;%d;%s", flake.sourceInfo->storePath.to_string(), - flake.sourceInfo->info.revCount.value_or(0), - flake.sourceInfo->info.lastModified.value_or(0), + flake.lockedRef.input.getRevCount().value_or(0), + flake.lockedRef.input.getLastModified().value_or(0), lockFile)); } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 59a1adb3b..ebf81362c 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -104,7 +104,7 @@ void callFlake( void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, - std::shared_ptr<const fetchers::Input> input, + const fetchers::Input & input, Value & v); } diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index a70261a41..615269218 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -15,7 +15,7 @@ const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRege std::string FlakeRef::to_string() const { - auto url = input->toURL(); + auto url = input.toURL(); if (subdir != "") url.query.insert_or_assign("dir", subdir); return url.to_string(); @@ -23,7 +23,7 @@ std::string FlakeRef::to_string() const fetchers::Attrs FlakeRef::toAttrs() const { - auto attrs = input->toAttrs(); + auto attrs = input.toAttrs(); if (subdir != "") attrs.emplace("dir", subdir); return attrs; @@ -37,13 +37,13 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) bool FlakeRef::operator ==(const FlakeRef & other) const { - return *input == *other.input && subdir == other.subdir; + return input == other.input && subdir == other.subdir; } FlakeRef FlakeRef::resolve(ref<Store> store) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); - return FlakeRef(input2, fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); + return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); } FlakeRef parseFlakeRef( @@ -98,7 +98,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(inputFromURL(parsedURL), ""), + FlakeRef(Input::fromURL(parsedURL), ""), percentDecode(std::string(match[6]))); } @@ -143,7 +143,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( } return std::make_pair( - FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), fragment); } @@ -155,7 +155,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( attrs.insert_or_assign("type", "path"); attrs.insert_or_assign("path", path); - return std::make_pair(FlakeRef(inputFromAttrs(attrs), ""), fragment); + return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); } else { @@ -163,7 +163,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( std::string fragment; std::swap(fragment, parsedURL.fragment); return std::make_pair( - FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), fragment); } } @@ -183,14 +183,14 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) auto attrs2(attrs); attrs2.erase("dir"); return FlakeRef( - fetchers::inputFromAttrs(attrs2), + fetchers::Input::fromAttrs(std::move(attrs2)), fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const { - auto [tree, lockedInput] = input->fetchTree(store); - return {std::move(tree), FlakeRef(lockedInput, subdir)}; + auto [tree, lockedInput] = input.fetch(store); + return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; } } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 72cbb2908..f4eb825a6 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -14,17 +14,15 @@ typedef std::string FlakeId; struct FlakeRef { - std::shared_ptr<const fetchers::Input> input; + fetchers::Input input; Path subdir; bool operator==(const FlakeRef & other) const; - FlakeRef(const std::shared_ptr<const fetchers::Input> & input, const Path & subdir) - : input(input), subdir(subdir) - { - assert(input); - } + FlakeRef(fetchers::Input && input, const Path & subdir) + : input(std::move(input)), subdir(subdir) + { } // FIXME: change to operator <<. std::string to_string() const; diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 5c58d6080..68e587650 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -5,35 +5,40 @@ namespace nix::flake { -FlakeRef flakeRefFromJson(const nlohmann::json & json) -{ - return FlakeRef::fromAttrs(jsonToAttrs(json)); -} - FlakeRef getFlakeRef( const nlohmann::json & json, - const char * attr) + const char * attr, + const char * info) { auto i = json.find(attr); - if (i != json.end()) - return flakeRefFromJson(*i); + if (i != json.end()) { + auto attrs = jsonToAttrs(*i); + // FIXME: remove when we drop support for version 5. + if (info) { + auto j = json.find(info); + if (j != json.end()) { + for (auto k : jsonToAttrs(*j)) + attrs.insert_or_assign(k.first, k.second); + } + } + return FlakeRef::fromAttrs(attrs); + } throw Error("attribute '%s' missing in lock file", attr); } LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked")) - , originalRef(getFlakeRef(json, "original")) - , info(TreeInfo::fromJson(json)) + : lockedRef(getFlakeRef(json, "locked", "info")) + , originalRef(getFlakeRef(json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { - if (!lockedRef.input->isImmutable()) - throw Error("lockfile contains mutable flakeref '%s'", lockedRef); + if (!lockedRef.input.isImmutable()) + throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs())); } StorePath LockedNode::computeStorePath(Store & store) const { - return info.computeStorePath(store); + return lockedRef.input.computeStorePath(store); } std::shared_ptr<Node> Node::findInput(const InputPath & path) @@ -53,7 +58,7 @@ std::shared_ptr<Node> Node::findInput(const InputPath & path) LockFile::LockFile(const nlohmann::json & json, const Path & path) { auto version = json.value("version", 0); - if (version != 5) + if (version < 5 || version > 6) throw Error("lock file '%s' has unsupported version %d", path, version); std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap; @@ -119,7 +124,6 @@ nlohmann::json LockFile::toJson() const if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) { n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs()); n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs()); - n["info"] = lockedNode->info.toJson(); if (!lockedNode->isFlake) n["flake"] = false; } @@ -129,7 +133,7 @@ nlohmann::json LockFile::toJson() const }; nlohmann::json json; - json["version"] = 5; + json["version"] = 6; json["root"] = dumpNode("root", root); json["nodes"] = std::move(nodes); @@ -176,7 +180,7 @@ bool LockFile::isImmutable() const for (auto & i : nodes) { if (i == root) continue; auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i); - if (lockedNode && !lockedNode->lockedRef.input->isImmutable()) return false; + if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false; } return true; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index c34939ebc..ba47f9b89 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -31,15 +31,13 @@ struct Node : std::enable_shared_from_this<Node> struct LockedNode : Node { FlakeRef lockedRef, originalRef; - TreeInfo info; bool isFlake = true; LockedNode( const FlakeRef & lockedRef, const FlakeRef & originalRef, - const TreeInfo & info, bool isFlake = true) - : lockedRef(lockedRef), originalRef(originalRef), info(info), isFlake(isFlake) + : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) { } LockedNode(const nlohmann::json & json); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 1993fa6c1..1ac5217ba 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -689,7 +689,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { res = { true, store->toRealPath(fetchers::downloadTarball( - store, resolveUri(elem.second), "source", false).storePath) }; + store, resolveUri(elem.second), "source", false).first.storePath) }; } catch (FileTransferError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 1a8798fcc..a6539e888 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -56,23 +56,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - if (fetchSubmodules) attrs.insert_or_assign("submodules", true); - auto input = fetchers::inputFromAttrs(attrs); + if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit<bool>{true}); + auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name? - auto [tree, input2] = input->fetchTree(state.store); + auto [tree, input2] = input.fetch(state.store); state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(htSHA1)); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); // Backward compatibility: set 'revCount' to 0 for a dirty tree. mkInt(*state.allocAttr(v, state.symbols.create("revCount")), - tree.info.revCount.value_or(0)); + input2.getRevCount().value_or(0)); mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); v.attrs->sort(); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 0a1ba49d5..90030ea76 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -59,23 +59,23 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::inputFromAttrs(attrs); + auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input->fetchTree(state.store); + auto [tree, input2] = input.fetch(state.store); state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - if (input2->getRef()) - mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + if (input2.getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(htSHA1)); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (auto revCount = input2.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); v.attrs->sort(); if (state.allowedPaths) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f9dfb1164..a1ad0a7b9 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -13,31 +13,36 @@ namespace nix { void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, - std::shared_ptr<const fetchers::Input> input, + const fetchers::Input & input, Value & v) { + assert(input.isImmutable()); + state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - assert(tree.info.narHash); + // FIXME: support arbitrary input attributes. + + auto narHash = input.getNarHash(); + assert(narHash); mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI)); + narHash->to_string(SRI)); - if (input->getRev()) { - mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + if (auto rev = input.getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); } - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (auto revCount = input.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); - if (tree.info.lastModified) { - mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *tree.info.lastModified); + if (auto lastModified = input.getLastModified()) { + mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")), - fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S"))); } v.attrs->sort(); @@ -47,7 +52,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V { settings.requireExperimentalFeature("flakes"); - std::shared_ptr<const fetchers::Input> input; + fetchers::Input input; PathSet context; state.forceValue(*args[0]); @@ -62,7 +67,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V if (attr.value->type == tString) attrs.emplace(attr.name, attr.value->string.s); else if (attr.value->type == tBool) - attrs.emplace(attr.name, attr.value->boolean); + attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean}); else if (attr.value->type == tInt) attrs.emplace(attr.name, attr.value->integer); else @@ -73,18 +78,42 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V if (!attrs.count("type")) throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos); - input = fetchers::inputFromAttrs(attrs); + input = fetchers::Input::fromAttrs(std::move(attrs)); } else - input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false)); - if (!evalSettings.pureEval && !input->isDirect()) + if (!evalSettings.pureEval && !input.isDirect()) input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input->isImmutable()) + if (evalSettings.pureEval && !input.isImmutable()) throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos); - // FIXME: use fetchOrSubstituteTree - auto [tree, input2] = input->fetchTree(state.store); + /* The tree may already be in the Nix store, or it could be + substituted (which is often faster than fetching from the + original source). So check that. */ + if (input.hasAllInfo()) { + auto storePath = input.computeStorePath(*state.store); + + try { + state.store->ensurePath(storePath); + + debug("using substituted/cached input '%s' in '%s'", + input.to_string(), state.store->printStorePath(storePath)); + + auto actualPath = state.store->toRealPath(storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(actualPath); + + emitTreeAttrs(state, fetchers::Tree { .actualPath = actualPath, .storePath = std::move(storePath) }, input, v); + + return; + } catch (Error & e) { + debug("substitution of input '%s' failed: %s", input.to_string(), e.what()); + } + } + + auto [tree, input2] = input.fetch(state.store); if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); @@ -137,7 +166,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; auto path = state.store->toRealPath(storePath); diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index feb0a6085..1e59faa73 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if<int64_t>(&attr.second)) { + if (auto v = std::get_if<uint64_t>(&attr.second)) { json[attr.first] = *v; } else if (auto v = std::get_if<std::string>(&attr.second)) { json[attr.first] = *v; @@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name) return *s; } -std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name) +std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if<int64_t>(&i->second)) + if (auto v = std::get_if<uint64_t>(&i->second)) return *v; throw Error("input attribute '%s' is not an integer", name); } -int64_t getIntAttr(const Attrs & attrs, const std::string & name) +uint64_t getIntAttr(const Attrs & attrs, const std::string & name) { auto s = maybeGetIntAttr(attrs, name); if (!s) @@ -76,8 +76,8 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if<int64_t>(&i->second)) - return *v; + if (auto v = std::get_if<Explicit<bool>>(&i->second)) + return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -93,7 +93,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs) { std::map<std::string, std::string> query; for (auto & attr : attrs) { - if (auto v = std::get_if<int64_t>(&attr.second)) { + if (auto v = std::get_if<uint64_t>(&attr.second)) { query.insert_or_assign(attr.first, fmt("%d", *v)); } else if (auto v = std::get_if<std::string>(&attr.second)) { query.insert_or_assign(attr.first, *v); diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index d6e0ae000..4b4630c80 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,9 +13,14 @@ namespace nix::fetchers { template<typename T> struct Explicit { T t; + + bool operator ==(const Explicit<T> & other) const + { + return t == other.t; + } }; -typedef std::variant<std::string, int64_t, Explicit<bool>> Attr; +typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr; typedef std::map<std::string, Attr> Attrs; Attrs jsonToAttrs(const nlohmann::json & json); @@ -26,9 +31,9 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin std::string getStrAttr(const Attrs & attrs, const std::string & name); -std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name); +std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name); -int64_t getIntAttr(const Attrs & attrs, const std::string & name); +uint64_t getIntAttr(const Attrs & attrs, const std::string & name); std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 83268b4bf..e4852d662 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -5,82 +5,234 @@ namespace nix::fetchers { -std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr; +std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr; -void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme) +void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>(); + if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>(); inputSchemes->push_back(std::move(inputScheme)); } -std::unique_ptr<Input> inputFromURL(const ParsedURL & url) +Input Input::fromURL(const std::string & url) +{ + return fromURL(parseURL(url)); +} + +static void fixupInput(Input & input) +{ + // Check common attributes. + input.getType(); + input.getRef(); + if (input.getRev()) + input.immutable = true; + input.getRevCount(); + input.getLastModified(); + if (input.getNarHash()) + input.immutable = true; +} + +Input Input::fromURL(const ParsedURL & url) { for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromURL(url); - if (res) return res; + if (res) { + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); + } } - throw Error("input '%s' is unsupported", url.url); -} -std::unique_ptr<Input> inputFromURL(const std::string & url) -{ - return inputFromURL(parseURL(url)); + throw Error("input '%s' is unsupported", url.url); } -std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) +Input Input::fromAttrs(Attrs && attrs) { - auto attrs2(attrs); - attrs2.erase("narHash"); for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs2); + auto res = inputScheme->inputFromAttrs(attrs); if (res) { - if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - // FIXME: require SRI hash. - res->narHash = Hash(*narHash); - return res; + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); } } - throw Error("input '%s' is unsupported", attrsToJson(attrs)); + + Input input; + input.attrs = attrs; + fixupInput(input); + return input; +} + +ParsedURL Input::toURL() const +{ + if (!scheme) + throw Error("cannot show unsupported input '%s'", attrsToJson(attrs)); + return scheme->toURL(*this); +} + +std::string Input::to_string() const +{ + return toURL().to_string(); } Attrs Input::toAttrs() const { - auto attrs = toAttrsInternal(); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - attrs.emplace("type", type()); return attrs; } -std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const +bool Input::hasAllInfo() const { - auto [tree, input] = fetchTreeInternal(store); + return getNarHash() && scheme && scheme->hasAllInfo(*this); +} + +bool Input::operator ==(const Input & other) const +{ + return attrs == other.attrs; +} + +bool Input::contains(const Input & other) const +{ + auto other2(other); + other2.attrs.erase("ref"); + other2.attrs.erase("rev"); + if (*this == other2) return true; + return false; +} + +std::pair<Tree, Input> Input::fetch(ref<Store> store) const +{ + if (!scheme) + throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs())); + + auto [tree, input] = scheme->fetch(store, *this); if (tree.actualPath == "") tree.actualPath = store->toRealPath(tree.storePath); - if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + auto narHash = store->queryPathInfo(tree.storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(SRI)); - if (input->narHash) - assert(input->narHash == tree.info.narHash); + if (auto narHash2 = getNarHash()) { + if (narHash != *narHash2) + throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, narHash2->to_string(SRI), narHash.to_string(SRI)); + } - if (narHash && narHash != input->narHash) - throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); + // FIXME: check lastModified, revCount + + input.immutable = true; + + assert(input.hasAllInfo()); return {std::move(tree), input}; } -std::shared_ptr<const Input> Input::applyOverrides( +Input Input::applyOverrides( std::optional<std::string> ref, std::optional<Hash> rev) const { + if (!scheme) return *this; + return scheme->applyOverrides(*this, ref, rev); +} + +void Input::clone(const Path & destDir) const +{ + assert(scheme); + scheme->clone(*this, destDir); +} + +std::optional<Path> Input::getSourcePath() const +{ + assert(scheme); + return scheme->getSourcePath(*this); +} + +void Input::markChangedFile( + std::string_view file, + std::optional<std::string> commitMsg) const +{ + assert(scheme); + return scheme->markChangedFile(*this, file, commitMsg); +} + +StorePath Input::computeStorePath(Store & store) const +{ + auto narHash = getNarHash(); + if (!narHash) + throw Error("cannot compute store path for mutable input '%s'", to_string()); + return store.makeFixedOutputPath(true, *narHash, "source"); +} + +std::string Input::getType() const +{ + return getStrAttr(attrs, "type"); +} + +std::optional<Hash> Input::getNarHash() const +{ + if (auto s = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + return Hash(*s, htSHA256); + return {}; +} + +std::optional<std::string> Input::getRef() const +{ + if (auto s = maybeGetStrAttr(attrs, "ref")) + return *s; + return {}; +} + +std::optional<Hash> Input::getRev() const +{ + if (auto s = maybeGetStrAttr(attrs, "rev")) + return Hash(*s, htSHA1); + return {}; +} + +std::optional<uint64_t> Input::getRevCount() const +{ + if (auto n = maybeGetIntAttr(attrs, "revCount")) + return *n; + return {}; +} + +std::optional<time_t> Input::getLastModified() const +{ + if (auto n = maybeGetIntAttr(attrs, "lastModified")) + return *n; + return {}; +} + +ParsedURL InputScheme::toURL(const Input & input) +{ + throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs)); +} + +Input InputScheme::applyOverrides( + const Input & input, + std::optional<std::string> ref, + std::optional<Hash> rev) +{ if (ref) - throw Error("don't know how to apply '%s' to '%s'", *ref, to_string()); + throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref); if (rev) - throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string()); - return shared_from_this(); + throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev()); + return input; +} + +std::optional<Path> InputScheme::getSourcePath(const Input & input) +{ + return {}; +} + +void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) +{ + assert(false); +} + +void InputScheme::clone(const Input & input, const Path & destDir) +{ + throw Error("do not know how to clone input '%s'", input.to_string()); } } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b75dcffa5..c43cfe50c 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "hash.hh" #include "path.hh" -#include "tree-info.hh" #include "attrs.hh" #include "url.hh" @@ -13,89 +12,100 @@ namespace nix { class Store; } namespace nix::fetchers { -struct Input; - struct Tree { Path actualPath; StorePath storePath; - TreeInfo info; }; -struct Input : std::enable_shared_from_this<Input> +struct InputScheme; + +struct Input { - std::optional<Hash> narHash; // FIXME: implement + friend class InputScheme; + + std::shared_ptr<InputScheme> scheme; // note: can be null + Attrs attrs; + bool immutable = false; + bool direct = true; + +public: + static Input fromURL(const std::string & url); + + static Input fromURL(const ParsedURL & url); - virtual std::string type() const = 0; + static Input fromAttrs(Attrs && attrs); - virtual ~Input() { } + ParsedURL toURL() const; - virtual bool operator ==(const Input & other) const { return false; } + std::string to_string() const; + + Attrs toAttrs() const; /* Check whether this is a "direct" input, that is, not one that goes through a registry. */ - virtual bool isDirect() const { return true; } + bool isDirect() const { return direct; } /* Check whether this is an "immutable" input, that is, one that contains a commit hash or content hash. */ - virtual bool isImmutable() const { return (bool) narHash; } - - virtual bool contains(const Input & other) const { return false; } - - virtual std::optional<std::string> getRef() const { return {}; } - - virtual std::optional<Hash> getRev() const { return {}; } + bool isImmutable() const { return immutable; } - virtual ParsedURL toURL() const = 0; + bool hasAllInfo() const; - std::string to_string() const - { - return toURL().to_string(); - } + bool operator ==(const Input & other) const; - Attrs toAttrs() const; + bool contains(const Input & other) const; - std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const; + std::pair<Tree, Input> fetch(ref<Store> store) const; - virtual std::shared_ptr<const Input> applyOverrides( + Input applyOverrides( std::optional<std::string> ref, std::optional<Hash> rev) const; - virtual std::optional<Path> getSourcePath() const { return {}; } - - virtual void markChangedFile( - std::string_view file, - std::optional<std::string> commitMsg) const - { assert(false); } + void clone(const Path & destDir) const; - virtual void clone(const Path & destDir) const - { - throw Error("do not know how to clone input '%s'", to_string()); - } + std::optional<Path> getSourcePath() const; -private: + void markChangedFile( + std::string_view file, + std::optional<std::string> commitMsg) const; - virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0; + StorePath computeStorePath(Store & store) const; - virtual Attrs toAttrsInternal() const = 0; + // Convience functions for common attributes. + std::string getType() const; + std::optional<Hash> getNarHash() const; + std::optional<std::string> getRef() const; + std::optional<Hash> getRev() const; + std::optional<uint64_t> getRevCount() const; + std::optional<time_t> getLastModified() const; }; struct InputScheme { - virtual ~InputScheme() { } + virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0; - virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0; + virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0; - virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0; -}; + virtual ParsedURL toURL(const Input & input); -std::unique_ptr<Input> inputFromURL(const ParsedURL & url); + virtual bool hasAllInfo(const Input & input) = 0; -std::unique_ptr<Input> inputFromURL(const std::string & url); + virtual Input applyOverrides( + const Input & input, + std::optional<std::string> ref, + std::optional<Hash> rev); + + virtual void clone(const Input & input, const Path & destDir); + + virtual std::optional<Path> getSourcePath(const Input & input); -std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs); + virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg); + + virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0; +}; -void registerInputScheme(std::unique_ptr<InputScheme> && fetcher); +void registerInputScheme(std::shared_ptr<InputScheme> && fetcher); struct DownloadFileResult { @@ -110,7 +120,7 @@ DownloadFileResult downloadFile( const std::string & name, bool immutable); -Tree downloadTarball( +std::pair<Tree, time_t> downloadTarball( ref<Store> store, const std::string & url, const std::string & name, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c9f9a4b23..4fcf3f542 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -22,110 +22,121 @@ static bool isNotDotGitDirectory(const Path & path) return not std::regex_match(path, gitDirRegex); } -struct GitInput : Input +struct GitInputScheme : InputScheme { - ParsedURL url; - std::optional<std::string> ref; - std::optional<Hash> rev; - bool shallow = false; - bool submodules = false; + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return {}; - GitInput(const ParsedURL & url) : url(url) - { } + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); - std::string type() const override { return "git"; } + Attrs attrs; + attrs.emplace("type", "git"); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const GitInput *>(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); } - bool isImmutable() const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != "git") return {}; - std::optional<std::string> getRef() const override { return ref; } + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash") + throw Error("unsupported Git input attribute '%s'", name); + + parseURL(getStrAttr(attrs, "url")); + maybeGetBoolAttr(attrs, "shallow"); + maybeGetBoolAttr(attrs, "submodules"); + + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + } - std::optional<Hash> getRev() const override { return rev; } + Input input; + input.attrs = attrs; + return input; + } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - ParsedURL url2(url); - if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); - if (shallow) url2.query.insert_or_assign("shallow", "1"); - return url2; + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme != "git") url.scheme = "git+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); + if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + url.query.insert_or_assign("shallow", "1"); + return url; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (shallow) - attrs.emplace("shallow", true); - if (submodules) - attrs.emplace("submodules", true); - return attrs; + bool maybeDirty = !input.getRef(); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + return + maybeGetIntAttr(input.attrs, "lastModified") + && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); } - void clone(const Path & destDir) const override + Input applyOverrides( + const Input & input, + std::optional<std::string> ref, + std::optional<Hash> rev) override + { + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); + if (!res.getRef() && res.getRev()) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string()); + return res; + } + + void clone(const Input & input, const Path & destDir) override { - auto [isLocal, actualUrl] = getActualUrl(); + auto [isLocal, actualUrl] = getActualUrl(input); Strings args = {"clone"}; args.push_back(actualUrl); - if (ref) { + if (auto ref = input.getRef()) { args.push_back("--branch"); args.push_back(*ref); } - if (rev) throw Error("cloning a specific revision is not implemented"); + if (input.getRev()) throw Error("cloning a specific revision is not implemented"); args.push_back(destDir); runProgram("git", true, args); } - std::shared_ptr<const Input> applyOverrides( - std::optional<std::string> ref, - std::optional<Hash> rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared<GitInput>(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - if (!res->ref && res->rev) - throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); - - return res; - } - - std::optional<Path> getSourcePath() const override + std::optional<Path> getSourcePath(const Input & input) override { - if (url.scheme == "file" && !ref && !rev) + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) return url.path; return {}; } - void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override { - auto sourcePath = getSourcePath(); + auto sourcePath = getSourcePath(input); assert(sourcePath); runProgram("git", true, @@ -136,23 +147,25 @@ struct GitInput : Input { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); } - std::pair<bool, std::string> getActualUrl() const + std::pair<bool, std::string> getActualUrl(const Input & input) const { // Don't clone file:// URIs (but otherwise treat them the // same as remote URIs, i.e. don't use the working tree or // HEAD). static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file" && !forceHttp; return {isLocal, isLocal ? url.path : url.base}; } - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared<GitInput>(*this); + Input input(_input); - assert(!rev || rev->type == htSHA1); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -163,39 +176,38 @@ struct GitInput : Input return Attrs({ {"type", cacheType}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair<Tree, std::shared_ptr<const Input>> + -> std::pair<Tree, Input> { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + if (!shallow) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return { Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), - .lastModified = getIntAttr(infoAttrs, "lastModified"), - }, }, input }; }; - if (rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. - if (!input->ref && !input->rev && isLocal) { + if (!input.getRef() && !input.getRev() && isLocal) { bool clean = false; /* Check whether this repo has any commits. There are @@ -254,35 +266,37 @@ struct GitInput : Input auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); - auto tree = Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, - } + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + input.attrs.insert_or_assign( + "lastModified", + haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0); + + return { + Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input }; - - return {std::move(tree), input}; } } - if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master"); Attrs mutableAttrs({ {"type", cacheType}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); Path repoDir; if (isLocal) { - if (!input->rev) - input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", + Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; @@ -290,8 +304,8 @@ struct GitInput : Input if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -305,18 +319,18 @@ struct GitInput : Input } Path localRefFile = - input->ref->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input->ref - : cacheDir + "/refs/heads/" + *input->ref; + input.getRef()->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input.getRef() + : cacheDir + "/refs/heads/" + *input.getRef(); bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input->rev) { + if (input.getRev()) { try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -339,7 +353,7 @@ struct GitInput : Input // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. try { - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input.getRef(), *input.getRef()) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); @@ -354,8 +368,8 @@ struct GitInput : Input utimes(localRefFile.c_str(), times); } - if (!input->rev) - input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev()); } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; @@ -365,7 +379,7 @@ struct GitInput : Input // FIXME: check whether rev is an ancestor of ref. - printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); /* Now that we know the ref, check again whether we have it in the store. */ @@ -387,7 +401,7 @@ struct GitInput : Input runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); @@ -396,7 +410,7 @@ struct GitInput : Input // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() }); gitOptions.standardOut = &sink; runProgram2(gitOptions); }); @@ -406,18 +420,18 @@ struct GitInput : Input auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() })); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"lastModified", lastModified}, }); if (!shallow) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -436,60 +450,6 @@ struct GitInput : Input } }; -struct GitInputScheme : InputScheme -{ - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "git" && - url.scheme != "git+http" && - url.scheme != "git+https" && - url.scheme != "git+ssh" && - url.scheme != "git+file") return nullptr; - - auto url2(url); - if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "git"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules") - throw Error("unsupported Git input attribute '%s'", name); - - auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Git branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - - input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); - - input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false); - - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); }); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index e59c83be4..8d113967e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,37 +8,80 @@ namespace nix::fetchers { -struct GitArchiveInput : Input +struct GitArchiveInputScheme : InputScheme { - std::string owner; - std::string repo; - std::optional<std::string> ref; - std::optional<Hash> rev; - - virtual std::shared_ptr<GitArchiveInput> _clone() const = 0; + virtual std::string type() = 0; - bool operator ==(const Input & other) const override + std::optional<Input> inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast<const GitArchiveInput *>(&other); - return - other2 - && owner == other2->owner - && repo == other2->repo - && rev == other2->rev - && ref == other2->ref; + if (url.scheme != type()) return {}; + + auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); + + std::optional<Hash> rev; + std::optional<std::string> ref; + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + ref = path[2]; + else + throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (rev) + throw BadURL("URL '%s' contains multiple commit hashes", url.url); + rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); + if (ref) + throw BadURL("URL '%s' contains multiple branch/tag names", url.url); + ref = value; + } + } + + if (ref && rev) + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url); + + Input input; + input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("owner", path[0]); + input.attrs.insert_or_assign("repo", path[1]); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + + return input; } - bool isImmutable() const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != type()) return {}; - std::optional<std::string> getRef() const override { return ref; } + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified") + throw Error("unsupported input attribute '%s'", name); - std::optional<Hash> getRev() const override { return rev; } + getStrAttr(attrs, "owner"); + getStrAttr(attrs, "repo"); + + Input input; + input.attrs = attrs; + return input; + } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { + auto owner = getStrAttr(input.attrs, "owner"); + auto repo = getStrAttr(input.attrs, "repo"); + auto ref = input.getRef(); + auto rev = input.getRev(); auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; @@ -49,32 +92,44 @@ struct GitArchiveInput : Input }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("owner", owner); - attrs.emplace("repo", repo); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); + } + + Input applyOverrides( + const Input & _input, + std::optional<std::string> ref, + std::optional<Hash> rev) override + { + auto input(_input); + if (rev) { + input.attrs.insert_or_assign("rev", rev->gitRev()); + input.attrs.erase("ref"); + } + if (ref) { + if (input.getRev()) + throw BadURL("input '%s' contains both a commit hash and a branch/tag name", input.to_string()); + input.attrs.insert_or_assign("ref", *ref); + } + return input; } - virtual Hash getRevFromRef(nix::ref<Store> store, std::string_view ref) const = 0; + virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0; - virtual std::string getDownloadUrl() const = 0; + virtual std::string getDownloadUrl(const Input & input) const = 0; - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override { - auto rev = this->rev; - auto ref = this->ref.value_or("master"); + Input input(_input); - if (!rev) rev = getRevFromRef(store, ref); + if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "master"); - auto input = _clone(); - input->ref = {}; - input->rev = *rev; + auto rev = input.getRev(); + if (!rev) rev = getRevFromRef(store, input); + + input.attrs.erase("ref"); + input.attrs.insert_or_assign("rev", rev->gitRev()); Attrs immutableAttrs({ {"type", "git-tarball"}, @@ -82,131 +137,44 @@ struct GitArchiveInput : Input }); if (auto res = getCache()->lookup(store, immutableAttrs)) { + input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); return { Tree{ .actualPath = store->toRealPath(res->second), .storePath = std::move(res->second), - .info = TreeInfo { - .lastModified = getIntAttr(res->first, "lastModified"), - }, }, input }; } - auto url = input->getDownloadUrl(); + auto url = getDownloadUrl(input); + auto [tree, lastModified] = downloadTarball(store, url, "source", true); - auto tree = downloadTarball(store, url, "source", true); + input.attrs.insert_or_assign("lastModified", lastModified); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", *tree.info.lastModified} + {"lastModified", lastModified} }, tree.storePath, true); return {std::move(tree), input}; } - - std::shared_ptr<const Input> applyOverrides( - std::optional<std::string> ref, - std::optional<Hash> rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = _clone(); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } }; -struct GitArchiveInputScheme : InputScheme -{ - std::string type; - - GitArchiveInputScheme(std::string && type) : type(type) - { } - - virtual std::unique_ptr<GitArchiveInput> create() = 0; - - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != type) return nullptr; - - auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); - auto input = create(); - - if (path.size() == 2) { - } else if (path.size() == 3) { - if (std::regex_match(path[2], revRegex)) - input->rev = Hash(path[2], htSHA1); - else if (std::regex_match(path[2], refRegex)) - input->ref = path[2]; - else - throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else - throw BadURL("URL '%s' is invalid", url.url); - - for (auto &[name, value] : url.query) { - if (name == "rev") { - if (input->rev) - throw BadURL("URL '%s' contains multiple commit hashes", url.url); - input->rev = Hash(value, htSHA1); - } - else if (name == "ref") { - if (!std::regex_match(value, refRegex)) - throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); - if (input->ref) - throw BadURL("URL '%s' contains multiple branch/tag names", url.url); - input->ref = value; - } - } - - if (input->ref && input->rev) - throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url); - - input->owner = path[0]; - input->repo = path[1]; - - return input; - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != type) return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") - throw Error("unsupported input attribute '%s'", name); - - auto input = create(); - input->owner = getStrAttr(attrs, "owner"); - input->repo = getStrAttr(attrs, "repo"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - -struct GitHubInput : GitArchiveInput +struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() const override { return "github"; } + std::string type() override { return "github"; } - std::shared_ptr<GitArchiveInput> _clone() const override - { return std::make_shared<GitHubInput>(*this); } - - Hash getRevFromRef(nix::ref<Store> store, std::string_view ref) const override + Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override { auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - owner, repo, ref); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( @@ -216,13 +184,14 @@ struct GitHubInput : GitArchiveInput return rev; } - std::string getDownloadUrl() const override + std::string getDownloadUrl(const Input & input) const override { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - owner, repo, rev->to_string(Base16, false)); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); std::string accessToken = settings.githubAccessToken.get(); if (accessToken != "") @@ -231,35 +200,23 @@ struct GitHubInput : GitArchiveInput return url; } - void clone(const Path & destDir) const override - { - std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); - } -}; - -struct GitHubInputScheme : GitArchiveInputScheme -{ - GitHubInputScheme() : GitArchiveInputScheme("github") { } - - std::unique_ptr<GitArchiveInput> create() override + void clone(const Input & input, const Path & destDir) override { - return std::make_unique<GitHubInput>(); + Input::fromURL(fmt("git+ssh://git@github.com/%s/%s.git", + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("master"), input.getRev()) + .clone(destDir); } }; -struct GitLabInput : GitArchiveInput +struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() const override { return "gitlab"; } - - std::shared_ptr<GitArchiveInput> _clone() const override - { return std::make_shared<GitLabInput>(*this); } + std::string type() override { return "gitlab"; } - Hash getRevFromRef(nix::ref<Store> store, std::string_view ref) const override + Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override { auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/branches/%s", - owner, repo, ref); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( @@ -269,12 +226,13 @@ struct GitLabInput : GitArchiveInput return rev; } - std::string getDownloadUrl() const override + std::string getDownloadUrl(const Input & input) const override { // FIXME: This endpoint has a rate limit threshold of 5 requests per minute. auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", - owner, repo, rev->to_string(Base16, false)); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`) std::string accessToken = settings.githubAccessToken.get(); @@ -284,21 +242,12 @@ struct GitLabInput : GitArchiveInput return url; } - void clone(const Path & destDir) const override - { - std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); - } -}; - -struct GitLabInputScheme : GitArchiveInputScheme -{ - GitLabInputScheme() : GitArchiveInputScheme("gitlab") { } - - std::unique_ptr<GitArchiveInput> create() override + void clone(const Input & input, const Path & destDir) override { - return std::make_unique<GitLabInput>(); + Input::fromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("master"), input.getRev()) + .clone(destDir); } }; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 380b69fe0..91dc83740 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -4,135 +4,99 @@ namespace nix::fetchers { std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); -struct IndirectInput : Input -{ - std::string id; - std::optional<Hash> rev; - std::optional<std::string> ref; - - std::string type() const override { return "indirect"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const IndirectInput *>(&other); - return - other2 - && id == other2->id - && rev == other2->rev - && ref == other2->ref; - } - - bool isDirect() const override - { - return false; - } - - std::optional<std::string> getRef() const override { return ref; } - - std::optional<Hash> getRev() const override { return rev; } - - bool contains(const Input & other) const override - { - auto other2 = dynamic_cast<const IndirectInput *>(&other); - return - other2 - && id == other2->id - && (!ref || ref == other2->ref) - && (!rev || rev == other2->rev); - } - - ParsedURL toURL() const override - { - ParsedURL url; - url.scheme = "flake"; - url.path = id; - if (ref) { url.path += '/'; url.path += *ref; }; - if (rev) { url.path += '/'; url.path += rev->gitRev(); }; - return url; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("id", id); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; - } - - std::shared_ptr<const Input> applyOverrides( - std::optional<std::string> ref, - std::optional<Hash> rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared<IndirectInput>(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } - - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override - { - throw Error("indirect input '%s' cannot be fetched directly", to_string()); - } -}; - struct IndirectInputScheme : InputScheme { - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) override { - if (url.scheme != "flake") return nullptr; + if (url.scheme != "flake") return {}; auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); - auto input = std::make_unique<IndirectInput>(); + + std::optional<Hash> rev; + std::optional<std::string> ref; if (path.size() == 1) { } else if (path.size() == 2) { if (std::regex_match(path[1], revRegex)) - input->rev = Hash(path[1], htSHA1); + rev = Hash(path[1], htSHA1); else if (std::regex_match(path[1], refRegex)) - input->ref = path[1]; + ref = path[1]; else throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); } else if (path.size() == 3) { if (!std::regex_match(path[1], refRegex)) throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); - input->ref = path[1]; + ref = path[1]; if (!std::regex_match(path[2], revRegex)) throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); - input->rev = Hash(path[2], htSHA1); + rev = Hash(path[2], htSHA1); } else throw BadURL("GitHub URL '%s' is invalid", url.url); + std::string id = path[0]; + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + // FIXME: forbid query params? - input->id = path[0]; - if (!std::regex_match(input->id, flakeRegex)) - throw BadURL("'%s' is not a valid flake ID", input->id); + Input input; + input.direct = false; + input.attrs.insert_or_assign("type", "indirect"); + input.attrs.insert_or_assign("id", id); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); return input; } - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev") + if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") throw Error("unsupported indirect input attribute '%s'", name); - auto input = std::make_unique<IndirectInput>(); - input->id = getStrAttr(attrs, "id"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); + auto id = getStrAttr(attrs, "id"); + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + + Input input; + input.direct = false; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override + { + ParsedURL url; + url.scheme = "flake"; + url.path = getStrAttr(input.attrs, "id"); + if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; }; + if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); }; + return url; + } + + bool hasAllInfo(const Input & input) override + { + return false; + } + + Input applyOverrides( + const Input & _input, + std::optional<std::string> ref, + std::optional<Hash> rev) override + { + auto input(_input); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); return input; } + + std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override + { + throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); }); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 5abb00172..49ed63243 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -10,80 +10,92 @@ using namespace std::string_literals; namespace nix::fetchers { -struct MercurialInput : Input +struct MercurialInputScheme : InputScheme { - ParsedURL url; - std::optional<std::string> ref; - std::optional<Hash> rev; + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return {}; + + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); - MercurialInput(const ParsedURL & url) : url(url) - { } + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } - std::string type() const override { return "hg"; } + attrs.emplace("url", url2.to_string()); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const MercurialInput *>(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + return inputFromAttrs(attrs); } - bool isImmutable() const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash") + throw Error("unsupported Mercurial input attribute '%s'", name); - std::optional<std::string> getRef() const override { return ref; } + parseURL(getStrAttr(attrs, "url")); - std::optional<Hash> getRev() const override { return rev; } + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + } - ParsedURL toURL() const override + Input input; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override { - ParsedURL url2(url); - url2.scheme = "hg+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); + auto url = parseURL(getStrAttr(input.attrs, "url")); + url.scheme = "hg+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); return url; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + // FIXME: ugly, need to distinguish between dirty and clean + // default trees. + return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); } - std::shared_ptr<const Input> applyOverrides( + Input applyOverrides( + const Input & input, std::optional<std::string> ref, - std::optional<Hash> rev) const override + std::optional<Hash> rev) override { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared<MercurialInput>(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); return res; } - std::optional<Path> getSourcePath() const + std::optional<Path> getSourcePath(const Input & input) override { - if (url.scheme == "file" && !ref && !rev) + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) return url.path; return {}; } - void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override { - auto sourcePath = getSourcePath(); + auto sourcePath = getSourcePath(input); assert(sourcePath); // FIXME: shut up if file is already tracked. @@ -95,26 +107,27 @@ struct MercurialInput : Input { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); } - std::pair<bool, std::string> getActualUrl() const + std::pair<bool, std::string> getActualUrl(const Input & input) const { + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file"; return {isLocal, isLocal ? url.path : url.base}; } - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared<MercurialInput>(*this); + Input input(_input); - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // FIXME: return lastModified. // FIXME: don't clone local repositories. - if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) { bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; @@ -129,7 +142,7 @@ struct MercurialInput : Input if (settings.warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); - input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl }))); auto files = tokenizeString<std::set<std::string>>( runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); @@ -158,53 +171,50 @@ struct MercurialInput : Input } } - if (!input->ref) input->ref = "default"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); auto getImmutableAttrs = [&]() { return Attrs({ {"type", "hg"}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair<Tree, std::shared_ptr<const Input>> + -> std::pair<Tree, Input> { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); return { Tree{ .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = getIntAttr(infoAttrs, "revCount"), - }, }, input }; }; - if (input->rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - assert(input->rev || input->ref); - auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef(); Attrs mutableAttrs({ {"type", "hg"}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -213,10 +223,10 @@ struct MercurialInput : Input /* If this is a commit hash that we already have, we don't have to pull again. */ - if (!(input->rev + if (!(input.getRev() && pathExists(cacheDir) && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) .killStderr(true)).second == "1")) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); @@ -245,9 +255,9 @@ struct MercurialInput : Input runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input->rev = Hash(tokens[0], htSHA1); + input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev()); auto revCount = std::stoull(tokens[1]); - input->ref = tokens[2]; + input.attrs.insert_or_assign("ref", tokens[2]); if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); @@ -255,18 +265,18 @@ struct MercurialInput : Input Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); deletePath(tmpDir + "/.hg_archival.txt"); auto storePath = store->addToStore(name, tmpDir); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"revCount", (int64_t) revCount}, }); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -285,54 +295,6 @@ struct MercurialInput : Input } }; -struct MercurialInputScheme : InputScheme -{ - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "hg+http" && - url.scheme != "hg+https" && - url.scheme != "hg+ssh" && - url.scheme != "hg+file") return nullptr; - - auto url2(url); - url2.scheme = std::string(url2.scheme, 3); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "hg"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev") - throw Error("unsupported Mercurial input attribute '%s'", name); - - auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); }); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 77fe87d59..cbbb0fa02 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -3,76 +3,86 @@ namespace nix::fetchers { -struct PathInput : Input +struct PathInputScheme : InputScheme { - Path path; + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "path") return {}; - /* Allow the user to pass in "fake" tree info attributes. This is - useful for making a pinned tree work the same as the repository - from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - std::optional<Hash> rev; - std::optional<uint64_t> revCount; - std::optional<time_t> lastModified; + if (url.authority && *url.authority != "") + throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - std::string type() const override { return "path"; } + Input input; + input.attrs.insert_or_assign("type", "path"); + input.attrs.insert_or_assign("path", url.path); - std::optional<Hash> getRev() const override { return rev; } + for (auto & [name, value] : url.query) + if (name == "rev" || name == "narHash") + input.attrs.insert_or_assign(name, value); + else if (name == "revCount" || name == "lastModified") { + uint64_t n; + if (!string2Int(value, n)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input.attrs.insert_or_assign(name, n); + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const PathInput *>(&other); - return - other2 - && path == other2->path - && rev == other2->rev - && revCount == other2->revCount - && lastModified == other2->lastModified; + return input; } - bool isImmutable() const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - return narHash || rev; + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + /* Allow the user to pass in "fake" tree info + attributes. This is useful for making a pinned tree + work the same as the repository from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") + // checked in Input::fromAttrs + ; + else + throw Error("unsupported path input attribute '%s'", name); + + Input input; + input.attrs = attrs; + return input; } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - auto query = attrsToQuery(toAttrsInternal()); + auto query = attrsToQuery(input.attrs); query.erase("path"); + query.erase("type"); return ParsedURL { .scheme = "path", - .path = path, + .path = getStrAttr(input.attrs, "path"), .query = query, }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("path", path); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (revCount) - attrs.emplace("revCount", *revCount); - if (lastModified) - attrs.emplace("lastModified", *lastModified); - if (!rev && narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - return attrs; + return true; } - std::optional<Path> getSourcePath() const override + std::optional<Path> getSourcePath(const Input & input) override { - return path; + return getStrAttr(input.attrs, "path"); } - void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override { + // nothing to do } - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override { - auto input = std::make_shared<PathInput>(*this); + auto path = getStrAttr(input.attrs, "path"); // FIXME: check whether access to 'path' is allowed. @@ -85,83 +95,13 @@ struct PathInput : Input // FIXME: try to substitute storePath. storePath = store->addToStore("source", path); - input->narHash = store->queryPathInfo(*storePath)->narHash; - - return - { - Tree { - .actualPath = store->toRealPath(*storePath), - .storePath = std::move(*storePath), - .info = TreeInfo { - .revCount = revCount, - .lastModified = lastModified - } - }, - input - }; - } - -}; - -struct PathInputScheme : InputScheme -{ - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "path") return nullptr; - - auto input = std::make_unique<PathInput>(); - input->path = url.path; - - if (url.authority && *url.authority != "") - throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - - for (auto & [name, value] : url.query) - if (name == "rev") - input->rev = Hash(value, htSHA1); - else if (name == "revCount") { - uint64_t revCount; - if (!string2Int(value, revCount)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->revCount = revCount; - } - else if (name == "lastModified") { - time_t lastModified; - if (!string2Int(value, lastModified)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->lastModified = lastModified; - } - else if (name == "narHash") - // FIXME: require SRI hash. - input->narHash = Hash(value); - else - throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); - - return input; - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - - auto input = std::make_unique<PathInput>(); - input->path = getStrAttr(attrs, "path"); - - for (auto & [name, value] : attrs) - if (name == "rev") - input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); - else if (name == "revCount") - input->revCount = getIntAttr(attrs, "revCount"); - else if (name == "lastModified") - input->lastModified = getIntAttr(attrs, "lastModified"); - else if (name == "narHash") - // FIXME: require SRI hash. - input->narHash = Hash(getStrAttr(attrs, "narHash")); - else if (name == "type" || name == "path") - ; - else - throw Error("unsupported path input attribute '%s'", name); - - return input; + return { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + }, + input + }; } }; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index f6760d2d0..914a0e1e8 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -22,20 +22,7 @@ std::shared_ptr<Registry> Registry::read( auto version = json.value("version", 0); - // FIXME: remove soon - if (version == 1) { - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - auto url = i->value("url", i->value("uri", "")); - if (url.empty()) - throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", - path, i.key()); - registry->entries.push_back( - {inputFromURL(i.key()), inputFromURL(url), {}}); - } - } - - else if (version == 2) { + if (version == 2) { for (auto & i : json["flakes"]) { auto toAttrs = jsonToAttrs(i["to"]); Attrs extraAttrs; @@ -47,8 +34,8 @@ std::shared_ptr<Registry> Registry::read( auto exact = i.find("exact"); registry->entries.push_back( Entry { - .from = inputFromAttrs(jsonToAttrs(i["from"])), - .to = inputFromAttrs(toAttrs), + .from = Input::fromAttrs(jsonToAttrs(i["from"])), + .to = Input::fromAttrs(std::move(toAttrs)), .extraAttrs = extraAttrs, .exact = exact != i.end() && exact.value() }); @@ -72,8 +59,8 @@ void Registry::write(const Path & path) nlohmann::json arr; for (auto & entry : entries) { nlohmann::json obj; - obj["from"] = attrsToJson(entry.from->toAttrs()); - obj["to"] = attrsToJson(entry.to->toAttrs()); + obj["from"] = attrsToJson(entry.from.toAttrs()); + obj["to"] = attrsToJson(entry.to.toAttrs()); if (!entry.extraAttrs.empty()) obj["to"].update(attrsToJson(entry.extraAttrs)); if (entry.exact) @@ -90,8 +77,8 @@ void Registry::write(const Path & path) } void Registry::add( - const std::shared_ptr<const Input> & from, - const std::shared_ptr<const Input> & to, + const Input & from, + const Input & to, const Attrs & extraAttrs) { entries.emplace_back( @@ -102,11 +89,11 @@ void Registry::add( }); } -void Registry::remove(const std::shared_ptr<const Input> & input) +void Registry::remove(const Input & input) { // FIXME: use C++20 std::erase. for (auto i = entries.begin(); i != entries.end(); ) - if (*i->from == *input) + if (i->from == input) i = entries.erase(i); else ++i; @@ -145,8 +132,8 @@ std::shared_ptr<Registry> getFlagRegistry() } void overrideRegistry( - const std::shared_ptr<const Input> & from, - const std::shared_ptr<const Input> & to, + const Input & from, + const Input & to, const Attrs & extraAttrs) { flagRegistry->add(from, to, extraAttrs); @@ -180,32 +167,33 @@ Registries getRegistries(ref<Store> store) return registries; } -std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries( +std::pair<Input, Attrs> lookupInRegistries( ref<Store> store, - std::shared_ptr<const Input> input) + const Input & _input) { Attrs extraAttrs; int n = 0; + Input input(_input); restart: n++; - if (n > 100) throw Error("cycle detected in flake registr for '%s'", input); + if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); for (auto & registry : getRegistries(store)) { // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { - if (*entry.from == *input) { + if (entry.from == input) { input = entry.to; extraAttrs = entry.extraAttrs; goto restart; } } else { - if (entry.from->contains(*input)) { - input = entry.to->applyOverrides( - !entry.from->getRef() && input->getRef() ? input->getRef() : std::optional<std::string>(), - !entry.from->getRev() && input->getRev() ? input->getRev() : std::optional<Hash>()); + if (entry.from.contains(input)) { + input = entry.to.applyOverrides( + !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(), + !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>()); extraAttrs = entry.extraAttrs; goto restart; } @@ -213,8 +201,8 @@ std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries( } } - if (!input->isDirect()) - throw Error("cannot find flake '%s' in the flake registries", input->to_string()); + if (!input.isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input.to_string()); return {input, extraAttrs}; } diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index c3ce948a8..1077af020 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -20,8 +20,7 @@ struct Registry struct Entry { - std::shared_ptr<const Input> from; - std::shared_ptr<const Input> to; + Input from, to; Attrs extraAttrs; bool exact = false; }; @@ -38,11 +37,11 @@ struct Registry void write(const Path & path); void add( - const std::shared_ptr<const Input> & from, - const std::shared_ptr<const Input> & to, + const Input & from, + const Input & to, const Attrs & extraAttrs); - void remove(const std::shared_ptr<const Input> & input); + void remove(const Input & input); }; typedef std::vector<std::shared_ptr<Registry>> Registries; @@ -54,12 +53,12 @@ Path getUserRegistryPath(); Registries getRegistries(ref<Store> store); void overrideRegistry( - const std::shared_ptr<const Input> & from, - const std::shared_ptr<const Input> & to, + const Input & from, + const Input & to, const Attrs & extraAttrs); -std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries( +std::pair<Input, Attrs> lookupInRegistries( ref<Store> store, - std::shared_ptr<const Input> input); + const Input & input); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 695525b31..624f8b3fb 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -101,7 +101,7 @@ DownloadFileResult downloadFile( }; } -Tree downloadTarball( +std::pair<Tree, time_t> downloadTarball( ref<Store> store, const std::string & url, const std::string & name, @@ -116,12 +116,12 @@ Tree downloadTarball( auto cached = getCache()->lookupExpired(store, inAttrs); if (cached && !cached->expired) - return Tree { - .actualPath = store->toRealPath(cached->storePath), - .storePath = std::move(cached->storePath), - .info = TreeInfo { - .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), + return { + Tree { + .actualPath = store->toRealPath(cached->storePath), + .storePath = std::move(cached->storePath), }, + getIntAttr(cached->infoAttrs, "lastModified") }; auto res = downloadFile(store, url, name, immutable); @@ -156,118 +156,75 @@ Tree downloadTarball( *unpackedStorePath, immutable); - return Tree { - .actualPath = store->toRealPath(*unpackedStorePath), - .storePath = std::move(*unpackedStorePath), - .info = TreeInfo { - .lastModified = lastModified, + return { + Tree { + .actualPath = store->toRealPath(*unpackedStorePath), + .storePath = std::move(*unpackedStorePath), }, + lastModified, }; } -struct TarballInput : Input -{ - ParsedURL url; - std::optional<Hash> hash; - - TarballInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "tarball"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const TarballInput *>(&other); - return - other2 - && to_string() == other2->to_string() - && hash == other2->hash; - } - - bool isImmutable() const override - { - return hash || narHash; - } - - ParsedURL toURL() const override - { - auto url2(url); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. - if (narHash) - url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); - else if (hash) - url2.query.insert_or_assign("hash", hash->to_string(SRI)); - return url2; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (hash) - attrs.emplace("hash", hash->to_string(SRI)); - return attrs; - } - - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override - { - auto tree = downloadTarball(store, url.to_string(), "source", false); - - auto input = std::make_shared<TarballInput>(*this); - input->narHash = store->queryPathInfo(tree.storePath)->narHash; - - return {std::move(tree), input}; - } -}; - struct TarballInputScheme : InputScheme { - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) override { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {}; if (!hasSuffix(url.path, ".zip") && !hasSuffix(url.path, ".tar") && !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.xz") && !hasSuffix(url.path, ".tar.bz2")) - return nullptr; - - auto input = std::make_unique<TarballInput>(url); - - auto hash = input->url.query.find("hash"); - if (hash != input->url.query.end()) { - // FIXME: require SRI hash. - input->hash = Hash(hash->second); - input->url.query.erase(hash); - } - - auto narHash = input->url.query.find("narHash"); - if (narHash != input->url.query.end()) { - // FIXME: require SRI hash. - input->narHash = Hash(narHash->second); - input->url.query.erase(narHash); - } - + return {}; + + Input input; + input.attrs.insert_or_assign("type", "tarball"); + input.attrs.insert_or_assign("url", url.to_string()); + auto narHash = url.query.find("narHash"); + if (narHash != url.query.end()) + input.attrs.insert_or_assign("narHash", narHash->second); return input; } - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "hash") + if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash") throw Error("unsupported tarball input attribute '%s'", name); - auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url"))); - if (auto hash = maybeGetStrAttr(attrs, "hash")) - // FIXME: require SRI hash. - input->hash = Hash(*hash); - + Input input; + input.attrs = attrs; + //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash"); return input; } + + ParsedURL toURL(const Input & input) override + { + auto url = parseURL(getStrAttr(input.attrs, "url")); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (auto narHash = input.getNarHash()) + url.query.insert_or_assign("narHash", narHash->to_string(SRI)); + /* + else if (auto hash = maybeGetStrAttr(input.attrs, "hash")) + url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI)); + */ + return url; + } + + bool hasAllInfo(const Input & input) override + { + return true; + } + + std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override + { + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + return {std::move(tree), input}; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); }); diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc deleted file mode 100644 index 42a19cbc8..000000000 --- a/src/libfetchers/tree-info.cc +++ /dev/null @@ -1,60 +0,0 @@ -#include "tree-info.hh" -#include "store-api.hh" - -#include <nlohmann/json.hpp> - -namespace nix::fetchers { - -StorePath TreeInfo::computeStorePath(Store & store) const -{ - assert(narHash); - return store.makeFixedOutputPath(true, narHash, "source"); -} - -TreeInfo TreeInfo::fromJson(const nlohmann::json & json) -{ - TreeInfo info; - - auto i = json.find("info"); - if (i != json.end()) { - const nlohmann::json & i2(*i); - - auto j = i2.find("narHash"); - if (j != i2.end()) - info.narHash = Hash((std::string) *j); - else - throw Error("attribute 'narHash' missing in lock file"); - - j = i2.find("revCount"); - if (j != i2.end()) - info.revCount = *j; - - j = i2.find("lastModified"); - if (j != i2.end()) - info.lastModified = *j; - - return info; - } - - i = json.find("narHash"); - if (i != json.end()) { - info.narHash = Hash((std::string) *i); - return info; - } - - throw Error("attribute 'info' missing in lock file"); -} - -nlohmann::json TreeInfo::toJson() const -{ - nlohmann::json json; - assert(narHash); - json["narHash"] = narHash.to_string(SRI); - if (revCount) - json["revCount"] = *revCount; - if (lastModified) - json["lastModified"] = *lastModified; - return json; -} - -} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh deleted file mode 100644 index 3b62151c6..000000000 --- a/src/libfetchers/tree-info.hh +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "path.hh" -#include "hash.hh" - -#include <nlohmann/json_fwd.hpp> - -namespace nix { class Store; } - -namespace nix::fetchers { - -struct TreeInfo -{ - Hash narHash; - std::optional<uint64_t> revCount; - std::optional<time_t> lastModified; - - bool operator ==(const TreeInfo & other) const - { - return - narHash == other.narHash - && revCount == other.revCount - && lastModified == other.lastModified; - } - - StorePath computeStorePath(Store & store) const; - - static TreeInfo fromJson(const nlohmann::json & json); - - nlohmann::json toJson() const; -}; - -} diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 59bb40110..15a3e261a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -62,13 +62,13 @@ static void printFlakeInfo(const Store & store, const Flake & flake) if (flake.description) logger->stdout("Description: %s", *flake.description); logger->stdout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); - if (auto rev = flake.lockedRef.input->getRev()) + if (auto rev = flake.lockedRef.input.getRev()) logger->stdout("Revision: %s", rev->to_string(Base16, false)); - if (flake.sourceInfo->info.revCount) - logger->stdout("Revisions: %s", *flake.sourceInfo->info.revCount); - if (flake.sourceInfo->info.lastModified) + if (auto revCount = flake.lockedRef.input.getRevCount()) + logger->stdout("Revisions: %s", *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) logger->stdout("Last modified: %s", - std::put_time(std::localtime(&*flake.sourceInfo->info.lastModified), "%F %T")); + std::put_time(std::localtime(&*lastModified), "%F %T")); } static nlohmann::json flakeToJson(const Store & store, const Flake & flake) @@ -82,13 +82,12 @@ static nlohmann::json flakeToJson(const Store & store, const Flake & flake) j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs()); j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl j["locked"] = attrsToJson(flake.lockedRef.toAttrs()); - j["info"] = flake.sourceInfo->info.toJson(); - if (auto rev = flake.lockedRef.input->getRev()) + if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(Base16, false); - if (flake.sourceInfo->info.revCount) - j["revCount"] = *flake.sourceInfo->info.revCount; - if (flake.sourceInfo->info.lastModified) - j["lastModified"] = *flake.sourceInfo->info.lastModified; + if (auto revCount = flake.lockedRef.input.getRevCount()) + j["revCount"] = *revCount; + if (auto lastModified = flake.lockedRef.input.getLastModified()) + j["lastModified"] = *lastModified; j["path"] = store.printStorePath(flake.sourceInfo->storePath); return j; } @@ -172,7 +171,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, lockedNode ? lockedNode->lockedRef : flake.flake.lockedRef); - + if (firstVisit) recurse(*input.second, prefix + (last ? treeNull : treeLine)); } }; @@ -501,7 +500,7 @@ struct CmdFlakeClone : FlakeCommand if (destDir.empty()) throw Error("missing flag '--dest'"); - getFlakeRef().resolve(store).input->clone(destDir); + getFlakeRef().resolve(store).input.clone(destDir); } }; @@ -563,9 +562,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun auto lockedInput = std::dynamic_pointer_cast<const LockedNode>(input.second); assert(lockedInput); auto jsonObj3 = jsonObj2 ? jsonObj2->object(input.first) : std::optional<JSONObject>(); - if (!dryRun) - lockedInput->lockedRef.input->fetchTree(store); - auto storePath = lockedInput->computeStorePath(*store); + auto storePath = + dryRun + ? lockedInput->lockedRef.input.computeStorePath(*store) + : lockedInput->lockedRef.input.fetch(store).first.storePath; if (jsonObj3) jsonObj3->attr("path", store->printStorePath(storePath)); sources.insert(std::move(storePath)); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index fde1ca7aa..86d3bfd20 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -194,7 +194,7 @@ void EvalCommand::completeFlakeRef(std::string_view prefix) /* Look for registry entries that match the prefix. */ for (auto & registry : fetchers::getRegistries(getStore())) { for (auto & entry : registry->entries) { - auto from = entry.from->to_string(); + auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index cc239052d..f39213b8f 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -330,7 +330,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); if (element.source - && !element.source->originalRef.input->isImmutable() + && !element.source->originalRef.input.isImmutable() && matches(*store, element, i, matchers)) { Activity act(*logger, lvlChatty, actUnknown, diff --git a/src/nix/registry.cc b/src/nix/registry.cc index e9c76e6c6..16d7e511f 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -31,8 +31,8 @@ struct CmdRegistryList : StoreCommand registry->type == Registry::User ? "user " : registry->type == Registry::System ? "system" : "global", - entry.from->to_string(), - entry.to->to_string()); + entry.from.to_string(), + entry.to.to_string()); } } } @@ -107,7 +107,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand auto ref = parseFlakeRef(url); auto userRegistry = fetchers::getUserRegistry(); userRegistry->remove(ref.input); - auto [tree, resolved] = ref.resolve(store).input->fetchTree(store); + auto [tree, resolved] = ref.resolve(store).input.fetch(store); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; userRegistry->add(ref.input, resolved, extraAttrs); |