diff options
30 files changed, 939 insertions, 1121 deletions
diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 8665abe8c..6b48ead1f 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -76,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s) if (isUri(s)) { return state.store->toRealPath( fetchers::downloadTarball( - state.store, resolveUri(s), "source", false).storePath); + state.store, resolveUri(s), "source", false).first.storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index 8ee17b8f4..2084e3fb3 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -11,7 +11,7 @@ let sourceInfo = if key == lockFile.root then rootSrc - else fetchTree ({ inherit (node.info) narHash; } // removeAttrs node.locked ["dir"]); + else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); inputs = builtins.mapAttrs (inputName: key: allNodes.${key}) (node.inputs or {}); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 78b58cdfa..9741d98c5 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -19,7 +19,7 @@ static FlakeRef maybeLookupFlake( const FlakeRef & flakeRef, bool allowLookup) { - if (!flakeRef.input->isDirect()) { + if (!flakeRef.input.isDirect()) { if (allowLookup) return flakeRef.resolve(store); else @@ -49,16 +49,15 @@ static FlakeRef lookupInFlakeCache( static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, - std::optional<TreeInfo> treeInfo, bool allowLookup, FlakeCache & flakeCache) { /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. */ - if (treeInfo && originalRef.input->isDirect() && originalRef.input->isImmutable()) { + if (originalRef.input.isDirect() && originalRef.input.isImmutable() && originalRef.input.hasAllInfo()) { try { - auto storePath = treeInfo->computeStorePath(*state.store); + auto storePath = originalRef.input.computeStorePath(*state.store); state.store->ensurePath(storePath); @@ -74,7 +73,6 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( Tree { .actualPath = actualPath, .storePath = std::move(storePath), - .info = *treeInfo, }, originalRef, originalRef @@ -99,8 +97,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); - if (treeInfo) - assert(tree.storePath == treeInfo->computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); return {std::move(tree), resolvedRef, lockedRef}; } @@ -202,12 +199,11 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs( static Flake getFlake( EvalState & state, const FlakeRef & originalRef, - std::optional<TreeInfo> treeInfo, bool allowLookup, FlakeCache & flakeCache) { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, originalRef, treeInfo, allowLookup, flakeCache); + state, originalRef, allowLookup, flakeCache); // Guard against symlink attacks. auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix"); @@ -278,7 +274,7 @@ static Flake getFlake( Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) { FlakeCache flakeCache; - return getFlake(state, originalRef, {}, allowLookup, flakeCache); + return getFlake(state, originalRef, allowLookup, flakeCache); } /* Compute an in-memory lock file for the specified top-level flake, @@ -292,7 +288,7 @@ LockedFlake lockFlake( FlakeCache flakeCache; - auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache); + auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); // FIXME: symlink attack auto oldLockFile = LockFile::read( @@ -393,7 +389,7 @@ LockedFlake lockFlake( didn't change and there is no override from a higher level flake. */ auto childNode = std::make_shared<LockedNode>( - oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake); + oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); node->inputs.insert_or_assign(id, childNode); @@ -409,7 +405,7 @@ LockedFlake lockFlake( if (hasChildUpdate) { auto inputFlake = getFlake( - state, oldLock->lockedRef, oldLock->info, false, flakeCache); + state, oldLock->lockedRef, false, flakeCache); computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); } else { /* No need to fetch this flake, we can be @@ -440,11 +436,11 @@ LockedFlake lockFlake( /* We need to create a new lock file entry. So fetch this input. */ - if (!lockFlags.allowMutable && !input.ref.input->isImmutable()) + if (!lockFlags.allowMutable && !input.ref.input.isImmutable()) throw Error("cannot update flake input '%s' in pure mode", inputPathS); if (input.isFlake) { - auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache); + auto inputFlake = getFlake(state, input.ref, lockFlags.useRegistries, flakeCache); /* Note: in case of an --override-input, we use the *original* ref (input2.ref) for the @@ -454,7 +450,7 @@ LockedFlake lockFlake( file. That is, overrides are sticky unless you use --no-write-lock-file. */ auto childNode = std::make_shared<LockedNode>( - inputFlake.lockedRef, input2.ref, inputFlake.sourceInfo->info); + inputFlake.lockedRef, input2.ref); node->inputs.insert_or_assign(id, childNode); @@ -479,9 +475,9 @@ LockedFlake lockFlake( else { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, input.ref, {}, lockFlags.useRegistries, flakeCache); + state, input.ref, lockFlags.useRegistries, flakeCache); node->inputs.insert_or_assign(id, - std::make_shared<LockedNode>(lockedRef, input.ref, sourceInfo.info, false)); + std::make_shared<LockedNode>(lockedRef, input.ref, false)); } } } @@ -534,7 +530,7 @@ LockedFlake lockFlake( printInfo("inputs of flake '%s' changed:\n%s", topRef, chomp(diff)); if (lockFlags.writeLockFile) { - if (auto sourcePath = topRef.input->getSourcePath()) { + if (auto sourcePath = topRef.input.getSourcePath()) { if (!newLockFile.isImmutable()) { if (settings.warnDirty) warn("will not write lock file of flake '%s' because it has a mutable input", topRef); @@ -555,7 +551,7 @@ LockedFlake lockFlake( newLockFile.write(path); - topRef.input->markChangedFile( + topRef.input.markChangedFile( (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", lockFlags.commitLockFile ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s", @@ -567,19 +563,19 @@ LockedFlake lockFlake( also just clear the 'rev' field... */ auto prevLockedRef = flake.lockedRef; FlakeCache dummyCache; - flake = getFlake(state, topRef, {}, lockFlags.useRegistries, dummyCache); + flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache); if (lockFlags.commitLockFile && - flake.lockedRef.input->getRev() && - prevLockedRef.input->getRev() != flake.lockedRef.input->getRev()) - warn("committed new revision '%s'", flake.lockedRef.input->getRev()->gitRev()); + flake.lockedRef.input.getRev() && + prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) + warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); /* Make sure that we picked up the change, i.e. the tree should usually be dirty now. Corner case: we could have reverted from a dirty to a clean tree! */ if (flake.lockedRef.input == prevLockedRef.input - && !flake.lockedRef.input->isImmutable()) + && !flake.lockedRef.input.isImmutable()) throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); } } else @@ -625,7 +621,7 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va { auto flakeRefS = state.forceStringNoCtx(*args[0], pos); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (evalSettings.pureEval && !flakeRef.input->isImmutable()) + if (evalSettings.pureEval && !flakeRef.input.isImmutable()) throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); callFlake(state, @@ -650,8 +646,8 @@ Fingerprint LockedFlake::getFingerprint() const return hashString(htSHA256, fmt("%s;%d;%d;%s", flake.sourceInfo->storePath.to_string(), - flake.sourceInfo->info.revCount.value_or(0), - flake.sourceInfo->info.lastModified.value_or(0), + flake.lockedRef.input.getRevCount().value_or(0), + flake.lockedRef.input.getLastModified().value_or(0), lockFile)); } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 59a1adb3b..ebf81362c 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -104,7 +104,7 @@ void callFlake( void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, - std::shared_ptr<const fetchers::Input> input, + const fetchers::Input & input, Value & v); } diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index a70261a41..615269218 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -15,7 +15,7 @@ const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRege std::string FlakeRef::to_string() const { - auto url = input->toURL(); + auto url = input.toURL(); if (subdir != "") url.query.insert_or_assign("dir", subdir); return url.to_string(); @@ -23,7 +23,7 @@ std::string FlakeRef::to_string() const fetchers::Attrs FlakeRef::toAttrs() const { - auto attrs = input->toAttrs(); + auto attrs = input.toAttrs(); if (subdir != "") attrs.emplace("dir", subdir); return attrs; @@ -37,13 +37,13 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) bool FlakeRef::operator ==(const FlakeRef & other) const { - return *input == *other.input && subdir == other.subdir; + return input == other.input && subdir == other.subdir; } FlakeRef FlakeRef::resolve(ref<Store> store) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); - return FlakeRef(input2, fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); + return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); } FlakeRef parseFlakeRef( @@ -98,7 +98,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(inputFromURL(parsedURL), ""), + FlakeRef(Input::fromURL(parsedURL), ""), percentDecode(std::string(match[6]))); } @@ -143,7 +143,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( } return std::make_pair( - FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), fragment); } @@ -155,7 +155,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( attrs.insert_or_assign("type", "path"); attrs.insert_or_assign("path", path); - return std::make_pair(FlakeRef(inputFromAttrs(attrs), ""), fragment); + return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); } else { @@ -163,7 +163,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( std::string fragment; std::swap(fragment, parsedURL.fragment); return std::make_pair( - FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), fragment); } } @@ -183,14 +183,14 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) auto attrs2(attrs); attrs2.erase("dir"); return FlakeRef( - fetchers::inputFromAttrs(attrs2), + fetchers::Input::fromAttrs(std::move(attrs2)), fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const { - auto [tree, lockedInput] = input->fetchTree(store); - return {std::move(tree), FlakeRef(lockedInput, subdir)}; + auto [tree, lockedInput] = input.fetch(store); + return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; } } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 72cbb2908..f4eb825a6 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -14,17 +14,15 @@ typedef std::string FlakeId; struct FlakeRef { - std::shared_ptr<const fetchers::Input> input; + fetchers::Input input; Path subdir; bool operator==(const FlakeRef & other) const; - FlakeRef(const std::shared_ptr<const fetchers::Input> & input, const Path & subdir) - : input(input), subdir(subdir) - { - assert(input); - } + FlakeRef(fetchers::Input && input, const Path & subdir) + : input(std::move(input)), subdir(subdir) + { } // FIXME: change to operator <<. std::string to_string() const; diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 5c58d6080..68e587650 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -5,35 +5,40 @@ namespace nix::flake { -FlakeRef flakeRefFromJson(const nlohmann::json & json) -{ - return FlakeRef::fromAttrs(jsonToAttrs(json)); -} - FlakeRef getFlakeRef( const nlohmann::json & json, - const char * attr) + const char * attr, + const char * info) { auto i = json.find(attr); - if (i != json.end()) - return flakeRefFromJson(*i); + if (i != json.end()) { + auto attrs = jsonToAttrs(*i); + // FIXME: remove when we drop support for version 5. + if (info) { + auto j = json.find(info); + if (j != json.end()) { + for (auto k : jsonToAttrs(*j)) + attrs.insert_or_assign(k.first, k.second); + } + } + return FlakeRef::fromAttrs(attrs); + } throw Error("attribute '%s' missing in lock file", attr); } LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked")) - , originalRef(getFlakeRef(json, "original")) - , info(TreeInfo::fromJson(json)) + : lockedRef(getFlakeRef(json, "locked", "info")) + , originalRef(getFlakeRef(json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { - if (!lockedRef.input->isImmutable()) - throw Error("lockfile contains mutable flakeref '%s'", lockedRef); + if (!lockedRef.input.isImmutable()) + throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs())); } StorePath LockedNode::computeStorePath(Store & store) const { - return info.computeStorePath(store); + return lockedRef.input.computeStorePath(store); } std::shared_ptr<Node> Node::findInput(const InputPath & path) @@ -53,7 +58,7 @@ std::shared_ptr<Node> Node::findInput(const InputPath & path) LockFile::LockFile(const nlohmann::json & json, const Path & path) { auto version = json.value("version", 0); - if (version != 5) + if (version < 5 || version > 6) throw Error("lock file '%s' has unsupported version %d", path, version); std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap; @@ -119,7 +124,6 @@ nlohmann::json LockFile::toJson() const if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) { n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs()); n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs()); - n["info"] = lockedNode->info.toJson(); if (!lockedNode->isFlake) n["flake"] = false; } @@ -129,7 +133,7 @@ nlohmann::json LockFile::toJson() const }; nlohmann::json json; - json["version"] = 5; + json["version"] = 6; json["root"] = dumpNode("root", root); json["nodes"] = std::move(nodes); @@ -176,7 +180,7 @@ bool LockFile::isImmutable() const for (auto & i : nodes) { if (i == root) continue; auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i); - if (lockedNode && !lockedNode->lockedRef.input->isImmutable()) return false; + if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false; } return true; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index c34939ebc..ba47f9b89 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -31,15 +31,13 @@ struct Node : std::enable_shared_from_this<Node> struct LockedNode : Node { FlakeRef lockedRef, originalRef; - TreeInfo info; bool isFlake = true; LockedNode( const FlakeRef & lockedRef, const FlakeRef & originalRef, - const TreeInfo & info, bool isFlake = true) - : lockedRef(lockedRef), originalRef(originalRef), info(info), isFlake(isFlake) + : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) { } LockedNode(const nlohmann::json & json); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 1993fa6c1..1ac5217ba 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -689,7 +689,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { res = { true, store->toRealPath(fetchers::downloadTarball( - store, resolveUri(elem.second), "source", false).storePath) }; + store, resolveUri(elem.second), "source", false).first.storePath) }; } catch (FileTransferError & e) { printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); res = { false, "" }; diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc index 1a8798fcc..a6539e888 100644 --- a/src/libexpr/primops/fetchGit.cc +++ b/src/libexpr/primops/fetchGit.cc @@ -56,23 +56,23 @@ static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Va attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - if (fetchSubmodules) attrs.insert_or_assign("submodules", true); - auto input = fetchers::inputFromAttrs(attrs); + if (fetchSubmodules) attrs.insert_or_assign("submodules", fetchers::Explicit<bool>{true}); + auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name? - auto [tree, input2] = input->fetchTree(state.store); + auto [tree, input2] = input.fetch(state.store); state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(htSHA1)); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); // Backward compatibility: set 'revCount' to 0 for a dirty tree. mkInt(*state.allocAttr(v, state.symbols.create("revCount")), - tree.info.revCount.value_or(0)); + input2.getRevCount().value_or(0)); mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); v.attrs->sort(); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 0a1ba49d5..90030ea76 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -59,23 +59,23 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::inputFromAttrs(attrs); + auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input->fetchTree(state.store); + auto [tree, input2] = input.fetch(state.store); state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - if (input2->getRef()) - mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + if (input2.getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(htSHA1)); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (auto revCount = input2.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); v.attrs->sort(); if (state.allowedPaths) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f9dfb1164..a1ad0a7b9 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -13,31 +13,36 @@ namespace nix { void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, - std::shared_ptr<const fetchers::Input> input, + const fetchers::Input & input, Value & v) { + assert(input.isImmutable()); + state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - assert(tree.info.narHash); + // FIXME: support arbitrary input attributes. + + auto narHash = input.getNarHash(); + assert(narHash); mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI)); + narHash->to_string(SRI)); - if (input->getRev()) { - mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + if (auto rev = input.getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); } - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (auto revCount = input.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); - if (tree.info.lastModified) { - mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *tree.info.lastModified); + if (auto lastModified = input.getLastModified()) { + mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")), - fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S"))); } v.attrs->sort(); @@ -47,7 +52,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V { settings.requireExperimentalFeature("flakes"); - std::shared_ptr<const fetchers::Input> input; + fetchers::Input input; PathSet context; state.forceValue(*args[0]); @@ -62,7 +67,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V if (attr.value->type == tString) attrs.emplace(attr.name, attr.value->string.s); else if (attr.value->type == tBool) - attrs.emplace(attr.name, attr.value->boolean); + attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean}); else if (attr.value->type == tInt) attrs.emplace(attr.name, attr.value->integer); else @@ -73,18 +78,42 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V if (!attrs.count("type")) throw Error("attribute 'type' is missing in call to 'fetchTree', at %s", pos); - input = fetchers::inputFromAttrs(attrs); + input = fetchers::Input::fromAttrs(std::move(attrs)); } else - input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + input = fetchers::Input::fromURL(state.coerceToString(pos, *args[0], context, false, false)); - if (!evalSettings.pureEval && !input->isDirect()) + if (!evalSettings.pureEval && !input.isDirect()) input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input->isImmutable()) + if (evalSettings.pureEval && !input.isImmutable()) throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos); - // FIXME: use fetchOrSubstituteTree - auto [tree, input2] = input->fetchTree(state.store); + /* The tree may already be in the Nix store, or it could be + substituted (which is often faster than fetching from the + original source). So check that. */ + if (input.hasAllInfo()) { + auto storePath = input.computeStorePath(*state.store); + + try { + state.store->ensurePath(storePath); + + debug("using substituted/cached input '%s' in '%s'", + input.to_string(), state.store->printStorePath(storePath)); + + auto actualPath = state.store->toRealPath(storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(actualPath); + + emitTreeAttrs(state, fetchers::Tree { .actualPath = actualPath, .storePath = std::move(storePath) }, input, v); + + return; + } catch (Error & e) { + debug("substitution of input '%s' failed: %s", input.to_string(), e.what()); + } + } + + auto [tree, input2] = input.fetch(state.store); if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); @@ -137,7 +166,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; auto path = state.store->toRealPath(storePath); diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index feb0a6085..1e59faa73 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs) { nlohmann::json json; for (auto & attr : attrs) { - if (auto v = std::get_if<int64_t>(&attr.second)) { + if (auto v = std::get_if<uint64_t>(&attr.second)) { json[attr.first] = *v; } else if (auto v = std::get_if<std::string>(&attr.second)) { json[attr.first] = *v; @@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name) return *s; } -std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name) +std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name) { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if<int64_t>(&i->second)) + if (auto v = std::get_if<uint64_t>(&i->second)) return *v; throw Error("input attribute '%s' is not an integer", name); } -int64_t getIntAttr(const Attrs & attrs, const std::string & name) +uint64_t getIntAttr(const Attrs & attrs, const std::string & name) { auto s = maybeGetIntAttr(attrs, name); if (!s) @@ -76,8 +76,8 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na { auto i = attrs.find(name); if (i == attrs.end()) return {}; - if (auto v = std::get_if<int64_t>(&i->second)) - return *v; + if (auto v = std::get_if<Explicit<bool>>(&i->second)) + return v->t; throw Error("input attribute '%s' is not a Boolean", name); } @@ -93,7 +93,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs) { std::map<std::string, std::string> query; for (auto & attr : attrs) { - if (auto v = std::get_if<int64_t>(&attr.second)) { + if (auto v = std::get_if<uint64_t>(&attr.second)) { query.insert_or_assign(attr.first, fmt("%d", *v)); } else if (auto v = std::get_if<std::string>(&attr.second)) { query.insert_or_assign(attr.first, *v); diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index d6e0ae000..4b4630c80 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,9 +13,14 @@ namespace nix::fetchers { template<typename T> struct Explicit { T t; + + bool operator ==(const Explicit<T> & other) const + { + return t == other.t; + } }; -typedef std::variant<std::string, int64_t, Explicit<bool>> Attr; +typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr; typedef std::map<std::string, Attr> Attrs; Attrs jsonToAttrs(const nlohmann::json & json); @@ -26,9 +31,9 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin std::string getStrAttr(const Attrs & attrs, const std::string & name); -std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name); +std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name); -int64_t getIntAttr(const Attrs & attrs, const std::string & name); +uint64_t getIntAttr(const Attrs & attrs, const std::string & name); std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 83268b4bf..e4852d662 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -5,82 +5,234 @@ namespace nix::fetchers { -std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr; +std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr; -void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme) +void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>(); + if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>(); inputSchemes->push_back(std::move(inputScheme)); } -std::unique_ptr<Input> inputFromURL(const ParsedURL & url) +Input Input::fromURL(const std::string & url) +{ + return fromURL(parseURL(url)); +} + +static void fixupInput(Input & input) +{ + // Check common attributes. + input.getType(); + input.getRef(); + if (input.getRev()) + input.immutable = true; + input.getRevCount(); + input.getLastModified(); + if (input.getNarHash()) + input.immutable = true; +} + +Input Input::fromURL(const ParsedURL & url) { for (auto & inputScheme : *inputSchemes) { auto res = inputScheme->inputFromURL(url); - if (res) return res; + if (res) { + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); + } } - throw Error("input '%s' is unsupported", url.url); -} -std::unique_ptr<Input> inputFromURL(const std::string & url) -{ - return inputFromURL(parseURL(url)); + throw Error("input '%s' is unsupported", url.url); } -std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) +Input Input::fromAttrs(Attrs && attrs) { - auto attrs2(attrs); - attrs2.erase("narHash"); for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs2); + auto res = inputScheme->inputFromAttrs(attrs); if (res) { - if (auto narHash = maybeGetStrAttr(attrs, "narHash")) - // FIXME: require SRI hash. - res->narHash = Hash(*narHash); - return res; + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); } } - throw Error("input '%s' is unsupported", attrsToJson(attrs)); + + Input input; + input.attrs = attrs; + fixupInput(input); + return input; +} + +ParsedURL Input::toURL() const +{ + if (!scheme) + throw Error("cannot show unsupported input '%s'", attrsToJson(attrs)); + return scheme->toURL(*this); +} + +std::string Input::to_string() const +{ + return toURL().to_string(); } Attrs Input::toAttrs() const { - auto attrs = toAttrsInternal(); - if (narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - attrs.emplace("type", type()); return attrs; } -std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const +bool Input::hasAllInfo() const { - auto [tree, input] = fetchTreeInternal(store); + return getNarHash() && scheme && scheme->hasAllInfo(*this); +} + +bool Input::operator ==(const Input & other) const +{ + return attrs == other.attrs; +} + +bool Input::contains(const Input & other) const +{ + auto other2(other); + other2.attrs.erase("ref"); + other2.attrs.erase("rev"); + if (*this == other2) return true; + return false; +} + +std::pair<Tree, Input> Input::fetch(ref<Store> store) const +{ + if (!scheme) + throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs())); + + auto [tree, input] = scheme->fetch(store, *this); if (tree.actualPath == "") tree.actualPath = store->toRealPath(tree.storePath); - if (!tree.info.narHash) - tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash; + auto narHash = store->queryPathInfo(tree.storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(SRI)); - if (input->narHash) - assert(input->narHash == tree.info.narHash); + if (auto narHash2 = getNarHash()) { + if (narHash != *narHash2) + throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", + to_string(), tree.actualPath, narHash2->to_string(SRI), narHash.to_string(SRI)); + } - if (narHash && narHash != input->narHash) - throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, narHash->to_string(SRI), input->narHash->to_string(SRI)); + // FIXME: check lastModified, revCount + + input.immutable = true; + + assert(input.hasAllInfo()); return {std::move(tree), input}; } -std::shared_ptr<const Input> Input::applyOverrides( +Input Input::applyOverrides( std::optional<std::string> ref, std::optional<Hash> rev) const { + if (!scheme) return *this; + return scheme->applyOverrides(*this, ref, rev); +} + +void Input::clone(const Path & destDir) const +{ + assert(scheme); + scheme->clone(*this, destDir); +} + +std::optional<Path> Input::getSourcePath() const +{ + assert(scheme); + return scheme->getSourcePath(*this); +} + +void Input::markChangedFile( + std::string_view file, + std::optional<std::string> commitMsg) const +{ + assert(scheme); + return scheme->markChangedFile(*this, file, commitMsg); +} + +StorePath Input::computeStorePath(Store & store) const +{ + auto narHash = getNarHash(); + if (!narHash) + throw Error("cannot compute store path for mutable input '%s'", to_string()); + return store.makeFixedOutputPath(true, *narHash, "source"); +} + +std::string Input::getType() const +{ + return getStrAttr(attrs, "type"); +} + +std::optional<Hash> Input::getNarHash() const +{ + if (auto s = maybeGetStrAttr(attrs, "narHash")) + // FIXME: require SRI hash. + return Hash(*s, htSHA256); + return {}; +} + +std::optional<std::string> Input::getRef() const +{ + if (auto s = maybeGetStrAttr(attrs, "ref")) + return *s; + return {}; +} + +std::optional<Hash> Input::getRev() const +{ + if (auto s = maybeGetStrAttr(attrs, "rev")) + return Hash(*s, htSHA1); + return {}; +} + +std::optional<uint64_t> Input::getRevCount() const +{ + if (auto n = maybeGetIntAttr(attrs, "revCount")) + return *n; + return {}; +} + +std::optional<time_t> Input::getLastModified() const +{ + if (auto n = maybeGetIntAttr(attrs, "lastModified")) + return *n; + return {}; +} + +ParsedURL InputScheme::toURL(const Input & input) +{ + throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs)); +} + +Input InputScheme::applyOverrides( + const Input & input, + std::optional<std::string> ref, + std::optional<Hash> rev) +{ if (ref) - throw Error("don't know how to apply '%s' to '%s'", *ref, to_string()); + throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref); if (rev) - throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string()); - return shared_from_this(); + throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev()); + return input; +} + +std::optional<Path> InputScheme::getSourcePath(const Input & input) +{ + return {}; +} + +void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) +{ + assert(false); +} + +void InputScheme::clone(const Input & input, const Path & destDir) +{ + throw Error("do not know how to clone input '%s'", input.to_string()); } } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index b75dcffa5..c43cfe50c 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "hash.hh" #include "path.hh" -#include "tree-info.hh" #include "attrs.hh" #include "url.hh" @@ -13,89 +12,100 @@ namespace nix { class Store; } namespace nix::fetchers { -struct Input; - struct Tree { Path actualPath; StorePath storePath; - TreeInfo info; }; -struct Input : std::enable_shared_from_this<Input> +struct InputScheme; + +struct Input { - std::optional<Hash> narHash; // FIXME: implement + friend class InputScheme; + + std::shared_ptr<InputScheme> scheme; // note: can be null + Attrs attrs; + bool immutable = false; + bool direct = true; + +public: + static Input fromURL(const std::string & url); + + static Input fromURL(const ParsedURL & url); - virtual std::string type() const = 0; + static Input fromAttrs(Attrs && attrs); - virtual ~Input() { } + ParsedURL toURL() const; - virtual bool operator ==(const Input & other) const { return false; } + std::string to_string() const; + + Attrs toAttrs() const; /* Check whether this is a "direct" input, that is, not one that goes through a registry. */ - virtual bool isDirect() const { return true; } + bool isDirect() const { return direct; } /* Check whether this is an "immutable" input, that is, one that contains a commit hash or content hash. */ - virtual bool isImmutable() const { return (bool) narHash; } - - virtual bool contains(const Input & other) const { return false; } - - virtual std::optional<std::string> getRef() const { return {}; } - - virtual std::optional<Hash> getRev() const { return {}; } + bool isImmutable() const { return immutable; } - virtual ParsedURL toURL() const = 0; + bool hasAllInfo() const; - std::string to_string() const - { - return toURL().to_string(); - } + bool operator ==(const Input & other) const; - Attrs toAttrs() const; + bool contains(const Input & other) const; - std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const; + std::pair<Tree, Input> fetch(ref<Store> store) const; - virtual std::shared_ptr<const Input> applyOverrides( + Input applyOverrides( std::optional<std::string> ref, std::optional<Hash> rev) const; - virtual std::optional<Path> getSourcePath() const { return {}; } - - virtual void markChangedFile( - std::string_view file, - std::optional<std::string> commitMsg) const - { assert(false); } + void clone(const Path & destDir) const; - virtual void clone(const Path & destDir) const - { - throw Error("do not know how to clone input '%s'", to_string()); - } + std::optional<Path> getSourcePath() const; -private: + void markChangedFile( + std::string_view file, + std::optional<std::string> commitMsg) const; - virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0; + StorePath computeStorePath(Store & store) const; - virtual Attrs toAttrsInternal() const = 0; + // Convience functions for common attributes. + std::string getType() const; + std::optional<Hash> getNarHash() const; + std::optional<std::string> getRef() const; + std::optional<Hash> getRev() const; + std::optional<uint64_t> getRevCount() const; + std::optional<time_t> getLastModified() const; }; struct InputScheme { - virtual ~InputScheme() { } + virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0; - virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0; + virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0; - virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0; -}; + virtual ParsedURL toURL(const Input & input); -std::unique_ptr<Input> inputFromURL(const ParsedURL & url); + virtual bool hasAllInfo(const Input & input) = 0; -std::unique_ptr<Input> inputFromURL(const std::string & url); + virtual Input applyOverrides( + const Input & input, + std::optional<std::string> ref, + std::optional<Hash> rev); + + virtual void clone(const Input & input, const Path & destDir); + + virtual std::optional<Path> getSourcePath(const Input & input); -std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs); + virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg); + + virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0; +}; -void registerInputScheme(std::unique_ptr<InputScheme> && fetcher); +void registerInputScheme(std::shared_ptr<InputScheme> && fetcher); struct DownloadFileResult { @@ -110,7 +120,7 @@ DownloadFileResult downloadFile( const std::string & name, bool immutable); -Tree downloadTarball( +std::pair<Tree, time_t> downloadTarball( ref<Store> store, const std::string & url, const std::string & name, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c9f9a4b23..4fcf3f542 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -22,110 +22,121 @@ static bool isNotDotGitDirectory(const Path & path) return not std::regex_match(path, gitDirRegex); } -struct GitInput : Input +struct GitInputScheme : InputScheme { - ParsedURL url; - std::optional<std::string> ref; - std::optional<Hash> rev; - bool shallow = false; - bool submodules = false; + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "git" && + url.scheme != "git+http" && + url.scheme != "git+https" && + url.scheme != "git+ssh" && + url.scheme != "git+file") return {}; - GitInput(const ParsedURL & url) : url(url) - { } + auto url2(url); + if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); + url2.query.clear(); - std::string type() const override { return "git"; } + Attrs attrs; + attrs.emplace("type", "git"); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const GitInput *>(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } + + attrs.emplace("url", url2.to_string()); + + return inputFromAttrs(attrs); } - bool isImmutable() const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != "git") return {}; - std::optional<std::string> getRef() const override { return ref; } + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash") + throw Error("unsupported Git input attribute '%s'", name); + + parseURL(getStrAttr(attrs, "url")); + maybeGetBoolAttr(attrs, "shallow"); + maybeGetBoolAttr(attrs, "submodules"); + + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Git branch/tag name '%s'", *ref); + } - std::optional<Hash> getRev() const override { return rev; } + Input input; + input.attrs = attrs; + return input; + } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - ParsedURL url2(url); - if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); - if (shallow) url2.query.insert_or_assign("shallow", "1"); - return url2; + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme != "git") url.scheme = "git+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); + if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + url.query.insert_or_assign("shallow", "1"); + return url; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (shallow) - attrs.emplace("shallow", true); - if (submodules) - attrs.emplace("submodules", true); - return attrs; + bool maybeDirty = !input.getRef(); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + return + maybeGetIntAttr(input.attrs, "lastModified") + && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); } - void clone(const Path & destDir) const override + Input applyOverrides( + const Input & input, + std::optional<std::string> ref, + std::optional<Hash> rev) override + { + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); + if (!res.getRef() && res.getRev()) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string()); + return res; + } + + void clone(const Input & input, const Path & destDir) override { - auto [isLocal, actualUrl] = getActualUrl(); + auto [isLocal, actualUrl] = getActualUrl(input); Strings args = {"clone"}; args.push_back(actualUrl); - if (ref) { + if (auto ref = input.getRef()) { args.push_back("--branch"); args.push_back(*ref); } - if (rev) throw Error("cloning a specific revision is not implemented"); + if (input.getRev()) throw Error("cloning a specific revision is not implemented"); args.push_back(destDir); runProgram("git", true, args); } - std::shared_ptr<const Input> applyOverrides( - std::optional<std::string> ref, - std::optional<Hash> rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared<GitInput>(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - if (!res->ref && res->rev) - throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); - - return res; - } - - std::optional<Path> getSourcePath() const override + std::optional<Path> getSourcePath(const Input & input) override { - if (url.scheme == "file" && !ref && !rev) + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) return url.path; return {}; } - void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override { - auto sourcePath = getSourcePath(); + auto sourcePath = getSourcePath(input); assert(sourcePath); runProgram("git", true, @@ -136,23 +147,25 @@ struct GitInput : Input { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); } - std::pair<bool, std::string> getActualUrl() const + std::pair<bool, std::string> getActualUrl(const Input & input) const { // Don't clone file:// URIs (but otherwise treat them the // same as remote URIs, i.e. don't use the working tree or // HEAD). static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file" && !forceHttp; return {isLocal, isLocal ? url.path : url.base}; } - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared<GitInput>(*this); + Input input(_input); - assert(!rev || rev->type == htSHA1); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; @@ -163,39 +176,38 @@ struct GitInput : Input return Attrs({ {"type", cacheType}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair<Tree, std::shared_ptr<const Input>> + -> std::pair<Tree, Input> { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + if (!shallow) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); return { Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")), - .lastModified = getIntAttr(infoAttrs, "lastModified"), - }, }, input }; }; - if (rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. - if (!input->ref && !input->rev && isLocal) { + if (!input.getRef() && !input.getRev() && isLocal) { bool clean = false; /* Check whether this repo has any commits. There are @@ -254,35 +266,37 @@ struct GitInput : Input auto storePath = store->addToStore("source", actualUrl, true, htSHA256, filter); - auto tree = Tree { - .actualPath = store->printStorePath(storePath), - .storePath = std::move(storePath), - .info = TreeInfo { - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - .lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0, - } + // FIXME: maybe we should use the timestamp of the last + // modified dirty file? + input.attrs.insert_or_assign( + "lastModified", + haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0); + + return { + Tree { + .actualPath = store->printStorePath(storePath), + .storePath = std::move(storePath), + }, input }; - - return {std::move(tree), input}; } } - if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master"); Attrs mutableAttrs({ {"type", cacheType}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); Path repoDir; if (isLocal) { - if (!input->rev) - input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", + Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev()); repoDir = actualUrl; @@ -290,8 +304,8 @@ struct GitInput : Input if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -305,18 +319,18 @@ struct GitInput : Input } Path localRefFile = - input->ref->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input->ref - : cacheDir + "/refs/heads/" + *input->ref; + input.getRef()->compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + *input.getRef() + : cacheDir + "/refs/heads/" + *input.getRef(); bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input->rev) { + if (input.getRev()) { try { - runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() }); + runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() }); doFetch = false; } catch (ExecError & e) { if (WIFEXITED(e.status)) { @@ -339,7 +353,7 @@ struct GitInput : Input // FIXME: git stderr messes up our progress indicator, so // we're using --quiet for now. Should process its stderr. try { - runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input->ref, *input->ref) }); + runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", *input.getRef(), *input.getRef()) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); @@ -354,8 +368,8 @@ struct GitInput : Input utimes(localRefFile.c_str(), times); } - if (!input->rev) - input->rev = Hash(chomp(readFile(localRefFile)), htSHA1); + if (!input.getRev()) + input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev()); } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; @@ -365,7 +379,7 @@ struct GitInput : Input // FIXME: check whether rev is an ancestor of ref. - printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl); + printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); /* Now that we know the ref, check again whether we have it in the store. */ @@ -387,7 +401,7 @@ struct GitInput : Input runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() }); + runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl }); runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); @@ -396,7 +410,7 @@ struct GitInput : Input // FIXME: should pipe this, or find some better way to extract a // revision. auto source = sinkToSource([&](Sink & sink) { - RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() }); + RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() }); gitOptions.standardOut = &sink; runProgram2(gitOptions); }); @@ -406,18 +420,18 @@ struct GitInput : Input auto storePath = store->addToStore(name, tmpDir, true, htSHA256, filter); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() })); + auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() })); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"lastModified", lastModified}, }); if (!shallow) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() }))); + std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() }))); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -436,60 +450,6 @@ struct GitInput : Input } }; -struct GitInputScheme : InputScheme -{ - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "git" && - url.scheme != "git+http" && - url.scheme != "git+https" && - url.scheme != "git+ssh" && - url.scheme != "git+file") return nullptr; - - auto url2(url); - if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "git"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules") - throw Error("unsupported Git input attribute '%s'", name); - - auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Git branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - - input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false); - - input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false); - - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); }); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index e59c83be4..8d113967e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,37 +8,80 @@ namespace nix::fetchers { -struct GitArchiveInput : Input +struct GitArchiveInputScheme : InputScheme { - std::string owner; - std::string repo; - std::optional<std::string> ref; - std::optional<Hash> rev; - - virtual std::shared_ptr<GitArchiveInput> _clone() const = 0; + virtual std::string type() = 0; - bool operator ==(const Input & other) const override + std::optional<Input> inputFromURL(const ParsedURL & url) override { - auto other2 = dynamic_cast<const GitArchiveInput *>(&other); - return - other2 - && owner == other2->owner - && repo == other2->repo - && rev == other2->rev - && ref == other2->ref; + if (url.scheme != type()) return {}; + + auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); + + std::optional<Hash> rev; + std::optional<std::string> ref; + + if (path.size() == 2) { + } else if (path.size() == 3) { + if (std::regex_match(path[2], revRegex)) + rev = Hash(path[2], htSHA1); + else if (std::regex_match(path[2], refRegex)) + ref = path[2]; + else + throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + } else + throw BadURL("URL '%s' is invalid", url.url); + + for (auto &[name, value] : url.query) { + if (name == "rev") { + if (rev) + throw BadURL("URL '%s' contains multiple commit hashes", url.url); + rev = Hash(value, htSHA1); + } + else if (name == "ref") { + if (!std::regex_match(value, refRegex)) + throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); + if (ref) + throw BadURL("URL '%s' contains multiple branch/tag names", url.url); + ref = value; + } + } + + if (ref && rev) + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url); + + Input input; + input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("owner", path[0]); + input.attrs.insert_or_assign("repo", path[1]); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); + + return input; } - bool isImmutable() const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != type()) return {}; - std::optional<std::string> getRef() const override { return ref; } + for (auto & [name, value] : attrs) + if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified") + throw Error("unsupported input attribute '%s'", name); - std::optional<Hash> getRev() const override { return rev; } + getStrAttr(attrs, "owner"); + getStrAttr(attrs, "repo"); + + Input input; + input.attrs = attrs; + return input; + } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { + auto owner = getStrAttr(input.attrs, "owner"); + auto repo = getStrAttr(input.attrs, "repo"); + auto ref = input.getRef(); + auto rev = input.getRev(); auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; @@ -49,32 +92,44 @@ struct GitArchiveInput : Input }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("owner", owner); - attrs.emplace("repo", repo); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); + } + + Input applyOverrides( + const Input & _input, + std::optional<std::string> ref, + std::optional<Hash> rev) override + { + auto input(_input); + if (rev) { + input.attrs.insert_or_assign("rev", rev->gitRev()); + input.attrs.erase("ref"); + } + if (ref) { + if (input.getRev()) + throw BadURL("input '%s' contains both a commit hash and a branch/tag name", input.to_string()); + input.attrs.insert_or_assign("ref", *ref); + } + return input; } - virtual Hash getRevFromRef(nix::ref<Store> store, std::string_view ref) const = 0; + virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0; - virtual std::string getDownloadUrl() const = 0; + virtual std::string getDownloadUrl(const Input & input) const = 0; - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override { - auto rev = this->rev; - auto ref = this->ref.value_or("master"); + Input input(_input); - if (!rev) rev = getRevFromRef(store, ref); + if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "master"); - auto input = _clone(); - input->ref = {}; - input->rev = *rev; + auto rev = input.getRev(); + if (!rev) rev = getRevFromRef(store, input); + + input.attrs.erase("ref"); + input.attrs.insert_or_assign("rev", rev->gitRev()); Attrs immutableAttrs({ {"type", "git-tarball"}, @@ -82,131 +137,44 @@ struct GitArchiveInput : Input }); if (auto res = getCache()->lookup(store, immutableAttrs)) { + input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); return { Tree{ .actualPath = store->toRealPath(res->second), .storePath = std::move(res->second), - .info = TreeInfo { - .lastModified = getIntAttr(res->first, "lastModified"), - }, }, input }; } - auto url = input->getDownloadUrl(); + auto url = getDownloadUrl(input); + auto [tree, lastModified] = downloadTarball(store, url, "source", true); - auto tree = downloadTarball(store, url, "source", true); + input.attrs.insert_or_assign("lastModified", lastModified); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", *tree.info.lastModified} + {"lastModified", lastModified} }, tree.storePath, true); return {std::move(tree), input}; } - - std::shared_ptr<const Input> applyOverrides( - std::optional<std::string> ref, - std::optional<Hash> rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = _clone(); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } }; -struct GitArchiveInputScheme : InputScheme -{ - std::string type; - - GitArchiveInputScheme(std::string && type) : type(type) - { } - - virtual std::unique_ptr<GitArchiveInput> create() = 0; - - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != type) return nullptr; - - auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); - auto input = create(); - - if (path.size() == 2) { - } else if (path.size() == 3) { - if (std::regex_match(path[2], revRegex)) - input->rev = Hash(path[2], htSHA1); - else if (std::regex_match(path[2], refRegex)) - input->ref = path[2]; - else - throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else - throw BadURL("URL '%s' is invalid", url.url); - - for (auto &[name, value] : url.query) { - if (name == "rev") { - if (input->rev) - throw BadURL("URL '%s' contains multiple commit hashes", url.url); - input->rev = Hash(value, htSHA1); - } - else if (name == "ref") { - if (!std::regex_match(value, refRegex)) - throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); - if (input->ref) - throw BadURL("URL '%s' contains multiple branch/tag names", url.url); - input->ref = value; - } - } - - if (input->ref && input->rev) - throw BadURL("URL '%s' contains both a commit hash and a branch/tag name", url.url); - - input->owner = path[0]; - input->repo = path[1]; - - return input; - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != type) return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev") - throw Error("unsupported input attribute '%s'", name); - - auto input = create(); - input->owner = getStrAttr(attrs, "owner"); - input->repo = getStrAttr(attrs, "repo"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - -struct GitHubInput : GitArchiveInput +struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() const override { return "github"; } + std::string type() override { return "github"; } - std::shared_ptr<GitArchiveInput> _clone() const override - { return std::make_shared<GitHubInput>(*this); } - - Hash getRevFromRef(nix::ref<Store> store, std::string_view ref) const override + Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override { auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s", - owner, repo, ref); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( @@ -216,13 +184,14 @@ struct GitHubInput : GitArchiveInput return rev; } - std::string getDownloadUrl() const override + std::string getDownloadUrl(const Input & input) const override { // FIXME: use regular /archive URLs instead? api.github.com // might have stricter rate limits. auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - owner, repo, rev->to_string(Base16, false)); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); std::string accessToken = settings.githubAccessToken.get(); if (accessToken != "") @@ -231,35 +200,23 @@ struct GitHubInput : GitArchiveInput return url; } - void clone(const Path & destDir) const override - { - std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); - } -}; - -struct GitHubInputScheme : GitArchiveInputScheme -{ - GitHubInputScheme() : GitArchiveInputScheme("github") { } - - std::unique_ptr<GitArchiveInput> create() override + void clone(const Input & input, const Path & destDir) override { - return std::make_unique<GitHubInput>(); + Input::fromURL(fmt("git+ssh://git@github.com/%s/%s.git", + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("master"), input.getRev()) + .clone(destDir); } }; -struct GitLabInput : GitArchiveInput +struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() const override { return "gitlab"; } - - std::shared_ptr<GitArchiveInput> _clone() const override - { return std::make_shared<GitLabInput>(*this); } + std::string type() override { return "gitlab"; } - Hash getRevFromRef(nix::ref<Store> store, std::string_view ref) const override + Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override { auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/branches/%s", - owner, repo, ref); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); auto json = nlohmann::json::parse( readFile( store->toRealPath( @@ -269,12 +226,13 @@ struct GitLabInput : GitArchiveInput return rev; } - std::string getDownloadUrl() const override + std::string getDownloadUrl(const Input & input) const override { // FIXME: This endpoint has a rate limit threshold of 5 requests per minute. auto url = fmt("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", - owner, repo, rev->to_string(Base16, false)); + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + input.getRev()->to_string(Base16, false)); /* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`) std::string accessToken = settings.githubAccessToken.get(); @@ -284,21 +242,12 @@ struct GitLabInput : GitArchiveInput return url; } - void clone(const Path & destDir) const override - { - std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", owner, repo)); - input = input->applyOverrides(ref.value_or("master"), rev); - input->clone(destDir); - } -}; - -struct GitLabInputScheme : GitArchiveInputScheme -{ - GitLabInputScheme() : GitArchiveInputScheme("gitlab") { } - - std::unique_ptr<GitArchiveInput> create() override + void clone(const Input & input, const Path & destDir) override { - return std::make_unique<GitLabInput>(); + Input::fromURL(fmt("git+ssh://git@gitlab.com/%s/%s.git", + getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) + .applyOverrides(input.getRef().value_or("master"), input.getRev()) + .clone(destDir); } }; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 380b69fe0..91dc83740 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -4,135 +4,99 @@ namespace nix::fetchers { std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); -struct IndirectInput : Input -{ - std::string id; - std::optional<Hash> rev; - std::optional<std::string> ref; - - std::string type() const override { return "indirect"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const IndirectInput *>(&other); - return - other2 - && id == other2->id - && rev == other2->rev - && ref == other2->ref; - } - - bool isDirect() const override - { - return false; - } - - std::optional<std::string> getRef() const override { return ref; } - - std::optional<Hash> getRev() const override { return rev; } - - bool contains(const Input & other) const override - { - auto other2 = dynamic_cast<const IndirectInput *>(&other); - return - other2 - && id == other2->id - && (!ref || ref == other2->ref) - && (!rev || rev == other2->rev); - } - - ParsedURL toURL() const override - { - ParsedURL url; - url.scheme = "flake"; - url.path = id; - if (ref) { url.path += '/'; url.path += *ref; }; - if (rev) { url.path += '/'; url.path += rev->gitRev(); }; - return url; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("id", id); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; - } - - std::shared_ptr<const Input> applyOverrides( - std::optional<std::string> ref, - std::optional<Hash> rev) const override - { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared<IndirectInput>(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - - return res; - } - - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override - { - throw Error("indirect input '%s' cannot be fetched directly", to_string()); - } -}; - struct IndirectInputScheme : InputScheme { - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) override { - if (url.scheme != "flake") return nullptr; + if (url.scheme != "flake") return {}; auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); - auto input = std::make_unique<IndirectInput>(); + + std::optional<Hash> rev; + std::optional<std::string> ref; if (path.size() == 1) { } else if (path.size() == 2) { if (std::regex_match(path[1], revRegex)) - input->rev = Hash(path[1], htSHA1); + rev = Hash(path[1], htSHA1); else if (std::regex_match(path[1], refRegex)) - input->ref = path[1]; + ref = path[1]; else throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); } else if (path.size() == 3) { if (!std::regex_match(path[1], refRegex)) throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); - input->ref = path[1]; + ref = path[1]; if (!std::regex_match(path[2], revRegex)) throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); - input->rev = Hash(path[2], htSHA1); + rev = Hash(path[2], htSHA1); } else throw BadURL("GitHub URL '%s' is invalid", url.url); + std::string id = path[0]; + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + // FIXME: forbid query params? - input->id = path[0]; - if (!std::regex_match(input->id, flakeRegex)) - throw BadURL("'%s' is not a valid flake ID", input->id); + Input input; + input.direct = false; + input.attrs.insert_or_assign("type", "indirect"); + input.attrs.insert_or_assign("id", id); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); return input; } - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev") + if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") throw Error("unsupported indirect input attribute '%s'", name); - auto input = std::make_unique<IndirectInput>(); - input->id = getStrAttr(attrs, "id"); - input->ref = maybeGetStrAttr(attrs, "ref"); - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); + auto id = getStrAttr(attrs, "id"); + if (!std::regex_match(id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", id); + + Input input; + input.direct = false; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override + { + ParsedURL url; + url.scheme = "flake"; + url.path = getStrAttr(input.attrs, "id"); + if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; }; + if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); }; + return url; + } + + bool hasAllInfo(const Input & input) override + { + return false; + } + + Input applyOverrides( + const Input & _input, + std::optional<std::string> ref, + std::optional<Hash> rev) override + { + auto input(_input); + if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) input.attrs.insert_or_assign("ref", *ref); return input; } + + std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override + { + throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); }); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 5abb00172..49ed63243 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -10,80 +10,92 @@ using namespace std::string_literals; namespace nix::fetchers { -struct MercurialInput : Input +struct MercurialInputScheme : InputScheme { - ParsedURL url; - std::optional<std::string> ref; - std::optional<Hash> rev; + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "hg+http" && + url.scheme != "hg+https" && + url.scheme != "hg+ssh" && + url.scheme != "hg+file") return {}; + + auto url2(url); + url2.scheme = std::string(url2.scheme, 3); + url2.query.clear(); + + Attrs attrs; + attrs.emplace("type", "hg"); - MercurialInput(const ParsedURL & url) : url(url) - { } + for (auto &[name, value] : url.query) { + if (name == "rev" || name == "ref") + attrs.emplace(name, value); + else + url2.query.emplace(name, value); + } - std::string type() const override { return "hg"; } + attrs.emplace("url", url2.to_string()); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const MercurialInput *>(&other); - return - other2 - && url == other2->url - && rev == other2->rev - && ref == other2->ref; + return inputFromAttrs(attrs); } - bool isImmutable() const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - return (bool) rev || narHash; - } + if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash") + throw Error("unsupported Mercurial input attribute '%s'", name); - std::optional<std::string> getRef() const override { return ref; } + parseURL(getStrAttr(attrs, "url")); - std::optional<Hash> getRev() const override { return rev; } + if (auto ref = maybeGetStrAttr(attrs, "ref")) { + if (!std::regex_match(*ref, refRegex)) + throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); + } - ParsedURL toURL() const override + Input input; + input.attrs = attrs; + return input; + } + + ParsedURL toURL(const Input & input) override { - ParsedURL url2(url); - url2.scheme = "hg+" + url2.scheme; - if (rev) url2.query.insert_or_assign("rev", rev->gitRev()); - if (ref) url2.query.insert_or_assign("ref", *ref); + auto url = parseURL(getStrAttr(input.attrs, "url")); + url.scheme = "hg+" + url.scheme; + if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); + if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); return url; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (ref) - attrs.emplace("ref", *ref); - if (rev) - attrs.emplace("rev", rev->gitRev()); - return attrs; + // FIXME: ugly, need to distinguish between dirty and clean + // default trees. + return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); } - std::shared_ptr<const Input> applyOverrides( + Input applyOverrides( + const Input & input, std::optional<std::string> ref, - std::optional<Hash> rev) const override + std::optional<Hash> rev) override { - if (!ref && !rev) return shared_from_this(); - - auto res = std::make_shared<MercurialInput>(*this); - - if (ref) res->ref = ref; - if (rev) res->rev = rev; - + auto res(input); + if (rev) res.attrs.insert_or_assign("rev", rev->gitRev()); + if (ref) res.attrs.insert_or_assign("ref", *ref); return res; } - std::optional<Path> getSourcePath() const + std::optional<Path> getSourcePath(const Input & input) override { - if (url.scheme == "file" && !ref && !rev) + auto url = parseURL(getStrAttr(input.attrs, "url")); + if (url.scheme == "file" && !input.getRef() && !input.getRev()) return url.path; return {}; } - void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override { - auto sourcePath = getSourcePath(); + auto sourcePath = getSourcePath(input); assert(sourcePath); // FIXME: shut up if file is already tracked. @@ -95,26 +107,27 @@ struct MercurialInput : Input { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); } - std::pair<bool, std::string> getActualUrl() const + std::pair<bool, std::string> getActualUrl(const Input & input) const { + auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file"; return {isLocal, isLocal ? url.path : url.base}; } - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override { auto name = "source"; - auto input = std::make_shared<MercurialInput>(*this); + Input input(_input); - auto [isLocal, actualUrl_] = getActualUrl(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug // FIXME: return lastModified. // FIXME: don't clone local repositories. - if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) { + if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) { bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == ""; @@ -129,7 +142,7 @@ struct MercurialInput : Input if (settings.warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); - input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl })); + input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl }))); auto files = tokenizeString<std::set<std::string>>( runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); @@ -158,53 +171,50 @@ struct MercurialInput : Input } } - if (!input->ref) input->ref = "default"; + if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); auto getImmutableAttrs = [&]() { return Attrs({ {"type", "hg"}, {"name", name}, - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, }); }; auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair<Tree, std::shared_ptr<const Input>> + -> std::pair<Tree, Input> { - assert(input->rev); - assert(!rev || rev == input->rev); + assert(input.getRev()); + assert(!_input.getRev() || _input.getRev() == input.getRev()); + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); return { Tree{ .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath), - .info = TreeInfo { - .revCount = getIntAttr(infoAttrs, "revCount"), - }, }, input }; }; - if (input->rev) { + if (input.getRev()) { if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); } - assert(input->rev || input->ref); - auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref; + auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef(); Attrs mutableAttrs({ {"type", "hg"}, {"name", name}, {"url", actualUrl}, - {"ref", *input->ref}, + {"ref", *input.getRef()}, }); if (auto res = getCache()->lookup(store, mutableAttrs)) { auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1); - if (!rev || rev == rev2) { - input->rev = rev2; + if (!input.getRev() || input.getRev() == rev2) { + input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } @@ -213,10 +223,10 @@ struct MercurialInput : Input /* If this is a commit hash that we already have, we don't have to pull again. */ - if (!(input->rev + if (!(input.getRev() && pathExists(cacheDir) && runProgram( - RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" }) + RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" }) .killStderr(true)).second == "1")) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl)); @@ -245,9 +255,9 @@ struct MercurialInput : Input runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input->rev = Hash(tokens[0], htSHA1); + input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev()); auto revCount = std::stoull(tokens[1]); - input->ref = tokens[2]; + input.attrs.insert_or_assign("ref", tokens[2]); if (auto res = getCache()->lookup(store, getImmutableAttrs())) return makeResult(res->first, std::move(res->second)); @@ -255,18 +265,18 @@ struct MercurialInput : Input Path tmpDir = createTempDir(); AutoDelete delTmpDir(tmpDir, true); - runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir }); + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir }); deletePath(tmpDir + "/.hg_archival.txt"); auto storePath = store->addToStore(name, tmpDir); Attrs infoAttrs({ - {"rev", input->rev->gitRev()}, + {"rev", input.getRev()->gitRev()}, {"revCount", (int64_t) revCount}, }); - if (!this->rev) + if (!_input.getRev()) getCache()->add( store, mutableAttrs, @@ -285,54 +295,6 @@ struct MercurialInput : Input } }; -struct MercurialInputScheme : InputScheme -{ - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "hg+http" && - url.scheme != "hg+https" && - url.scheme != "hg+ssh" && - url.scheme != "hg+file") return nullptr; - - auto url2(url); - url2.scheme = std::string(url2.scheme, 3); - url2.query.clear(); - - Attrs attrs; - attrs.emplace("type", "hg"); - - for (auto &[name, value] : url.query) { - if (name == "rev" || name == "ref") - attrs.emplace(name, value); - else - url2.query.emplace(name, value); - } - - attrs.emplace("url", url2.to_string()); - - return inputFromAttrs(attrs); - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev") - throw Error("unsupported Mercurial input attribute '%s'", name); - - auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url"))); - if (auto ref = maybeGetStrAttr(attrs, "ref")) { - if (!std::regex_match(*ref, refRegex)) - throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); - input->ref = *ref; - } - if (auto rev = maybeGetStrAttr(attrs, "rev")) - input->rev = Hash(*rev, htSHA1); - return input; - } -}; - static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); }); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 77fe87d59..cbbb0fa02 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -3,76 +3,86 @@ namespace nix::fetchers { -struct PathInput : Input +struct PathInputScheme : InputScheme { - Path path; + std::optional<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "path") return {}; - /* Allow the user to pass in "fake" tree info attributes. This is - useful for making a pinned tree work the same as the repository - from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - std::optional<Hash> rev; - std::optional<uint64_t> revCount; - std::optional<time_t> lastModified; + if (url.authority && *url.authority != "") + throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - std::string type() const override { return "path"; } + Input input; + input.attrs.insert_or_assign("type", "path"); + input.attrs.insert_or_assign("path", url.path); - std::optional<Hash> getRev() const override { return rev; } + for (auto & [name, value] : url.query) + if (name == "rev" || name == "narHash") + input.attrs.insert_or_assign(name, value); + else if (name == "revCount" || name == "lastModified") { + uint64_t n; + if (!string2Int(value, n)) + throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + input.attrs.insert_or_assign(name, n); + } + else + throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const PathInput *>(&other); - return - other2 - && path == other2->path - && rev == other2->rev - && revCount == other2->revCount - && lastModified == other2->lastModified; + return input; } - bool isImmutable() const override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { - return narHash || rev; + if (maybeGetStrAttr(attrs, "type") != "path") return {}; + + getStrAttr(attrs, "path"); + + for (auto & [name, value] : attrs) + /* Allow the user to pass in "fake" tree info + attributes. This is useful for making a pinned tree + work the same as the repository from which is exported + (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ + if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") + // checked in Input::fromAttrs + ; + else + throw Error("unsupported path input attribute '%s'", name); + + Input input; + input.attrs = attrs; + return input; } - ParsedURL toURL() const override + ParsedURL toURL(const Input & input) override { - auto query = attrsToQuery(toAttrsInternal()); + auto query = attrsToQuery(input.attrs); query.erase("path"); + query.erase("type"); return ParsedURL { .scheme = "path", - .path = path, + .path = getStrAttr(input.attrs, "path"), .query = query, }; } - Attrs toAttrsInternal() const override + bool hasAllInfo(const Input & input) override { - Attrs attrs; - attrs.emplace("path", path); - if (rev) - attrs.emplace("rev", rev->gitRev()); - if (revCount) - attrs.emplace("revCount", *revCount); - if (lastModified) - attrs.emplace("lastModified", *lastModified); - if (!rev && narHash) - attrs.emplace("narHash", narHash->to_string(SRI)); - return attrs; + return true; } - std::optional<Path> getSourcePath() const override + std::optional<Path> getSourcePath(const Input & input) override { - return path; + return getStrAttr(input.attrs, "path"); } - void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override { + // nothing to do } - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override { - auto input = std::make_shared<PathInput>(*this); + auto path = getStrAttr(input.attrs, "path"); // FIXME: check whether access to 'path' is allowed. @@ -85,83 +95,13 @@ struct PathInput : Input // FIXME: try to substitute storePath. storePath = store->addToStore("source", path); - input->narHash = store->queryPathInfo(*storePath)->narHash; - - return - { - Tree { - .actualPath = store->toRealPath(*storePath), - .storePath = std::move(*storePath), - .info = TreeInfo { - .revCount = revCount, - .lastModified = lastModified - } - }, - input - }; - } - -}; - -struct PathInputScheme : InputScheme -{ - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override - { - if (url.scheme != "path") return nullptr; - - auto input = std::make_unique<PathInput>(); - input->path = url.path; - - if (url.authority && *url.authority != "") - throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - - for (auto & [name, value] : url.query) - if (name == "rev") - input->rev = Hash(value, htSHA1); - else if (name == "revCount") { - uint64_t revCount; - if (!string2Int(value, revCount)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->revCount = revCount; - } - else if (name == "lastModified") { - time_t lastModified; - if (!string2Int(value, lastModified)) - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input->lastModified = lastModified; - } - else if (name == "narHash") - // FIXME: require SRI hash. - input->narHash = Hash(value); - else - throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); - - return input; - } - - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override - { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - - auto input = std::make_unique<PathInput>(); - input->path = getStrAttr(attrs, "path"); - - for (auto & [name, value] : attrs) - if (name == "rev") - input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1); - else if (name == "revCount") - input->revCount = getIntAttr(attrs, "revCount"); - else if (name == "lastModified") - input->lastModified = getIntAttr(attrs, "lastModified"); - else if (name == "narHash") - // FIXME: require SRI hash. - input->narHash = Hash(getStrAttr(attrs, "narHash")); - else if (name == "type" || name == "path") - ; - else - throw Error("unsupported path input attribute '%s'", name); - - return input; + return { + Tree { + .actualPath = store->toRealPath(*storePath), + .storePath = std::move(*storePath), + }, + input + }; } }; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index f6760d2d0..914a0e1e8 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -22,20 +22,7 @@ std::shared_ptr<Registry> Registry::read( auto version = json.value("version", 0); - // FIXME: remove soon - if (version == 1) { - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) { - auto url = i->value("url", i->value("uri", "")); - if (url.empty()) - throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", - path, i.key()); - registry->entries.push_back( - {inputFromURL(i.key()), inputFromURL(url), {}}); - } - } - - else if (version == 2) { + if (version == 2) { for (auto & i : json["flakes"]) { auto toAttrs = jsonToAttrs(i["to"]); Attrs extraAttrs; @@ -47,8 +34,8 @@ std::shared_ptr<Registry> Registry::read( auto exact = i.find("exact"); registry->entries.push_back( Entry { - .from = inputFromAttrs(jsonToAttrs(i["from"])), - .to = inputFromAttrs(toAttrs), + .from = Input::fromAttrs(jsonToAttrs(i["from"])), + .to = Input::fromAttrs(std::move(toAttrs)), .extraAttrs = extraAttrs, .exact = exact != i.end() && exact.value() }); @@ -72,8 +59,8 @@ void Registry::write(const Path & path) nlohmann::json arr; for (auto & entry : entries) { nlohmann::json obj; - obj["from"] = attrsToJson(entry.from->toAttrs()); - obj["to"] = attrsToJson(entry.to->toAttrs()); + obj["from"] = attrsToJson(entry.from.toAttrs()); + obj["to"] = attrsToJson(entry.to.toAttrs()); if (!entry.extraAttrs.empty()) obj["to"].update(attrsToJson(entry.extraAttrs)); if (entry.exact) @@ -90,8 +77,8 @@ void Registry::write(const Path & path) } void Registry::add( - const std::shared_ptr<const Input> & from, - const std::shared_ptr<const Input> & to, + const Input & from, + const Input & to, const Attrs & extraAttrs) { entries.emplace_back( @@ -102,11 +89,11 @@ void Registry::add( }); } -void Registry::remove(const std::shared_ptr<const Input> & input) +void Registry::remove(const Input & input) { // FIXME: use C++20 std::erase. for (auto i = entries.begin(); i != entries.end(); ) - if (*i->from == *input) + if (i->from == input) i = entries.erase(i); else ++i; @@ -145,8 +132,8 @@ std::shared_ptr<Registry> getFlagRegistry() } void overrideRegistry( - const std::shared_ptr<const Input> & from, - const std::shared_ptr<const Input> & to, + const Input & from, + const Input & to, const Attrs & extraAttrs) { flagRegistry->add(from, to, extraAttrs); @@ -180,32 +167,33 @@ Registries getRegistries(ref<Store> store) return registries; } -std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries( +std::pair<Input, Attrs> lookupInRegistries( ref<Store> store, - std::shared_ptr<const Input> input) + const Input & _input) { Attrs extraAttrs; int n = 0; + Input input(_input); restart: n++; - if (n > 100) throw Error("cycle detected in flake registr for '%s'", input); + if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); for (auto & registry : getRegistries(store)) { // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { - if (*entry.from == *input) { + if (entry.from == input) { input = entry.to; extraAttrs = entry.extraAttrs; goto restart; } } else { - if (entry.from->contains(*input)) { - input = entry.to->applyOverrides( - !entry.from->getRef() && input->getRef() ? input->getRef() : std::optional<std::string>(), - !entry.from->getRev() && input->getRev() ? input->getRev() : std::optional<Hash>()); + if (entry.from.contains(input)) { + input = entry.to.applyOverrides( + !entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(), + !entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>()); extraAttrs = entry.extraAttrs; goto restart; } @@ -213,8 +201,8 @@ std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries( } } - if (!input->isDirect()) - throw Error("cannot find flake '%s' in the flake registries", input->to_string()); + if (!input.isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input.to_string()); return {input, extraAttrs}; } diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index c3ce948a8..1077af020 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -20,8 +20,7 @@ struct Registry struct Entry { - std::shared_ptr<const Input> from; - std::shared_ptr<const Input> to; + Input from, to; Attrs extraAttrs; bool exact = false; }; @@ -38,11 +37,11 @@ struct Registry void write(const Path & path); void add( - const std::shared_ptr<const Input> & from, - const std::shared_ptr<const Input> & to, + const Input & from, + const Input & to, const Attrs & extraAttrs); - void remove(const std::shared_ptr<const Input> & input); + void remove(const Input & input); }; typedef std::vector<std::shared_ptr<Registry>> Registries; @@ -54,12 +53,12 @@ Path getUserRegistryPath(); Registries getRegistries(ref<Store> store); void overrideRegistry( - const std::shared_ptr<const Input> & from, - const std::shared_ptr<const Input> & to, + const Input & from, + const Input & to, const Attrs & extraAttrs); -std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries( +std::pair<Input, Attrs> lookupInRegistries( ref<Store> store, - std::shared_ptr<const Input> input); + const Input & input); } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 695525b31..624f8b3fb 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -101,7 +101,7 @@ DownloadFileResult downloadFile( }; } -Tree downloadTarball( +std::pair<Tree, time_t> downloadTarball( ref<Store> store, const std::string & url, const std::string & name, @@ -116,12 +116,12 @@ Tree downloadTarball( auto cached = getCache()->lookupExpired(store, inAttrs); if (cached && !cached->expired) - return Tree { - .actualPath = store->toRealPath(cached->storePath), - .storePath = std::move(cached->storePath), - .info = TreeInfo { - .lastModified = getIntAttr(cached->infoAttrs, "lastModified"), + return { + Tree { + .actualPath = store->toRealPath(cached->storePath), + .storePath = std::move(cached->storePath), }, + getIntAttr(cached->infoAttrs, "lastModified") }; auto res = downloadFile(store, url, name, immutable); @@ -156,118 +156,75 @@ Tree downloadTarball( *unpackedStorePath, immutable); - return Tree { - .actualPath = store->toRealPath(*unpackedStorePath), - .storePath = std::move(*unpackedStorePath), - .info = TreeInfo { - .lastModified = lastModified, + return { + Tree { + .actualPath = store->toRealPath(*unpackedStorePath), + .storePath = std::move(*unpackedStorePath), }, + lastModified, }; } -struct TarballInput : Input -{ - ParsedURL url; - std::optional<Hash> hash; - - TarballInput(const ParsedURL & url) : url(url) - { } - - std::string type() const override { return "tarball"; } - - bool operator ==(const Input & other) const override - { - auto other2 = dynamic_cast<const TarballInput *>(&other); - return - other2 - && to_string() == other2->to_string() - && hash == other2->hash; - } - - bool isImmutable() const override - { - return hash || narHash; - } - - ParsedURL toURL() const override - { - auto url2(url); - // NAR hashes are preferred over file hashes since tar/zip files - // don't have a canonical representation. - if (narHash) - url2.query.insert_or_assign("narHash", narHash->to_string(SRI)); - else if (hash) - url2.query.insert_or_assign("hash", hash->to_string(SRI)); - return url2; - } - - Attrs toAttrsInternal() const override - { - Attrs attrs; - attrs.emplace("url", url.to_string()); - if (hash) - attrs.emplace("hash", hash->to_string(SRI)); - return attrs; - } - - std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override - { - auto tree = downloadTarball(store, url.to_string(), "source", false); - - auto input = std::make_shared<TarballInput>(*this); - input->narHash = store->queryPathInfo(tree.storePath)->narHash; - - return {std::move(tree), input}; - } -}; - struct TarballInputScheme : InputScheme { - std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + std::optional<Input> inputFromURL(const ParsedURL & url) override { - if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr; + if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {}; if (!hasSuffix(url.path, ".zip") && !hasSuffix(url.path, ".tar") && !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.xz") && !hasSuffix(url.path, ".tar.bz2")) - return nullptr; - - auto input = std::make_unique<TarballInput>(url); - - auto hash = input->url.query.find("hash"); - if (hash != input->url.query.end()) { - // FIXME: require SRI hash. - input->hash = Hash(hash->second); - input->url.query.erase(hash); - } - - auto narHash = input->url.query.find("narHash"); - if (narHash != input->url.query.end()) { - // FIXME: require SRI hash. - input->narHash = Hash(narHash->second); - input->url.query.erase(narHash); - } - + return {}; + + Input input; + input.attrs.insert_or_assign("type", "tarball"); + input.attrs.insert_or_assign("url", url.to_string()); + auto narHash = url.query.find("narHash"); + if (narHash != url.query.end()) + input.attrs.insert_or_assign("narHash", narHash->second); return input; } - std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + std::optional<Input> inputFromAttrs(const Attrs & attrs) override { if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "hash") + if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash") throw Error("unsupported tarball input attribute '%s'", name); - auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url"))); - if (auto hash = maybeGetStrAttr(attrs, "hash")) - // FIXME: require SRI hash. - input->hash = Hash(*hash); - + Input input; + input.attrs = attrs; + //input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash"); return input; } + + ParsedURL toURL(const Input & input) override + { + auto url = parseURL(getStrAttr(input.attrs, "url")); + // NAR hashes are preferred over file hashes since tar/zip files + // don't have a canonical representation. + if (auto narHash = input.getNarHash()) + url.query.insert_or_assign("narHash", narHash->to_string(SRI)); + /* + else if (auto hash = maybeGetStrAttr(input.attrs, "hash")) + url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI)); + */ + return url; + } + + bool hasAllInfo(const Input & input) override + { + return true; + } + + std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override + { + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + return {std::move(tree), input}; + } }; static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); }); diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc deleted file mode 100644 index 42a19cbc8..000000000 --- a/src/libfetchers/tree-info.cc +++ /dev/null @@ -1,60 +0,0 @@ -#include "tree-info.hh" -#include "store-api.hh" - -#include <nlohmann/json.hpp> - -namespace nix::fetchers { - -StorePath TreeInfo::computeStorePath(Store & store) const -{ - assert(narHash); - return store.makeFixedOutputPath(true, narHash, "source"); -} - -TreeInfo TreeInfo::fromJson(const nlohmann::json & json) -{ - TreeInfo info; - - auto i = json.find("info"); - if (i != json.end()) { - const nlohmann::json & i2(*i); - - auto j = i2.find("narHash"); - if (j != i2.end()) - info.narHash = Hash((std::string) *j); - else - throw Error("attribute 'narHash' missing in lock file"); - - j = i2.find("revCount"); - if (j != i2.end()) - info.revCount = *j; - - j = i2.find("lastModified"); - if (j != i2.end()) - info.lastModified = *j; - - return info; - } - - i = json.find("narHash"); - if (i != json.end()) { - info.narHash = Hash((std::string) *i); - return info; - } - - throw Error("attribute 'info' missing in lock file"); -} - -nlohmann::json TreeInfo::toJson() const -{ - nlohmann::json json; - assert(narHash); - json["narHash"] = narHash.to_string(SRI); - if (revCount) - json["revCount"] = *revCount; - if (lastModified) - json["lastModified"] = *lastModified; - return json; -} - -} diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh deleted file mode 100644 index 3b62151c6..000000000 --- a/src/libfetchers/tree-info.hh +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include "path.hh" -#include "hash.hh" - -#include <nlohmann/json_fwd.hpp> - -namespace nix { class Store; } - -namespace nix::fetchers { - -struct TreeInfo -{ - Hash narHash; - std::optional<uint64_t> revCount; - std::optional<time_t> lastModified; - - bool operator ==(const TreeInfo & other) const - { - return - narHash == other.narHash - && revCount == other.revCount - && lastModified == other.lastModified; - } - - StorePath computeStorePath(Store & store) const; - - static TreeInfo fromJson(const nlohmann::json & json); - - nlohmann::json toJson() const; -}; - -} diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 59bb40110..15a3e261a 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -62,13 +62,13 @@ static void printFlakeInfo(const Store & store, const Flake & flake) if (flake.description) logger->stdout("Description: %s", *flake.description); logger->stdout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); - if (auto rev = flake.lockedRef.input->getRev()) + if (auto rev = flake.lockedRef.input.getRev()) logger->stdout("Revision: %s", rev->to_string(Base16, false)); - if (flake.sourceInfo->info.revCount) - logger->stdout("Revisions: %s", *flake.sourceInfo->info.revCount); - if (flake.sourceInfo->info.lastModified) + if (auto revCount = flake.lockedRef.input.getRevCount()) + logger->stdout("Revisions: %s", *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) logger->stdout("Last modified: %s", - std::put_time(std::localtime(&*flake.sourceInfo->info.lastModified), "%F %T")); + std::put_time(std::localtime(&*lastModified), "%F %T")); } static nlohmann::json flakeToJson(const Store & store, const Flake & flake) @@ -82,13 +82,12 @@ static nlohmann::json flakeToJson(const Store & store, const Flake & flake) j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs()); j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl j["locked"] = attrsToJson(flake.lockedRef.toAttrs()); - j["info"] = flake.sourceInfo->info.toJson(); - if (auto rev = flake.lockedRef.input->getRev()) + if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(Base16, false); - if (flake.sourceInfo->info.revCount) - j["revCount"] = *flake.sourceInfo->info.revCount; - if (flake.sourceInfo->info.lastModified) - j["lastModified"] = *flake.sourceInfo->info.lastModified; + if (auto revCount = flake.lockedRef.input.getRevCount()) + j["revCount"] = *revCount; + if (auto lastModified = flake.lockedRef.input.getLastModified()) + j["lastModified"] = *lastModified; j["path"] = store.printStorePath(flake.sourceInfo->storePath); return j; } @@ -172,7 +171,7 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", prefix + (last ? treeLast : treeConn), input.first, lockedNode ? lockedNode->lockedRef : flake.flake.lockedRef); - + if (firstVisit) recurse(*input.second, prefix + (last ? treeNull : treeLine)); } }; @@ -501,7 +500,7 @@ struct CmdFlakeClone : FlakeCommand if (destDir.empty()) throw Error("missing flag '--dest'"); - getFlakeRef().resolve(store).input->clone(destDir); + getFlakeRef().resolve(store).input.clone(destDir); } }; @@ -563,9 +562,10 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun auto lockedInput = std::dynamic_pointer_cast<const LockedNode>(input.second); assert(lockedInput); auto jsonObj3 = jsonObj2 ? jsonObj2->object(input.first) : std::optional<JSONObject>(); - if (!dryRun) - lockedInput->lockedRef.input->fetchTree(store); - auto storePath = lockedInput->computeStorePath(*store); + auto storePath = + dryRun + ? lockedInput->lockedRef.input.computeStorePath(*store) + : lockedInput->lockedRef.input.fetch(store).first.storePath; if (jsonObj3) jsonObj3->attr("path", store->printStorePath(storePath)); sources.insert(std::move(storePath)); diff --git a/src/nix/installables.cc b/src/nix/installables.cc index fde1ca7aa..86d3bfd20 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -194,7 +194,7 @@ void EvalCommand::completeFlakeRef(std::string_view prefix) /* Look for registry entries that match the prefix. */ for (auto & registry : fetchers::getRegistries(getStore())) { for (auto & entry : registry->entries) { - auto from = entry.from->to_string(); + auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index cc239052d..f39213b8f 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -330,7 +330,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < manifest.elements.size(); ++i) { auto & element(manifest.elements[i]); if (element.source - && !element.source->originalRef.input->isImmutable() + && !element.source->originalRef.input.isImmutable() && matches(*store, element, i, matchers)) { Activity act(*logger, lvlChatty, actUnknown, diff --git a/src/nix/registry.cc b/src/nix/registry.cc index e9c76e6c6..16d7e511f 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -31,8 +31,8 @@ struct CmdRegistryList : StoreCommand registry->type == Registry::User ? "user " : registry->type == Registry::System ? "system" : "global", - entry.from->to_string(), - entry.to->to_string()); + entry.from.to_string(), + entry.to.to_string()); } } } @@ -107,7 +107,7 @@ struct CmdRegistryPin : virtual Args, EvalCommand auto ref = parseFlakeRef(url); auto userRegistry = fetchers::getUserRegistry(); userRegistry->remove(ref.input); - auto [tree, resolved] = ref.resolve(store).input->fetchTree(store); + auto [tree, resolved] = ref.resolve(store).input.fetch(store); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; userRegistry->add(ref.input, resolved, extraAttrs); |