From 08133503494d023b646b3107acf159a5274466ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Jan 2021 21:51:46 +0100 Subject: Add 'nix store prefetch-{file,tarball}' These replace nix-prefetch-url and nix-prefetch-url --unpack, respectively. --- src/nix/prefetch.cc | 352 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 352 insertions(+) create mode 100644 src/nix/prefetch.cc (limited to 'src/nix/prefetch.cc') diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc new file mode 100644 index 000000000..969299489 --- /dev/null +++ b/src/nix/prefetch.cc @@ -0,0 +1,352 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "filetransfer.hh" +#include "finally.hh" +#include "progress-bar.hh" +#include "tarfile.hh" +#include "attr-path.hh" +#include "eval-inline.hh" +#include "legacy.hh" + +#include + +using namespace nix; + +/* If ‘url’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +string resolveMirrorUrl(EvalState & state, string url) +{ + if (url.substr(0, 9) != "mirror://") return url; + + std::string s(url, 9); + auto p = s.find('/'); + if (p == std::string::npos) throw Error("invalid mirror URL '%s'", url); + std::string mirrorName(s, 0, p); + + Value vMirrors; + // FIXME: use nixpkgs flake + state.eval(state.parseExprFromString("import ", "."), vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error("unknown mirror name '%s'", mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error("mirror URL '%s' did not expand to anything", url); + + auto mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} + +std::tuple prefetchFile( + ref store, + std::string_view url, + std::optional name, + HashType hashType, + std::optional expectedHash, + bool unpack, + bool executable) +{ + auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + + /* Figure out a name in the Nix store. */ + if (!name) { + name = baseNameOf(url); + if (name->empty()) + throw Error("cannot figure out file name for '%s'", url); + } + + std::optional storePath; + std::optional hash; + + /* If an expected hash is given, the file may already exist in + the store. */ + if (expectedHash) { + hashType = expectedHash->type; + storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + if (store->isValidPath(*storePath)) + hash = expectedHash; + else + storePath.reset(); + } + + if (!storePath) { + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/tmp"; + + /* Download the file. */ + { + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + FileTransferRequest req(url); + req.decompress = false; + getFileTransfer()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("unpacking '%s'", url)); + Path unpacked = (Path) tmpDir + "/unpacked"; + createDirs(unpacked); + unpackTarfile(tmpFile, unpacked); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + Activity act(*logger, lvlChatty, actUnknown, + fmt("adding '%s' to the store", url)); + + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); + } + + return {storePath.value(), hash.value()}; +} + +static int main_nix_prefetch_url(int argc, char * * argv) +{ + { + HashType ht = htSHA256; + std::vector args; + bool printPath = getEnv("PRINT_PATH") == "1"; + bool fromExpr = false; + string attrPath; + bool unpack = false; + bool executable = false; + std::optional name; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + } + else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } + else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--executable") + executable = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (args.size() > 2) + throw UsageError("too many arguments"); + + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + + auto store = openStore(); + auto state = std::make_unique(myArgs.searchPath, store); + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + /* If -A is given, get the URL from the specified Nix + expression. */ + string url; + if (!fromExpr) { + if (args.empty()) + throw UsageError("you must specify a URL"); + url = args[0]; + } else { + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); + state->forceAttrs(v); + + /* Extract the URL. */ + auto attr = v.attrs->find(state->symbols.create("urls")); + if (attr == v.attrs->end()) + throw Error("attribute set does not contain a 'urls' attribute"); + state->forceList(*attr->value); + if (attr->value->listSize() < 1) + throw Error("'urls' list is empty"); + url = state->forceString(*attr->value->listElems()[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state->symbols.create("outputHashMode")); + if (attr == v.attrs->end()) + printInfo("warning: this does not look like a fetchurl call"); + else + unpack = state->forceString(*attr->value) == "recursive"; + + /* Extract the name. */ + if (!name) { + attr = v.attrs->find(state->symbols.create("name")); + if (attr != v.attrs->end()) + name = state->forceString(*attr->value); + } + } + + std::optional expectedHash; + if (args.size() == 2) + expectedHash = Hash::parseAny(args[1], ht); + + auto [storePath, hash] = prefetchFile( + store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + + stopProgressBar(); + + if (!printPath) + printInfo("path is '%s'", store->printStorePath(storePath)); + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) + std::cout << store->printStorePath(storePath) << std::endl; + + return 0; + } +} + +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); + +struct CmdStorePrefetch : StoreCommand, MixJSON +{ + std::string url; + bool executable = false; + bool unpack; + std::optional name; + HashType hashType = htSHA256; + std::optional expectedHash; + + CmdStorePrefetch(bool unpack) + : unpack(unpack) + { + addFlag({ + .longName = "name", + .description = "store path name", + .labels = {"name"}, + .handler = {&name} + }); + + addFlag({ + .longName = "expected-hash", + .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .labels = {"hash"}, + .handler = {[&](std::string s) { + expectedHash = Hash::parseAny(s, hashType); + }} + }); + + addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + + expectArg("url", &url); + } + + Category category() override { return catUtility; } + + void run(ref store) override + { + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + url, + store->printStorePath(storePath), + hash.to_string(SRI, true)); + } + } +}; + +struct CmdStorePrefetchFile : CmdStorePrefetch +{ + CmdStorePrefetchFile() + : CmdStorePrefetch(false) + { + name = "source"; + + addFlag({ + .longName = "executable", + .description = "make the resulting file executable", + .handler = {&executable, true}, + }); + } + + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } +}; + +static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); + +struct CmdStorePrefetchTarball : CmdStorePrefetch +{ + CmdStorePrefetchTarball() + : CmdStorePrefetch(true) + { + name = "source"; + } + + std::string description() override + { + return "download and unpack a tarball into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-tarball.md" + ; + } +}; + +static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); -- cgit v1.2.3 From 93ad6430edf3d7efa5948d1e0ca0447e4666b121 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Jan 2021 12:36:39 +0100 Subject: nix store prefetch-tarball -> nix flake prefetch --- src/nix/prefetch.cc | 77 +++++++++++++++-------------------------------------- 1 file changed, 21 insertions(+), 56 deletions(-) (limited to 'src/nix/prefetch.cc') diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 969299489..ce8c85ecf 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -246,17 +246,15 @@ static int main_nix_prefetch_url(int argc, char * * argv) static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); -struct CmdStorePrefetch : StoreCommand, MixJSON +struct CmdStorePrefetchFile : StoreCommand, MixJSON { std::string url; bool executable = false; - bool unpack; std::optional name; HashType hashType = htSHA256; std::optional expectedHash; - CmdStorePrefetch(bool unpack) - : unpack(unpack) + CmdStorePrefetchFile() { addFlag({ .longName = "name", @@ -267,7 +265,7 @@ struct CmdStorePrefetch : StoreCommand, MixJSON addFlag({ .longName = "expected-hash", - .description = unpack ? "expected NAR hash of the unpacked tarball" : "expected hash of the file", + .description = "expected hash of the file", .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashType); @@ -276,43 +274,17 @@ struct CmdStorePrefetch : StoreCommand, MixJSON addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); - expectArg("url", &url); - } - - Category category() override { return catUtility; } - - void run(ref store) override - { - auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, unpack, executable); - - if (json) { - auto res = nlohmann::json::object(); - res["storePath"] = store->printStorePath(storePath); - res["hash"] = hash.to_string(SRI, true); - logger->cout(res.dump()); - } else { - notice("Downloaded '%s' to '%s' (hash '%s').", - url, - store->printStorePath(storePath), - hash.to_string(SRI, true)); - } - } -}; - -struct CmdStorePrefetchFile : CmdStorePrefetch -{ - CmdStorePrefetchFile() - : CmdStorePrefetch(false) - { - name = "source"; - addFlag({ .longName = "executable", .description = "make the resulting file executable", .handler = {&executable, true}, }); + + expectArg("url", &url); } + Category category() override { return catUtility; } + std::string description() override { return "download a file into the Nix store"; @@ -324,29 +296,22 @@ struct CmdStorePrefetchFile : CmdStorePrefetch #include "store-prefetch-file.md" ; } -}; - -static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); - -struct CmdStorePrefetchTarball : CmdStorePrefetch -{ - CmdStorePrefetchTarball() - : CmdStorePrefetch(true) - { - name = "source"; - } - - std::string description() override + void run(ref store) override { - return "download and unpack a tarball into the Nix store"; - } + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); - std::string doc() override - { - return - #include "store-prefetch-tarball.md" - ; + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + url, + store->printStorePath(storePath), + hash.to_string(SRI, true)); + } } }; -static auto rCmdStorePrefetchTarball = registerCommand2({"store", "prefetch-tarball"}); +static auto rCmdStorePrefetchFile = registerCommand2({"store", "prefetch-file"}); -- cgit v1.2.3 From 3da9a9241cb9f8c284426c220ea285398d0328dd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Jan 2021 14:18:04 +0100 Subject: Convert option descriptions to Markdown --- src/nix/prefetch.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src/nix/prefetch.cc') diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index ce8c85ecf..a831dcd15 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -258,14 +258,14 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON { addFlag({ .longName = "name", - .description = "store path name", + .description = "Override the name component of the resulting store path. It defaults to the base name of *url*.", .labels = {"name"}, .handler = {&name} }); addFlag({ .longName = "expected-hash", - .description = "expected hash of the file", + .description = "The expected hash of the file.", .labels = {"hash"}, .handler = {[&](std::string s) { expectedHash = Hash::parseAny(s, hashType); @@ -276,7 +276,9 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON addFlag({ .longName = "executable", - .description = "make the resulting file executable", + .description = + "Make the resulting file executable. Note that this causes the " + "resulting hash to be a NAR hash rather than a flat file hash.", .handler = {&executable, true}, }); -- cgit v1.2.3 From 98d1b64400cc7b75216fc885859883c707c18bef Mon Sep 17 00:00:00 2001 From: Shea Levy Date: Thu, 28 Jan 2021 09:37:43 -0500 Subject: Initialize plugins after handling initial command line flags This is technically a breaking change, since attempting to set plugin files after the first non-flag argument will now throw an error. This is acceptable given the relative lack of stability in a plugin interface and the need to tie the knot somewhere once plugins can actually define new subcommands. --- src/nix/prefetch.cc | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/nix/prefetch.cc') diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index a831dcd15..b7da3ea5a 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -171,8 +171,6 @@ static int main_nix_prefetch_url(int argc, char * * argv) myArgs.parseCmdline(argvToStrings(argc, argv)); - initPlugins(); - if (args.size() > 2) throw UsageError("too many arguments"); -- cgit v1.2.3 From 96c62fb66c55cde1246f20768452fac69bf24131 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 22 Jul 2021 10:11:04 +0200 Subject: Fix formatting error in 'nix store' manpage --- src/nix/prefetch.cc | 2 -- 1 file changed, 2 deletions(-) (limited to 'src/nix/prefetch.cc') diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b7da3ea5a..9c2309a5f 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -283,8 +283,6 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON expectArg("url", &url); } - Category category() override { return catUtility; } - std::string description() override { return "download a file into the Nix store"; -- cgit v1.2.3 From eadb45c4db1286267923827cc5c6d839c4d0f79b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Sep 2021 13:53:24 +0200 Subject: Use Bindings::{get,need} instead of find --- src/nix/prefetch.cc | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) (limited to 'src/nix/prefetch.cc') diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 9c2309a5f..768d37595 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -199,26 +199,24 @@ static int main_nix_prefetch_url(int argc, char * * argv) state->forceAttrs(v); /* Extract the URL. */ - auto attr = v.attrs->find(state->symbols.create("urls")); - if (attr == v.attrs->end()) - throw Error("attribute set does not contain a 'urls' attribute"); - state->forceList(*attr->value); - if (attr->value->listSize() < 1) + auto & attr = v.attrs->need(state->symbols.create("urls")); + state->forceList(*attr.value); + if (attr.value->listSize() < 1) throw Error("'urls' list is empty"); - url = state->forceString(*attr->value->listElems()[0]); + url = state->forceString(*attr.value->listElems()[0]); /* Extract the hash mode. */ - attr = v.attrs->find(state->symbols.create("outputHashMode")); - if (attr == v.attrs->end()) + auto attr2 = v.attrs->get(state->symbols.create("outputHashMode")); + if (!attr2) printInfo("warning: this does not look like a fetchurl call"); else - unpack = state->forceString(*attr->value) == "recursive"; + unpack = state->forceString(*attr2->value) == "recursive"; /* Extract the name. */ if (!name) { - attr = v.attrs->find(state->symbols.create("name")); - if (attr != v.attrs->end()) - name = state->forceString(*attr->value); + auto attr3 = v.attrs->get(state->symbols.create("name")); + if (!attr3) + name = state->forceString(*attr3->value); } } -- cgit v1.2.3