diff options
Diffstat (limited to 'src/nix')
115 files changed, 8488 insertions, 1751 deletions
diff --git a/src/nix/add-file.md b/src/nix/add-file.md new file mode 100644 index 000000000..ed237a035 --- /dev/null +++ b/src/nix/add-file.md @@ -0,0 +1,28 @@ +R""( + +# Description + +Copy the regular file *path* to the Nix store, and print the resulting +store path on standard output. + +> **Warning** +> +> The resulting store path is not registered as a garbage +> collector root, so it could be deleted before you have a +> chance to register it. + +# Examples + +Add a regular file to the store: + +```console +# echo foo > bar + +# nix store add-file ./bar +/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar + +# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar +foo +``` + +)"" diff --git a/src/nix/add-path.md b/src/nix/add-path.md new file mode 100644 index 000000000..87473611d --- /dev/null +++ b/src/nix/add-path.md @@ -0,0 +1,29 @@ +R""( + +# Description + +Copy *path* to the Nix store, and print the resulting store path on +standard output. + +> **Warning** +> +> The resulting store path is not registered as a garbage +> collector root, so it could be deleted before you have a +> chance to register it. + +# Examples + +Add a directory to the store: + +```console +# mkdir dir +# echo foo > dir/bar + +# nix store add-path ./dir +/nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir + +# cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar +foo +``` + +)"" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f9d6de16e..2ae042789 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -9,33 +9,22 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional<std::string> namePart; + FileIngestionMethod ingestionMethod; CmdAddToStore() { + // FIXME: completion expectArg("path", &path); addFlag({ .longName = "name", .shortName = 'n', - .description = "name component of the store path", + .description = "Override the name component of the store path. It defaults to the base name of *path*.", .labels = {"name"}, .handler = {&namePart}, }); } - std::string description() override - { - return "add a path to the Nix store"; - } - - Examples examples() override - { - return { - }; - } - - Category category() override { return catUtility; } - void run(ref<Store> store) override { if (!namePart) namePart = baseNameOf(path); @@ -45,12 +34,21 @@ struct CmdAddToStore : MixDryRun, StoreCommand auto narHash = hashString(htSHA256, *sink.s); - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, *namePart)); - info.narHash = narHash; + Hash hash = narHash; + if (ingestionMethod == FileIngestionMethod::Flat) { + HashSink hsink(htSHA256); + readFile(path, hsink); + hash = hsink.finish().first; + } + + ValidPathInfo info { + store->makeFixedOutputPath(ingestionMethod, hash, *namePart), + narHash, + }; info.narSize = sink.s->size(); info.ca = std::optional { FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, + .method = ingestionMethod, + .hash = hash, } }; if (!dryRun) { @@ -58,8 +56,49 @@ struct CmdAddToStore : MixDryRun, StoreCommand store->addToStore(info, source); } - logger->stdout("%s", store->printStorePath(info.path)); + logger->cout("%s", store->printStorePath(info.path)); + } +}; + +struct CmdAddFile : CmdAddToStore +{ + CmdAddFile() + { + ingestionMethod = FileIngestionMethod::Flat; + } + + std::string description() override + { + return "add a regular file to the Nix store"; + } + + std::string doc() override + { + return + #include "add-file.md" + ; + } +}; + +struct CmdAddPath : CmdAddToStore +{ + CmdAddPath() + { + ingestionMethod = FileIngestionMethod::Recursive; + } + + std::string description() override + { + return "add a path to the Nix store"; + } + + std::string doc() override + { + return + #include "add-path.md" + ; } }; -static auto r1 = registerCommand<CmdAddToStore>("add-to-store"); +static auto rCmdAddFile = registerCommand2<CmdAddFile>({"store", "add-file"}); +static auto rCmdAddPath = registerCommand2<CmdAddPath>({"store", "add-path"}); diff --git a/src/nix/app.cc b/src/nix/app.cc new file mode 100644 index 000000000..9719a65dd --- /dev/null +++ b/src/nix/app.cc @@ -0,0 +1,122 @@ +#include "installables.hh" +#include "store-api.hh" +#include "eval-inline.hh" +#include "eval-cache.hh" +#include "names.hh" +#include "command.hh" + +namespace nix { + +struct InstallableDerivedPath : Installable +{ + ref<Store> store; + const DerivedPath derivedPath; + + InstallableDerivedPath(ref<Store> store, const DerivedPath & derivedPath) + : store(store) + , derivedPath(derivedPath) + { + } + + + std::string what() override { return derivedPath.to_string(*store); } + + DerivedPaths toDerivedPaths() override + { + return {derivedPath}; + } + + std::optional<StorePath> getStorePath() override + { + return std::nullopt; + } +}; + +/** + * Return the rewrites that are needed to resolve a string whose context is + * included in `dependencies` + */ +StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) +{ + StringPairs res; + for (auto & dep : dependencies) + if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep)) + for (auto & [ outputName, outputPath ] : drvDep->outputs) + res.emplace( + downstreamPlaceholder(store, drvDep->drvPath, outputName), + store.printStorePath(outputPath) + ); + return res; +} + +/** + * Resolve the given string assuming the given context + */ +std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies) +{ + auto rewrites = resolveRewrites(store, dependencies); + return rewriteStrings(toResolve, rewrites); +} + +UnresolvedApp Installable::toApp(EvalState & state) +{ + auto [cursor, attrPath] = getCursor(state); + + auto type = cursor->getAttr("type")->getString(); + + if (type == "app") { + auto [program, context] = cursor->getAttr("program")->getStringWithContext(); + + + std::vector<StorePathWithOutputs> context2; + for (auto & [path, name] : context) + context2.push_back({state.store->parseStorePath(path), {name}}); + + return UnresolvedApp{App { + .context = std::move(context2), + .program = program, + }}; + } + + else if (type == "derivation") { + auto drvPath = cursor->forceDerivation(); + auto outPath = cursor->getAttr(state.sOutPath)->getString(); + auto outputName = cursor->getAttr(state.sOutputName)->getString(); + auto name = cursor->getAttr(state.sName)->getString(); + auto aMeta = cursor->maybeGetAttr("meta"); + auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr; + auto mainProgram = + aMainProgram + ? aMainProgram->getString() + : DrvName(name).name; + auto program = outPath + "/bin/" + mainProgram; + return UnresolvedApp { App { + .context = { { drvPath, {outputName} } }, + .program = program, + }}; + } + + else + throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); +} + +// FIXME: move to libcmd +App UnresolvedApp::resolve(ref<Store> evalStore, ref<Store> store) +{ + auto res = unresolved; + + std::vector<std::shared_ptr<Installable>> installableContext; + + for (auto & ctxElt : unresolved.context) + installableContext.push_back( + std::make_shared<InstallableDerivedPath>(store, ctxElt.toDerivedPath())); + + auto builtContext = build(evalStore, store, Realise::Outputs, installableContext); + res.program = resolveString(*store, unresolved.program, builtContext); + if (!store->isInStore(res.program)) + throw Error("app program '%s' is not in the Nix store", res.program); + + return res; +} + +} diff --git a/src/nix/build.cc b/src/nix/build.cc index 850e09ce8..ce6df7df8 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,29 +1,41 @@ +#include "eval.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" + +#include <nlohmann/json.hpp> using namespace nix; -struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile +struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile { Path outLink = "result"; + BuildMode buildMode = bmNormal; CmdBuild() { addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.", .labels = {"path"}, .handler = {&outLink}, + .completer = completePath }); addFlag({ .longName = "no-link", - .description = "do not create a symlink to the build result", + .description = "Do not create symlinks to the build results.", .handler = {&outLink, Path("")}, }); + + addFlag({ + .longName = "rebuild", + .description = "Rebuild an already built package and compare the result to the existing store paths.", + .handler = {&buildMode, bmCheck}, + }); } std::string description() override @@ -31,44 +43,47 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile return "build a derivation or fetch a store path"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To build and run GNU Hello from NixOS 17.03:", - "nix build -f channel:nixos-17.03 hello; ./result/bin/hello" - }, - Example{ - "To build the build.x86_64-linux attribute from release.nix:", - "nix build -f release.nix build.x86_64-linux" - }, - Example{ - "To make a profile point at GNU Hello:", - "nix build --profile /tmp/profile nixpkgs.hello" - }, - }; + return + #include "build.md" + ; } void run(ref<Store> store) override { - auto buildables = build(store, dryRun ? DryRun : Build, installables); + auto buildables = build( + getEvalStore(), store, + dryRun ? Realise::Derivation : Realise::Outputs, + installables, buildMode); + + if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); if (dryRun) return; - if (outLink != "") { - for (size_t i = 0; i < buildables.size(); ++i) { - for (auto & output : buildables[i].outputs) - if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { - std::string symlink = outLink; - if (i) symlink += fmt("-%d", i); - if (output.first != "out") symlink += fmt("-%s", output.first); - store2->addPermRoot(output.second, absPath(symlink), true); - } - } - } + if (outLink != "") + if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) + for (const auto & [_i, buildable] : enumerate(buildables)) { + auto i = _i; + std::visit(overloaded { + [&](BuiltPath::Opaque bo) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + store2->addPermRoot(bo.path, absPath(symlink)); + }, + [&](BuiltPath::Built bfd) { + for (auto & output : bfd.outputs) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store2->addPermRoot(output.second, absPath(symlink)); + } + }, + }, buildable.raw()); + } updateProfile(buildables); } }; -static auto r1 = registerCommand<CmdBuild>("build"); +static auto rCmdBuild = registerCommand<CmdBuild>("build"); diff --git a/src/nix/build.md b/src/nix/build.md new file mode 100644 index 000000000..20138b7e0 --- /dev/null +++ b/src/nix/build.md @@ -0,0 +1,92 @@ +R""( + +# Examples + +* Build the default package from the flake in the current directory: + + ```console + # nix build + ``` + +* Build and run GNU Hello from the `nixpkgs` flake: + + ```console + # nix build nixpkgs#hello + # ./result/bin/hello + Hello, world! + ``` + +* Build GNU Hello and Cowsay, leaving two result symlinks: + + ```console + # nix build nixpkgs#hello nixpkgs#cowsay + # ls -l result* + lrwxrwxrwx 1 … result -> /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + lrwxrwxrwx 1 … result-1 -> /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2 + ``` + +* Build a specific output: + + ```console + # nix build nixpkgs#glibc.dev + # ls -ld ./result-dev + lrwxrwxrwx 1 … ./result-dev -> /nix/store/dkm3gwl0xrx0wrw6zi5x3px3lpgjhlw4-glibc-2.32-dev + ``` + +* Build attribute `build.x86_64-linux` from (non-flake) Nix expression + `release.nix`: + + ```console + # nix build -f release.nix build.x86_64-linux + ``` + +* Build a NixOS system configuration from a flake, and make a profile + point to the result: + + ```console + # nix build --profile /nix/var/nix/profiles/system \ + ~/my-configurations#nixosConfigurations.machine.config.system.build.toplevel + ``` + + (This is essentially what `nixos-rebuild` does.) + +* Build an expression specified on the command line: + + ```console + # nix build --impure --expr \ + 'with import <nixpkgs> {}; + runCommand "foo" { + buildInputs = [ hello ]; + } + "hello > $out"' + # cat ./result + Hello, world! + ``` + + Note that `--impure` is needed because we're using `<nixpkgs>`, + which relies on the `$NIX_PATH` environment variable. + +* Fetch a store path from the configured substituters, if it doesn't + already exist: + + ```console + # nix build /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2 + ``` + +# Description + +`nix build` builds the specified *installables*. Installables that +resolve to derivations are built (or substituted if possible). Store +path installables are substituted. + +Unless `--no-link` is specified, after a successful build, it creates +symlinks to the store paths of the installables. These symlinks have +the prefix `./result` by default; this can be overridden using the +`--out-link` option. Each symlink has a suffix `-<N>-<outname>`, where +*N* is the index of the installable (with the left-most installable +having index 0), and *outname* is the symbolic derivation output name +(e.g. `bin`, `dev` or `lib`). `-<N>` is omitted if *N* = 0, and +`-<outname>` is omitted if *outname* = `out` (denoting the default +output). + +)"" diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc new file mode 100644 index 000000000..aca024bca --- /dev/null +++ b/src/nix/bundle.cc @@ -0,0 +1,123 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "local-fs-store.hh" +#include "fs-accessor.hh" + +using namespace nix; + +struct CmdBundle : InstallableCommand +{ + std::string bundler = "github:matthewbauer/nix-bundle"; + std::optional<Path> outLink; + + CmdBundle() + { + addFlag({ + .longName = "bundler", + .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), + .labels = {"flake-url"}, + .handler = {&bundler}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); + + addFlag({ + .longName = "out-link", + .shortName = 'o', + .description = "Override the name of the symlink to the build result. It defaults to the base name of the app.", + .labels = {"path"}, + .handler = {&outLink}, + .completer = completePath + }); + + } + + std::string description() override + { + return "bundle an application so that it works outside of the Nix store"; + } + + std::string doc() override + { + return + #include "bundle.md" + ; + } + + Category category() override { return catSecondary; } + + Strings getDefaultFlakeAttrPaths() override + { + Strings res{"defaultApp." + settings.thisSystem.get()}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths()) + res.push_back(s); + return res; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + Strings res{"apps." + settings.thisSystem.get() + "."}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) + res.push_back(s); + return res; + } + + void run(ref<Store> store) override + { + auto evalState = getEvalState(); + + auto app = installable->toApp(*evalState).resolve(getEvalStore(), store); + + auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath(".")); + const flake::LockFlags lockFlags{ .writeLockFile = false }; + auto bundler = InstallableFlake(this, + evalState, std::move(bundlerFlakeRef), + Strings{bundlerName == "" ? "defaultBundler" : bundlerName}, + Strings({"bundlers."}), lockFlags); + + Value * arg = evalState->allocValue(); + evalState->mkAttrs(*arg, 2); + + PathSet context; + for (auto & i : app.context) + context.insert("=" + store->printStorePath(i.path)); + mkString(*evalState->allocAttr(*arg, evalState->symbols.create("program")), app.program, context); + + mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); + + arg->attrs->sort(); + + auto vRes = evalState->allocValue(); + evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); + + if (!evalState->isDerivation(*vRes)) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + auto attr1 = vRes->attrs->get(evalState->sDrvPath); + if (!attr1) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + PathSet context2; + StorePath drvPath = store->parseStorePath(evalState->coerceToPath(*attr1->pos, *attr1->value, context2)); + + auto attr2 = vRes->attrs->get(evalState->sOutPath); + if (!attr2) + throw Error("the bundler '%s' does not produce a derivation", bundler.what()); + + StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2)); + + store->buildPaths({ DerivedPath::Built { drvPath } }); + + auto outPathS = store->printStorePath(outPath); + + if (!outLink) + outLink = baseNameOf(app.program); + + store.dynamic_pointer_cast<LocalFSStore>()->addPermRoot(outPath, absPath(*outLink)); + } +}; + +static auto r2 = registerCommand<CmdBundle>("bundle"); diff --git a/src/nix/bundle.md b/src/nix/bundle.md new file mode 100644 index 000000000..5e2298376 --- /dev/null +++ b/src/nix/bundle.md @@ -0,0 +1,36 @@ +R""( + +# Examples + +* Bundle Hello: + + ```console + # nix bundle nixpkgs#hello + # ./hello + Hello, world! + ``` + +* Bundle a specific version of Nix: + + ```console + # nix bundle github:NixOS/nix/e3ddffb27e5fc37a209cfd843c6f7f6a9460a8ec + # ./nix --version + nix (Nix) 2.4pre20201215_e3ddffb + ``` + +# Description + +`nix bundle` packs the closure of the [Nix app](./nix3-run.md) +*installable* into a single self-extracting executable. See the +[`nix-bundle` homepage](https://github.com/matthewbauer/nix-bundle) +for more details. + +> **Note** +> +> This command only works on Linux. + +# Bundler definitions + +TODO + +)"" diff --git a/src/nix/cat.cc b/src/nix/cat.cc index c82819af8..e28ee3c50 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -25,7 +25,11 @@ struct CmdCatStore : StoreCommand, MixCat { CmdCatStore() { - expectArg("path", &path); + expectArgs({ + .label = "path", + .handler = {&path}, + .completer = completePath + }); } std::string description() override @@ -33,7 +37,12 @@ struct CmdCatStore : StoreCommand, MixCat return "print the contents of a file in the Nix store on stdout"; } - Category category() override { return catUtility; } + std::string doc() override + { + return + #include "store-cat.md" + ; + } void run(ref<Store> store) override { @@ -47,7 +56,11 @@ struct CmdCatNar : StoreCommand, MixCat CmdCatNar() { - expectArg("nar", &narPath); + expectArgs({ + .label = "nar", + .handler = {&narPath}, + .completer = completePath + }); expectArg("path", &path); } @@ -56,7 +69,12 @@ struct CmdCatNar : StoreCommand, MixCat return "print the contents of a file inside a NAR file on stdout"; } - Category category() override { return catUtility; } + std::string doc() override + { + return + #include "nar-cat.md" + ; + } void run(ref<Store> store) override { @@ -64,5 +82,5 @@ struct CmdCatNar : StoreCommand, MixCat } }; -static auto r1 = registerCommand<CmdCatStore>("cat-store"); -static auto r2 = registerCommand<CmdCatNar>("cat-nar"); +static auto rCmdCatStore = registerCommand2<CmdCatStore>({"store", "cat"}); +static auto rCmdCatNar = registerCommand2<CmdCatNar>({"nar", "cat"}); diff --git a/src/nix/command.cc b/src/nix/command.cc deleted file mode 100644 index 3651a9e9c..000000000 --- a/src/nix/command.cc +++ /dev/null @@ -1,198 +0,0 @@ -#include "command.hh" -#include "store-api.hh" -#include "derivations.hh" -#include "nixexpr.hh" -#include "profiles.hh" - -extern char * * environ __attribute__((weak)); - -namespace nix { - -Commands * RegisterCommand::commands = nullptr; - -StoreCommand::StoreCommand() -{ -} - -ref<Store> StoreCommand::getStore() -{ - if (!_store) - _store = createStore(); - return ref<Store>(_store); -} - -ref<Store> StoreCommand::createStore() -{ - return openStore(); -} - -void StoreCommand::run() -{ - run(getStore()); -} - -StorePathsCommand::StorePathsCommand(bool recursive) - : recursive(recursive) -{ - if (recursive) - addFlag({ - .longName = "no-recursive", - .description = "apply operation to specified paths only", - .handler = {&this->recursive, false}, - }); - else - addFlag({ - .longName = "recursive", - .shortName = 'r', - .description = "apply operation to closure of the specified paths", - .handler = {&this->recursive, true}, - }); - - mkFlag(0, "all", "apply operation to the entire store", &all); -} - -void StorePathsCommand::run(ref<Store> store) -{ - StorePaths storePaths; - - if (all) { - if (installables.size()) - throw UsageError("'--all' does not expect arguments"); - for (auto & p : store->queryAllValidPaths()) - storePaths.push_back(p); - } - - else { - for (auto & p : toStorePaths(store, realiseMode, installables)) - storePaths.push_back(p); - - if (recursive) { - StorePathSet closure; - store->computeFSClosure(StorePathSet(storePaths.begin(), storePaths.end()), closure, false, false); - storePaths.clear(); - for (auto & p : closure) - storePaths.push_back(p); - } - } - - run(store, std::move(storePaths)); -} - -void StorePathCommand::run(ref<Store> store) -{ - auto storePaths = toStorePaths(store, NoBuild, installables); - - if (storePaths.size() != 1) - throw UsageError("this command requires exactly one store path"); - - run(store, *storePaths.begin()); -} - -Strings editorFor(const Pos & pos) -{ - auto editor = getEnv("EDITOR").value_or("cat"); - auto args = tokenizeString<Strings>(editor); - if (pos.line > 0 && ( - editor.find("emacs") != std::string::npos || - editor.find("nano") != std::string::npos || - editor.find("vim") != std::string::npos)) - args.push_back(fmt("+%d", pos.line)); - args.push_back(pos.file); - return args; -} - -MixProfile::MixProfile() -{ - addFlag({ - .longName = "profile", - .description = "profile to update", - .labels = {"path"}, - .handler = {&profile}, - }); -} - -void MixProfile::updateProfile(const StorePath & storePath) -{ - if (!profile) return; - auto store = getStore().dynamic_pointer_cast<LocalFSStore>(); - if (!store) throw Error("'--profile' is not supported for this Nix store"); - auto profile2 = absPath(*profile); - switchLink(profile2, - createGeneration( - ref<LocalFSStore>(store), - profile2, store->printStorePath(storePath))); -} - -void MixProfile::updateProfile(const Buildables & buildables) -{ - if (!profile) return; - - std::optional<StorePath> result; - - for (auto & buildable : buildables) { - for (auto & output : buildable.outputs) { - if (result) - throw Error("'--profile' requires that the arguments produce a single store path, but there are multiple"); - result = output.second; - } - } - - if (!result) - throw Error("'--profile' requires that the arguments produce a single store path, but there are none"); - - updateProfile(*result); -} - -MixDefaultProfile::MixDefaultProfile() -{ - profile = getDefaultProfile(); -} - -MixEnvironment::MixEnvironment() : ignoreEnvironment(false) -{ - addFlag({ - .longName = "ignore-environment", - .shortName = 'i', - .description = "clear the entire environment (except those specified with --keep)", - .handler = {&ignoreEnvironment, true}, - }); - - addFlag({ - .longName = "keep", - .shortName = 'k', - .description = "keep specified environment variable", - .labels = {"name"}, - .handler = {[&](std::string s) { keep.insert(s); }}, - }); - - addFlag({ - .longName = "unset", - .shortName = 'u', - .description = "unset specified environment variable", - .labels = {"name"}, - .handler = {[&](std::string s) { unset.insert(s); }}, - }); -} - -void MixEnvironment::setEnviron() { - if (ignoreEnvironment) { - if (!unset.empty()) - throw UsageError("--unset does not make sense with --ignore-environment"); - - for (const auto & var : keep) { - auto val = getenv(var.c_str()); - if (val) stringsEnv.emplace_back(fmt("%s=%s", var.c_str(), val)); - } - - vectorEnv = stringsToCharPtrs(stringsEnv); - environ = vectorEnv.data(); - } else { - if (!keep.empty()) - throw UsageError("--keep does not make sense without --ignore-environment"); - - for (const auto & var : unset) - unsetenv(var.c_str()); - } -} - -} diff --git a/src/nix/command.hh b/src/nix/command.hh deleted file mode 100644 index 959d5f19d..000000000 --- a/src/nix/command.hh +++ /dev/null @@ -1,197 +0,0 @@ -#pragma once - -#include "installables.hh" -#include "args.hh" -#include "common-eval-args.hh" -#include "path.hh" -#include "eval.hh" - -namespace nix { - -extern std::string programPath; - -static constexpr Command::Category catSecondary = 100; -static constexpr Command::Category catUtility = 101; -static constexpr Command::Category catNixInstallation = 102; - -/* A command that requires a Nix store. */ -struct StoreCommand : virtual Command -{ - StoreCommand(); - void run() override; - ref<Store> getStore(); - virtual ref<Store> createStore(); - virtual void run(ref<Store>) = 0; - -private: - std::shared_ptr<Store> _store; -}; - -struct SourceExprCommand : virtual StoreCommand, MixEvalArgs -{ - Path file; - - SourceExprCommand(); - - /* Return a value representing the Nix expression from which we - are installing. This is either the file specified by ‘--file’, - or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs - = import ...; bla = import ...; }’. */ - Value * getSourceExpr(EvalState & state); - - ref<EvalState> getEvalState(); - -private: - - std::shared_ptr<EvalState> evalState; - - RootValue vSourceExpr; -}; - -enum RealiseMode { Build, NoBuild, DryRun }; - -/* A command that operates on a list of "installables", which can be - store paths, attribute paths, Nix expressions, etc. */ -struct InstallablesCommand : virtual Args, SourceExprCommand -{ - std::vector<std::shared_ptr<Installable>> installables; - - InstallablesCommand() - { - expectArgs("installables", &_installables); - } - - void prepare() override; - - virtual bool useDefaultInstallables() { return true; } - -private: - - std::vector<std::string> _installables; -}; - -/* A command that operates on exactly one "installable" */ -struct InstallableCommand : virtual Args, SourceExprCommand -{ - std::shared_ptr<Installable> installable; - - InstallableCommand() - { - expectArg("installable", &_installable); - } - - void prepare() override; - -private: - - std::string _installable; -}; - -/* A command that operates on zero or more store paths. */ -struct StorePathsCommand : public InstallablesCommand -{ -private: - - bool recursive = false; - bool all = false; - -protected: - - RealiseMode realiseMode = NoBuild; - -public: - - StorePathsCommand(bool recursive = false); - - using StoreCommand::run; - - virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0; - - void run(ref<Store> store) override; - - bool useDefaultInstallables() override { return !all; } -}; - -/* A command that operates on exactly one store path. */ -struct StorePathCommand : public InstallablesCommand -{ - using StoreCommand::run; - - virtual void run(ref<Store> store, const StorePath & storePath) = 0; - - void run(ref<Store> store) override; -}; - -/* A helper class for registering commands globally. */ -struct RegisterCommand -{ - static Commands * commands; - - RegisterCommand(const std::string & name, - std::function<ref<Command>()> command) - { - if (!commands) commands = new Commands; - commands->emplace(name, command); - } -}; - -template<class T> -static RegisterCommand registerCommand(const std::string & name) -{ - return RegisterCommand(name, [](){ return make_ref<T>(); }); -} - -std::shared_ptr<Installable> parseInstallable( - SourceExprCommand & cmd, ref<Store> store, const std::string & installable, - bool useDefaultInstallables); - -Buildables build(ref<Store> store, RealiseMode mode, - std::vector<std::shared_ptr<Installable>> installables); - -std::set<StorePath> toStorePaths(ref<Store> store, RealiseMode mode, - std::vector<std::shared_ptr<Installable>> installables); - -StorePath toStorePath(ref<Store> store, RealiseMode mode, - std::shared_ptr<Installable> installable); - -std::set<StorePath> toDerivations(ref<Store> store, - std::vector<std::shared_ptr<Installable>> installables, - bool useDeriver = false); - -/* Helper function to generate args that invoke $EDITOR on - filename:lineno. */ -Strings editorFor(const Pos & pos); - -struct MixProfile : virtual StoreCommand -{ - std::optional<Path> profile; - - MixProfile(); - - /* If 'profile' is set, make it point at 'storePath'. */ - void updateProfile(const StorePath & storePath); - - /* If 'profile' is set, make it point at the store path produced - by 'buildables'. */ - void updateProfile(const Buildables & buildables); -}; - -struct MixDefaultProfile : MixProfile -{ - MixDefaultProfile(); -}; - -struct MixEnvironment : virtual Args { - - StringSet keep, unset; - Strings stringsEnv; - std::vector<char*> vectorEnv; - bool ignoreEnvironment; - - MixEnvironment(); - - /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ - void setEnviron(); -}; - -} diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 64099f476..0489dfe06 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -8,7 +8,7 @@ using namespace nix; -struct CmdCopy : StorePathsCommand +struct CmdCopy : BuiltPathsCommand { std::string srcUri, dstUri; @@ -16,35 +16,39 @@ struct CmdCopy : StorePathsCommand SubstituteFlag substitute = NoSubstitute; + using BuiltPathsCommand::run; + CmdCopy() - : StorePathsCommand(true) + : BuiltPathsCommand(true) { addFlag({ .longName = "from", - .description = "URI of the source Nix store", + .description = "URL of the source Nix store.", .labels = {"store-uri"}, .handler = {&srcUri}, }); addFlag({ .longName = "to", - .description = "URI of the destination Nix store", + .description = "URL of the destination Nix store.", .labels = {"store-uri"}, .handler = {&dstUri}, }); addFlag({ .longName = "no-check-sigs", - .description = "do not require that paths are signed by trusted keys", + .description = "Do not require that paths are signed by trusted keys.", .handler = {&checkSigs, NoCheckSigs}, }); addFlag({ .longName = "substitute-on-destination", .shortName = 's', - .description = "whether to try substitutes on the destination store (only supported by SSH)", + .description = "Whether to try substitutes on the destination store (only supported by SSH stores).", .handler = {&substitute, Substitute}, }); + + realiseMode = Realise::Outputs; } std::string description() override @@ -52,32 +56,11 @@ struct CmdCopy : StorePathsCommand return "copy paths between Nix stores"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To copy Firefox from the local store to a binary cache in file:///tmp/cache:", - "nix copy --to file:///tmp/cache $(type -p firefox)" - }, - Example{ - "To copy the entire current NixOS system closure to another machine via SSH:", - "nix copy --to ssh://server /run/current-system" - }, - Example{ - "To copy a closure from another machine via SSH:", - "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" - }, -#ifdef ENABLE_S3 - Example{ - "To copy Hello to an S3 binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello" - }, - Example{ - "To copy Hello to an S3-compatible binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello" - }, -#endif - }; + return + #include "copy.md" + ; } Category category() override { return catSecondary; } @@ -87,16 +70,28 @@ struct CmdCopy : StorePathsCommand return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); } - void run(ref<Store> srcStore, StorePaths storePaths) override + void run(ref<Store> store) override { if (srcUri.empty() && dstUri.empty()) throw UsageError("you must pass '--from' and/or '--to'"); + BuiltPathsCommand::run(store); + } + + void run(ref<Store> srcStore, BuiltPaths paths) override + { ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); - copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()), - NoRepair, checkSigs, substitute); + RealisedPath::Set stuffToCopy; + + for (auto & builtPath : paths) { + auto theseRealisations = builtPath.toRealisedPaths(*srcStore); + stuffToCopy.insert(theseRealisations.begin(), theseRealisations.end()); + } + + copyPaths( + *srcStore, *dstStore, stuffToCopy, NoRepair, checkSigs, substitute); } }; -static auto r1 = registerCommand<CmdCopy>("copy"); +static auto rCmdCopy = registerCommand<CmdCopy>("copy"); diff --git a/src/nix/copy.md b/src/nix/copy.md new file mode 100644 index 000000000..25e0ddadc --- /dev/null +++ b/src/nix/copy.md @@ -0,0 +1,58 @@ +R""( + +# Examples + +* Copy Firefox from the local store to a binary cache in `/tmp/cache`: + + ```console + # nix copy --to file:///tmp/cache $(type -p firefox) + ``` + + Note the `file://` - without this, the destination is a chroot + store, not a binary cache. + +* Copy the entire current NixOS system closure to another machine via + SSH: + + ```console + # nix copy -s --to ssh://server /run/current-system + ``` + + The `-s` flag causes the remote machine to try to substitute missing + store paths, which may be faster if the link between the local and + remote machines is slower than the link between the remote machine + and its substituters (e.g. `https://cache.nixos.org`). + +* Copy a closure from another machine via SSH: + + ```console + # nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2 + ``` + +* Copy Hello to a binary cache in an Amazon S3 bucket: + + ```console + # nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello + ``` + + or to an S3-compatible storage system: + + ```console + # nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello + ``` + + Note that this only works if Nix is built with AWS support. + +* Copy a closure from `/nix/store` to the chroot store `/tmp/nix/nix/store`: + + ```console + # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs + ``` + +# Description + +`nix copy` copies store path closures between two Nix stores. The +source store is specified using `--from` and the destination using +`--to`. If one of these is omitted, it defaults to the local store. + +)"" diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc new file mode 100644 index 000000000..2cf2a04c9 --- /dev/null +++ b/src/nix/daemon.cc @@ -0,0 +1,359 @@ +#include "command.hh" +#include "shared.hh" +#include "local-store.hh" +#include "remote-store.hh" +#include "util.hh" +#include "serialise.hh" +#include "archive.hh" +#include "globals.hh" +#include "derivations.hh" +#include "finally.hh" +#include "legacy.hh" +#include "daemon.hh" + +#include <algorithm> +#include <climits> +#include <cstring> + +#include <unistd.h> +#include <signal.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/stat.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <errno.h> +#include <pwd.h> +#include <grp.h> +#include <fcntl.h> + +#if __APPLE__ || __FreeBSD__ +#include <sys/ucred.h> +#endif + +using namespace nix; +using namespace nix::daemon; + +#ifndef __linux__ +#define SPLICE_F_MOVE 0 +static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) +{ + // We ignore most parameters, we just have them for conformance with the linux syscall + std::vector<char> buf(8192); + auto read_count = read(fd_in, buf.data(), buf.size()); + if (read_count == -1) + return read_count; + auto write_count = decltype(read_count)(0); + while (write_count < read_count) { + auto res = write(fd_out, buf.data() + write_count, read_count - write_count); + if (res == -1) + return res; + write_count += res; + } + return read_count; +} +#endif + + +static void sigChldHandler(int sigNo) +{ + // Ensure we don't modify errno of whatever we've interrupted + auto saved_errno = errno; + // Reap all dead children. + while (waitpid(-1, 0, WNOHANG) > 0) ; + errno = saved_errno; +} + + +static void setSigChldAction(bool autoReap) +{ + struct sigaction act, oact; + act.sa_handler = autoReap ? sigChldHandler : SIG_DFL; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, &oact)) + throw SysError("setting SIGCHLD handler"); +} + + +bool matchUser(const string & user, const string & group, const Strings & users) +{ + if (find(users.begin(), users.end(), "*") != users.end()) + return true; + + if (find(users.begin(), users.end(), user) != users.end()) + return true; + + for (auto & i : users) + if (string(i, 0, 1) == "@") { + if (group == string(i, 1)) return true; + struct group * gr = getgrnam(i.c_str() + 1); + if (!gr) continue; + for (char * * mem = gr->gr_mem; *mem; mem++) + if (user == string(*mem)) return true; + } + + return false; +} + + +struct PeerInfo +{ + bool pidKnown; + pid_t pid; + bool uidKnown; + uid_t uid; + bool gidKnown; + gid_t gid; +}; + + +// Get the identity of the caller, if possible. +static PeerInfo getPeerInfo(int remote) +{ + PeerInfo peer = { false, 0, false, 0, false, 0 }; + +#if defined(SO_PEERCRED) + + ucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = { true, cred.pid, true, cred.uid, true, cred.gid }; + +#elif defined(LOCAL_PEERCRED) + +#if !defined(SOL_LOCAL) +#define SOL_LOCAL 0 +#endif + + xucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = { false, 0, true, cred.cr_uid, false, 0 }; + +#endif + + return peer; +} + + +#define SD_LISTEN_FDS_START 3 + + +static ref<Store> openUncachedStore() +{ + Store::Params params; // FIXME: get params from somewhere + // Disable caching since the client already does that. + params["path-info-cache-size"] = "0"; + return openStore(settings.storeUri, params); +} + + +static void daemonLoop() +{ + if (chdir("/") == -1) + throw SysError("cannot change current directory"); + + // Get rid of children automatically; don't let them become zombies. + setSigChldAction(true); + + AutoCloseFD fdSocket; + + // Handle socket-based activation by systemd. + auto listenFds = getEnv("LISTEN_FDS"); + if (listenFds) { + if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") + throw Error("unexpected systemd environment variables"); + fdSocket = SD_LISTEN_FDS_START; + closeOnExec(fdSocket.get()); + } + + // Otherwise, create and bind to a Unix domain socket. + else { + createDirs(dirOf(settings.nixDaemonSocketFile)); + fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); + } + + // Loop accepting connections. + while (1) { + + try { + // Accept a connection. + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(fdSocket.get(), + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + checkInterrupt(); + if (!remote) { + if (errno == EINTR) continue; + throw SysError("accepting connection"); + } + + closeOnExec(remote.get()); + + TrustedFlag trusted = NotTrusted; + PeerInfo peer = getPeerInfo(remote.get()); + + struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; + string user = pw ? pw->pw_name : std::to_string(peer.uid); + + struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; + string group = gr ? gr->gr_name : std::to_string(peer.gid); + + Strings trustedUsers = settings.trustedUsers; + Strings allowedUsers = settings.allowedUsers; + + if (matchUser(user, group, trustedUsers)) + trusted = Trusted; + + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) + throw Error("user '%1%' is not allowed to connect to the Nix daemon", user); + + printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) + % (peer.pidKnown ? std::to_string(peer.pid) : "<unknown>") + % (peer.uidKnown ? user : "<unknown>")); + + // Fork a child to handle the connection. + ProcessOptions options; + options.errorPrefix = "unexpected Nix daemon error: "; + options.dieWithParent = false; + options.runExitHandlers = true; + options.allowVfork = false; + startProcess([&]() { + fdSocket = -1; + + // Background the daemon. + if (setsid() == -1) + throw SysError("creating a new session"); + + // Restore normal handling of SIGCHLD. + setSigChldAction(false); + + // For debugging, stuff the pid into argv[1]. + if (peer.pidKnown && savedArgv[1]) { + string processName = std::to_string(peer.pid); + strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1])); + } + + // Handle the connection. + FdSource from(remote.get()); + FdSink to(remote.get()); + processConnection(openUncachedStore(), from, to, trusted, NotRecursive, [&](Store & store) { +#if 0 + /* Prevent users from doing something very dangerous. */ + if (geteuid() == 0 && + querySetting("build-users-group", "") == "") + throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); +#endif + store.createUser(user, peer.uid); + }); + + exit(0); + }, options); + + } catch (Interrupted & e) { + return; + } catch (Error & error) { + ErrorInfo ei = error.info(); + // FIXME: add to trace? + ei.msg = hintfmt("error processing connection: %1%", ei.msg.str()); + logError(ei); + } + } +} + +static void runDaemon(bool stdio) +{ + if (stdio) { + if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) { + auto conn = store->openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } + } else { + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + /* Auth hook is empty because in this mode we blindly trust the + standard streams. Limiting access to those is explicitly + not `nix-daemon`'s responsibility. */ + processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){}); + } + } else + daemonLoop(); +} + +static int main_nix_daemon(int argc, char * * argv) +{ + { + auto stdio = false; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--daemon") + ; // ignored for backwards compatibility + else if (*arg == "--help") + showManPage("nix-daemon"); + else if (*arg == "--version") + printVersion("nix-daemon"); + else if (*arg == "--stdio") + stdio = true; + else return false; + return true; + }); + + runDaemon(stdio); + + return 0; + } +} + +static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); + +struct CmdDaemon : StoreCommand +{ + std::string description() override + { + return "daemon to perform store operations on behalf of non-root clients"; + } + + Category category() override { return catUtility; } + + std::string doc() override + { + return + #include "daemon.md" + ; + } + + void run(ref<Store> store) override + { + runDaemon(false); + } +}; + +static auto rCmdDaemon = registerCommand2<CmdDaemon>({"daemon"}); diff --git a/src/nix/daemon.md b/src/nix/daemon.md new file mode 100644 index 000000000..e97016a94 --- /dev/null +++ b/src/nix/daemon.md @@ -0,0 +1,21 @@ +R""( + +# Example + +* Run the daemon in the foreground: + + ```console + # nix daemon + ``` + +# Description + +This command runs the Nix daemon, which is a required component in +multi-user Nix installations. It performs build actions and other +operations on the Nix store on behalf of non-root users. Usually you +don't run the daemon directly; instead it's managed by a service +management framework such as `systemd`. + +Note that this daemon does not fork into the background. + +)"" diff --git a/src/nix/describe-stores.cc b/src/nix/describe-stores.cc new file mode 100644 index 000000000..1dd384c0e --- /dev/null +++ b/src/nix/describe-stores.cc @@ -0,0 +1,44 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +using namespace nix; + +struct CmdDescribeStores : Command, MixJSON +{ + std::string description() override + { + return "show registered store types and their available options"; + } + + Category category() override { return catUtility; } + + void run() override + { + auto res = nlohmann::json::object(); + for (auto & implem : *Implementations::registered) { + auto storeConfig = implem.getConfig(); + auto storeName = storeConfig->name(); + res[storeName] = storeConfig->toJSON(); + } + if (json) { + std::cout << res; + } else { + for (auto & [storeName, storeConfig] : res.items()) { + std::cout << "## " << storeName << std::endl << std::endl; + for (auto & [optionName, optionDesc] : storeConfig.items()) { + std::cout << "### " << optionName << std::endl << std::endl; + std::cout << optionDesc["description"].get<std::string>() << std::endl; + std::cout << "default: " << optionDesc["defaultValue"] << std::endl <<std::endl; + if (!optionDesc["aliases"].empty()) + std::cout << "aliases: " << optionDesc["aliases"] << std::endl << std::endl; + } + } + } + } +}; + +static auto rDescribeStore = registerCommand<CmdDescribeStores>("describe-stores"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index eb93f56fc..c20b9f272 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -3,97 +3,166 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include "derivations.hh" #include "affinity.hh" #include "progress-bar.hh" +#include "run.hh" -#include <regex> +#include <memory> +#include <nlohmann/json.hpp> using namespace nix; -struct Var +struct DevelopSettings : Config { - bool exported = true; - bool associative = false; - std::string value; // quoted string or array + Setting<std::string> bashPrompt{this, "", "bash-prompt", + "The bash prompt (`PS1`) in `nix develop` shells."}; + + Setting<std::string> bashPromptSuffix{this, "", "bash-prompt-suffix", + "Suffix appended to the `PS1` environment variable in `nix develop` shells."}; }; +static DevelopSettings developSettings; + +static GlobalConfig::Register rDevelopSettings(&developSettings); + struct BuildEnvironment { - std::map<std::string, Var> env; - std::string bashFunctions; -}; + struct String + { + bool exported; + std::string value; -BuildEnvironment readEnvironment(const Path & path) -{ - BuildEnvironment res; + bool operator == (const String & other) const + { + return exported == other.exported && value == other.value; + } + }; - std::set<std::string> exported; + using Array = std::vector<std::string>; - debug("reading environment file '%s'", path); + using Associative = std::map<std::string, std::string>; - auto file = readFile(path); + using Value = std::variant<String, Array, Associative>; - auto pos = file.cbegin(); + std::map<std::string, Value> vars; + std::map<std::string, std::string> bashFunctions; - static std::string varNameRegex = - R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; + static BuildEnvironment fromJSON(std::string_view in) + { + BuildEnvironment res; - static std::regex declareRegex( - "^declare -x (" + varNameRegex + ")" + - R"re((?:="((?:[^"\\]|\\.)*)")?\n)re"); + std::set<std::string> exported; - static std::string simpleStringRegex = - R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; + auto json = nlohmann::json::parse(in); - static std::string quotedStringRegex = - R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re"; + for (auto & [name, info] : json["variables"].items()) { + std::string type = info["type"]; + if (type == "var" || type == "exported") + res.vars.insert({name, BuildEnvironment::String { .exported = type == "exported", .value = info["value"] }}); + else if (type == "array") + res.vars.insert({name, (Array) info["value"]}); + else if (type == "associative") + res.vars.insert({name, (Associative) info["value"]}); + } - static std::string indexedArrayRegex = - R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; + for (auto & [name, def] : json["bashFunctions"].items()) { + res.bashFunctions.insert({name, def}); + } - static std::regex varRegex( - "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n"); + return res; + } - /* Note: we distinguish between an indexed and associative array - using the space before the closing parenthesis. Will - undoubtedly regret this some day. */ - static std::regex assocArrayRegex( - "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n"); + std::string toJSON() const + { + auto res = nlohmann::json::object(); + + auto vars2 = nlohmann::json::object(); + for (auto & [name, value] : vars) { + auto info = nlohmann::json::object(); + if (auto str = std::get_if<String>(&value)) { + info["type"] = str->exported ? "exported" : "var"; + info["value"] = str->value; + } + else if (auto arr = std::get_if<Array>(&value)) { + info["type"] = "array"; + info["value"] = *arr; + } + else if (auto arr = std::get_if<Associative>(&value)) { + info["type"] = "associative"; + info["value"] = *arr; + } + vars2[name] = std::move(info); + } + res["variables"] = std::move(vars2); - static std::regex functionRegex( - "^" + varNameRegex + " \\(\\) *\n"); + res["bashFunctions"] = bashFunctions; - while (pos != file.end()) { + auto json = res.dump(); - std::smatch match; + assert(BuildEnvironment::fromJSON(json) == *this); - if (std::regex_search(pos, file.cend(), match, declareRegex)) { - pos = match[0].second; - exported.insert(match[1]); - } + return json; + } - else if (std::regex_search(pos, file.cend(), match, varRegex)) { - pos = match[0].second; - res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .value = match[2] }}); + void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const + { + for (auto & [name, value] : vars) { + if (!ignoreVars.count(name)) { + if (auto str = std::get_if<String>(&value)) { + out << fmt("%s=%s\n", name, shellEscape(str->value)); + if (str->exported) + out << fmt("export %s\n", name); + } + else if (auto arr = std::get_if<Array>(&value)) { + out << "declare -a " << name << "=("; + for (auto & s : *arr) + out << shellEscape(s) << " "; + out << ")\n"; + } + else if (auto arr = std::get_if<Associative>(&value)) { + out << "declare -A " << name << "=("; + for (auto & [n, v] : *arr) + out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " "; + out << ")\n"; + } + } } - else if (std::regex_search(pos, file.cend(), match, assocArrayRegex)) { - pos = match[0].second; - res.env.insert({match[1], Var { .associative = true, .value = match[2] }}); + for (auto & [name, def] : bashFunctions) { + out << name << " ()\n{\n" << def << "}\n"; } + } - else if (std::regex_search(pos, file.cend(), match, functionRegex)) { - res.bashFunctions = std::string(pos, file.cend()); - break; - } + static std::string getString(const Value & value) + { + if (auto str = std::get_if<String>(&value)) + return str->value; + else + throw Error("bash variable is not a string"); + } - else throw Error("shell environment '%s' has unexpected line '%s'", - path, file.substr(pos - file.cbegin(), 60)); + static Array getStrings(const Value & value) + { + if (auto str = std::get_if<String>(&value)) + return tokenizeString<Array>(str->value); + else if (auto arr = std::get_if<Array>(&value)) { + return *arr; + } else if (auto assoc = std::get_if<Associative>(&value)) { + Array assocKeys; + std::for_each(assoc->begin(), assoc->end(), [&](auto & n) { assocKeys.push_back(n.first); }); + return assocKeys; + } + else + throw Error("bash variable is not a string or array"); } - return res; -} + bool operator == (const BuildEnvironment & other) const + { + return vars == other.vars && bashFunctions == other.bashFunctions; + } +}; const static std::string getEnvSh = #include "get-env.sh.gen.hh" @@ -104,15 +173,15 @@ const static std::string getEnvSh = modified derivation with the same dependencies and nearly the same initial environment variables, that just writes the resulting environment to a file and exits. */ -StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) +static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore, const StorePath & drvPath) { - auto drv = store->derivationFromPath(drvPath); + auto drv = evalStore->derivationFromPath(drvPath); auto builder = baseNameOf(drv.builder); if (builder != "bash") throw Error("'nix develop' only works on derivations that use 'bash' as their builder"); - auto getEnvShPath = store->addTextToStore("get-env.sh", getEnvSh, {}); + auto getEnvShPath = evalStore->addTextToStore("get-env.sh", getEnvSh, {}); drv.args = {store->printStorePath(getEnvShPath)}; @@ -124,42 +193,57 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ - auto drvName = std::string(drvPath.name()); - assert(hasSuffix(drvName, ".drv")); - drvName.resize(drvName.size() - 4); - drvName += "-env"; - for (auto & output : drv.outputs) - drv.env.erase(output.first); - drv.env["out"] = ""; - drv.env["outputs"] = "out"; + drv.name += "-env"; drv.inputSrcs.insert(std::move(getEnvShPath)); - Hash h = hashDerivationModulo(*store, drv, true); - auto shellOutPath = store->makeOutputPath("out", h, drvName); - drv.outputs.insert_or_assign("out", DerivationOutput { .path = shellOutPath }); - drv.env["out"] = store->printStorePath(shellOutPath); - auto shellDrvPath2 = writeDerivation(store, drv, drvName); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto & output : drv.outputs) { + output.second = { + .output = DerivationOutputDeferred{}, + }; + drv.env[output.first] = hashPlaceholder(output.first); + } + } else { + for (auto & output : drv.outputs) { + output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; + drv.env[output.first] = ""; + } + Hash h = std::get<0>(hashDerivationModulo(*evalStore, drv, true)); - /* Build the derivation. */ - store->buildPaths({{shellDrvPath2}}); + for (auto & output : drv.outputs) { + auto outPath = store->makeOutputPath(output.first, h, drv.name); + output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; + drv.env[output.first] = store->printStorePath(outPath); + } + } - assert(store->isValidPath(shellOutPath)); + auto shellDrvPath = writeDerivation(*evalStore, drv); - return shellOutPath; + /* Build the derivation. */ + store->buildPaths({DerivedPath::Built{shellDrvPath}}, bmNormal, evalStore); + + for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { + assert(optPath); + auto & outPath = *optPath; + assert(store->isValidPath(outPath)); + auto outPathS = store->toRealPath(outPath); + if (lstat(outPathS).st_size) + return outPath; + } + + throw Error("get-env.sh failed to produce an environment"); } struct Common : InstallableCommand, MixProfile { - std::set<string> ignoreVars{ + std::set<std::string> ignoreVars{ "BASHOPTS", - "EUID", "HOME", // FIXME: don't ignore in pure mode? "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "NIX_LOG_FD", + "NIX_REMOTE", "PPID", - "PWD", "SHELLOPTS", - "SHLVL", "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", "TEMPDIR", @@ -170,35 +254,85 @@ struct Common : InstallableCommand, MixProfile "UID", }; - void makeRcScript(const BuildEnvironment & buildEnvironment, std::ostream & out) + std::vector<std::pair<std::string, std::string>> redirects; + + Common() { + addFlag({ + .longName = "redirect", + .description = "Redirect a store path to a mutable location.", + .labels = {"installable", "outputs-dir"}, + .handler = {[&](std::string installable, std::string outputsDir) { + redirects.push_back({installable, outputsDir}); + }} + }); + } + + std::string makeRcScript( + ref<Store> store, + const BuildEnvironment & buildEnvironment, + const Path & outputsDir = absPath(".") + "/outputs") + { + std::ostringstream out; + out << "unset shellHook\n"; out << "nix_saved_PATH=\"$PATH\"\n"; - for (auto & i : buildEnvironment.env) { - if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { - if (i.second.associative) - out << fmt("declare -A %s=(%s)\n", i.first, i.second.value); - else { - out << fmt("%s=%s\n", i.first, i.second.value); - if (i.second.exported) - out << fmt("export %s\n", i.first); - } - } - } + buildEnvironment.toBash(out, ignoreVars); out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; - out << buildEnvironment.bashFunctions << "\n"; - - // FIXME: set outputs - - out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n"; + out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); out << "eval \"$shellHook\"\n"; + + auto script = out.str(); + + /* Substitute occurrences of output paths. */ + auto outputs = buildEnvironment.vars.find("outputs"); + assert(outputs != buildEnvironment.vars.end()); + + // FIXME: properly unquote 'outputs'. + StringMap rewrites; + for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { + auto from = buildEnvironment.vars.find(outputName); + assert(from != buildEnvironment.vars.end()); + // FIXME: unquote + rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName}); + } + + /* Substitute redirects. */ + for (auto & [installable_, dir_] : redirects) { + auto dir = absPath(dir_); + auto installable = parseInstallable(store, installable_); + auto builtPaths = toStorePaths( + getEvalStore(), store, Realise::Nothing, OperateOn::Output, {installable}); + for (auto & path: builtPaths) { + auto from = store->printStorePath(path); + if (script.find(from) == std::string::npos) + warn("'%s' (path '%s') is not used by this build environment", installable->what(), from); + else { + printInfo("redirecting '%s' to '%s'", from, dir); + rewrites.insert({from, dir}); + } + } + } + + return rewriteStrings(script, rewrites); + } + + Strings getDefaultFlakeAttrPaths() override + { + return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()}; + } + Strings getDefaultFlakeAttrPathPrefixes() override + { + auto res = SourceExprCommand::getDefaultFlakeAttrPathPrefixes(); + res.emplace_front("devShells." + settings.thisSystem.get() + "."); + return res; } StorePath getShellOutPath(ref<Store> store) @@ -215,7 +349,7 @@ struct Common : InstallableCommand, MixProfile auto & drvPath = *drvs.begin(); - return getDerivationEnvironment(store, drvPath); + return getDerivationEnvironment(store, getEvalStore(), drvPath); } } @@ -227,26 +361,66 @@ struct Common : InstallableCommand, MixProfile updateProfile(shellOutPath); - return {readEnvironment(strPath), strPath}; + debug("reading environment file '%s'", strPath); + + return {BuildEnvironment::fromJSON(readFile(store->toRealPath(shellOutPath))), strPath}; } }; struct CmdDevelop : Common, MixEnvironment { std::vector<std::string> command; + std::optional<std::string> phase; CmdDevelop() { addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed insted of an interactive shell", + .description = "Instead of starting an interactive shell, start the specified command and arguments.", .labels = {"command", "args"}, .handler = {[&](std::vector<std::string> ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); command = ss; }} }); + + addFlag({ + .longName = "phase", + .description = "The stdenv phase to run (e.g. `build` or `configure`).", + .labels = {"phase-name"}, + .handler = {&phase}, + }); + + addFlag({ + .longName = "configure", + .description = "Run the `configure` phase.", + .handler = {&phase, {"configure"}}, + }); + + addFlag({ + .longName = "build", + .description = "Run the `build` phase.", + .handler = {&phase, {"build"}}, + }); + + addFlag({ + .longName = "check", + .description = "Run the `check` phase.", + .handler = {&phase, {"check"}}, + }); + + addFlag({ + .longName = "install", + .description = "Run the `install` phase.", + .handler = {&phase, {"install"}}, + }); + + addFlag({ + .longName = "installcheck", + .description = "Run the `installcheck` phase.", + .handler = {&phase, {"installCheck"}}, + }); } std::string description() override @@ -254,22 +428,11 @@ struct CmdDevelop : Common, MixEnvironment return "run a bash shell that provides the build environment of a derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get the build environment of GNU hello:", - "nix develop nixpkgs.hello" - }, - Example{ - "To store the build environment in a profile:", - "nix develop --profile /tmp/my-shell nixpkgs.hello" - }, - Example{ - "To use a build environment previously recorded in a profile:", - "nix develop /tmp/my-shell" - }, - }; + return + #include "develop.md" + ; } void run(ref<Store> store) override @@ -278,54 +441,101 @@ struct CmdDevelop : Common, MixEnvironment auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); - std::ostringstream ss; - makeRcScript(buildEnvironment, ss); + auto script = makeRcScript(store, buildEnvironment); + + if (verbosity >= lvlDebug) + script += "set -x\n"; - ss << fmt("rm -f '%s'\n", rcFilePath); + script += fmt("command rm -f '%s'\n", rcFilePath); - if (!command.empty()) { + if (phase) { + if (!command.empty()) + throw UsageError("you cannot use both '--command' and '--phase'"); + // FIXME: foundMakefile is set by buildPhase, need to get + // rid of that. + script += fmt("foundMakefile=1\n"); + script += fmt("runHook %1%Phase\n", *phase); + } + + else if (!command.empty()) { std::vector<std::string> args; for (auto s : command) args.push_back(shellEscape(s)); - ss << fmt("exec %s\n", concatStringsSep(" ", args)); + script += fmt("exec %s\n", concatStringsSep(" ", args)); } - writeFull(rcFileFd.get(), ss.str()); - - stopProgressBar(); + else { + script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n" + script; + if (developSettings.bashPrompt != "") + script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt)); + if (developSettings.bashPromptSuffix != "") + script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", shellEscape(developSettings.bashPromptSuffix)); + } - auto shell = getEnv("SHELL").value_or("bash"); + writeFull(rcFileFd.get(), script); setEnviron(); // prevent garbage collection until shell exits setenv("NIX_GCROOT", gcroot.data(), 1); - auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; + Path shell = "bash"; + + try { + auto state = getEvalState(); - restoreAffinity(); - restoreSignals(); + auto nixpkgsLockFlags = lockFlags; + nixpkgsLockFlags.inputOverrides = {}; + nixpkgsLockFlags.inputUpdates = {}; - execvp(shell.c_str(), stringsToCharPtrs(args).data()); + auto bashInstallable = std::make_shared<InstallableFlake>( + this, + state, + installable->nixpkgsFlakeRef(), + Strings{"bashInteractive"}, + Strings{"legacyPackages." + settings.thisSystem.get() + "."}, + nixpkgsLockFlags); + + shell = store->printStorePath( + toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; + } catch (Error &) { + ignoreException(); + } + + // If running a phase or single command, don't want an interactive shell running after + // Ctrl-C, so don't pass --rcfile + auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath} + : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath}; + + // Need to chdir since phases assume in flake directory + if (phase) { + // chdir if installable is a flake of type git+file or path + auto installableFlake = std::dynamic_pointer_cast<InstallableFlake>(installable); + if (installableFlake) { + auto sourcePath = installableFlake->getLockedFlake()->flake.resolvedRef.input.getSourcePath(); + if (sourcePath) { + if (chdir(sourcePath->c_str()) == -1) { + throw SysError("chdir to '%s' failed", *sourcePath); + } + } + } + } - throw SysError("executing shell '%s'", shell); + runProgramInStore(store, shell, args); } }; -struct CmdPrintDevEnv : Common +struct CmdPrintDevEnv : Common, MixJSON { std::string description() override { return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To apply the build environment of GNU hello to the current shell:", - ". <(nix print-dev-env nixpkgs.hello)" - }, - }; + return + #include "print-dev-env.md" + ; } Category category() override { return catUtility; } @@ -336,9 +546,12 @@ struct CmdPrintDevEnv : Common stopProgressBar(); - makeRcScript(buildEnvironment, std::cout); + logger->writeToStdout( + json + ? buildEnvironment.toJSON() + : makeRcScript(store, buildEnvironment)); } }; -static auto r1 = registerCommand<CmdPrintDevEnv>("print-dev-env"); -static auto r2 = registerCommand<CmdDevelop>("develop"); +static auto rCmdPrintDevEnv = registerCommand<CmdPrintDevEnv>("print-dev-env"); +static auto rCmdDevelop = registerCommand<CmdDevelop>("develop"); diff --git a/src/nix/develop.md b/src/nix/develop.md new file mode 100644 index 000000000..c86c4872b --- /dev/null +++ b/src/nix/develop.md @@ -0,0 +1,103 @@ +R""( + +# Examples + +* Start a shell with the build environment of the default package of + the flake in the current directory: + + ```console + # nix develop + ``` + + Typical commands to run inside this shell are: + + ```console + # configurePhase + # buildPhase + # installPhase + ``` + + Alternatively, you can run whatever build tools your project uses + directly, e.g. for a typical Unix project: + + ```console + # ./configure --prefix=$out + # make + # make install + ``` + +* Run a particular build phase directly: + + ```console + # nix develop --configure + # nix develop --build + # nix develop --check + # nix develop --install + # nix develop --installcheck + ``` + +* Start a shell with the build environment of GNU Hello: + + ```console + # nix develop nixpkgs#hello + ``` + +* Record a build environment in a profile: + + ```console + # nix develop --profile /tmp/my-build-env nixpkgs#hello + ``` + +* Use a build environment previously recorded in a profile: + + ```console + # nix develop /tmp/my-build-env + ``` + +* Replace all occurences of the store path corresponding to + `glibc.dev` with a writable directory: + + ```console + # nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev + ``` + + Note that this is useful if you're running a `nix develop` shell for + `nixpkgs#glibc` in `~/my-glibc` and want to compile another package + against it. + +# Description + +`nix develop` starts a `bash` shell that provides an interactive build +environment nearly identical to what Nix would use to build +*installable*. Inside this shell, environment variables and shell +functions are set up so that you can interactively and incrementally +build your package. + +Nix determines the build environment by building a modified version of +the derivation *installable* that just records the environment +initialised by `stdenv` and exits. This build environment can be +recorded into a profile using `--profile`. + +The prompt used by the `bash` shell can be customised by setting the +`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in +the flake's `nixConfig` attribute. + +# Flake output attributes + +If no flake output attribute is given, `nix develop` tries the following +flake output attributes: + +* `devShell.<system>` + +* `defaultPackage.<system>` + +If a flake output *name* is given, `nix develop` tries the following flake +output attributes: + +* `devShells.<system>.<name>` + +* `packages.<system>.<name>` + +* `legacyPackages.<system>.<name>` + +)"" diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc new file mode 100644 index 000000000..734c41e0e --- /dev/null +++ b/src/nix/diff-closures.cc @@ -0,0 +1,141 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "common-args.hh" +#include "names.hh" + +#include <regex> + +namespace nix { + +struct Info +{ + std::string outputName; +}; + +// name -> version -> store paths +typedef std::map<std::string, std::map<std::string, std::map<StorePath, Info>>> GroupedPaths; + +GroupedPaths getClosureInfo(ref<Store> store, const StorePath & toplevel) +{ + StorePathSet closure; + store->computeFSClosure({toplevel}, closure); + + GroupedPaths groupedPaths; + + for (auto & path : closure) { + /* Strip the output name. Unfortunately this is ambiguous (we + can't distinguish between output names like "bin" and + version suffixes like "unstable"). */ + static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); + std::smatch match; + std::string name(path.name()); + std::string outputName; + if (std::regex_match(name, match, regex)) { + name = match[1]; + outputName = match[2]; + } + + DrvName drvName(name); + groupedPaths[drvName.name][drvName.version].emplace(path, Info { .outputName = outputName }); + } + + return groupedPaths; +} + +std::string showVersions(const std::set<std::string> & versions) +{ + if (versions.empty()) return "∅"; + std::set<std::string> versions2; + for (auto & version : versions) + versions2.insert(version.empty() ? "ε" : version); + return concatStringsSep(", ", versions2); +} + +void printClosureDiff( + ref<Store> store, + const StorePath & beforePath, + const StorePath & afterPath, + std::string_view indent) +{ + auto beforeClosure = getClosureInfo(store, beforePath); + auto afterClosure = getClosureInfo(store, afterPath); + + std::set<std::string> allNames; + for (auto & [name, _] : beforeClosure) allNames.insert(name); + for (auto & [name, _] : afterClosure) allNames.insert(name); + + for (auto & name : allNames) { + auto & beforeVersions = beforeClosure[name]; + auto & afterVersions = afterClosure[name]; + + auto totalSize = [&](const std::map<std::string, std::map<StorePath, Info>> & versions) + { + uint64_t sum = 0; + for (auto & [_, paths] : versions) + for (auto & [path, _] : paths) + sum += store->queryPathInfo(path)->narSize; + return sum; + }; + + auto beforeSize = totalSize(beforeVersions); + auto afterSize = totalSize(afterVersions); + auto sizeDelta = (int64_t) afterSize - (int64_t) beforeSize; + auto showDelta = std::abs(sizeDelta) >= 8 * 1024; + + std::set<std::string> removed, unchanged; + for (auto & [version, _] : beforeVersions) + if (!afterVersions.count(version)) removed.insert(version); else unchanged.insert(version); + + std::set<std::string> added; + for (auto & [version, _] : afterVersions) + if (!beforeVersions.count(version)) added.insert(version); + + if (showDelta || !removed.empty() || !added.empty()) { + std::vector<std::string> items; + if (!removed.empty() || !added.empty()) + items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); + if (showDelta) + items.push_back(fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); + std::cout << fmt("%s%s: %s\n", indent, name, concatStringsSep(", ", items)); + } + } +} + +} + +using namespace nix; + +struct CmdDiffClosures : SourceExprCommand +{ + std::string _before, _after; + + CmdDiffClosures() + { + expectArg("before", &_before); + expectArg("after", &_after); + } + + std::string description() override + { + return "show what packages and versions were added and removed between two closures"; + } + + std::string doc() override + { + return + #include "diff-closures.md" + ; + } + + void run(ref<Store> store) override + { + auto before = parseInstallable(store, _before); + auto beforePath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, before); + auto after = parseInstallable(store, _after); + auto afterPath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, after); + printClosureDiff(store, beforePath, afterPath, ""); + } +}; + +static auto rCmdDiffClosures = registerCommand2<CmdDiffClosures>({"store", "diff-closures"}); diff --git a/src/nix/diff-closures.md b/src/nix/diff-closures.md new file mode 100644 index 000000000..0294c0d8d --- /dev/null +++ b/src/nix/diff-closures.md @@ -0,0 +1,51 @@ +R""( + +# Examples + +* Show what got added and removed between two versions of the NixOS + system profile: + + ```console + # nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link + acpi-call: 2020-04-07-5.8.16 → 2020-04-07-5.8.18 + baloo-widgets: 20.08.1 → 20.08.2 + bluez-qt: +12.6 KiB + dolphin: 20.08.1 → 20.08.2, +13.9 KiB + kdeconnect: 20.08.2 → ∅, -6597.8 KiB + kdeconnect-kde: ∅ → 20.08.2, +6599.7 KiB + … + ``` + +# Description + +This command shows the differences between the two closures *before* +and *after* with respect to the addition, removal, or version change +of packages, as well as changes in store path sizes. + +For each package name in the two closures (where a package name is +defined as the name component of a store path excluding the version), +if there is a change in the set of versions of the package, or a +change in the size of the store paths of more than 8 KiB, it prints a +line like this: + +```console +dolphin: 20.08.1 → 20.08.2, +13.9 KiB +``` + +No size change is shown if it's below the threshold. If the package +does not exist in either the *before* or *after* closures, it is +represented using `∅` (empty set) on the appropriate side of the +arrow. If a package has an empty version string, the version is +rendered as `ε` (epsilon). + +There may be multiple versions of a package in each closure. In that +case, only the changed versions are shown. Thus, + +```console +libfoo: 1.2, 1.3 → 1.4 +``` + +leaves open the possibility that there are other versions (e.g. `1.1`) +that exist in both closures. + +)"" diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 82e92cdd0..4f3003448 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -5,6 +5,7 @@ #include "serve-protocol.hh" #include "shared.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "util.hh" #include "worker-protocol.hh" @@ -49,9 +50,7 @@ struct CmdDoctor : StoreCommand { logger->log("Running checks against store uri: " + store->getUri()); - auto type = getStoreType(); - - if (type < tOther) { + if (store.dynamic_pointer_cast<LocalFSStore>()) { success &= checkNixInPath(); success &= checkProfileRoots(store); } @@ -133,4 +132,4 @@ struct CmdDoctor : StoreCommand } }; -static auto r1 = registerCommand<CmdDoctor>("doctor"); +static auto rCmdDoctor = registerCommand<CmdDoctor>("doctor"); diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index e1de71bf8..c4edc894b 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "store-api.hh" +#include "archive.hh" using namespace nix; @@ -7,21 +8,16 @@ struct CmdDumpPath : StorePathCommand { std::string description() override { - return "dump a store path to stdout (in NAR format)"; + return "serialise a store path to stdout in NAR format"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get a NAR from the binary cache https://cache.nixos.org/:", - "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" - }, - }; + return + #include "store-dump-path.md" + ; } - Category category() override { return catUtility; } - void run(ref<Store> store, const StorePath & storePath) override { FdSink sink(STDOUT_FILENO); @@ -30,4 +26,39 @@ struct CmdDumpPath : StorePathCommand } }; -static auto r1 = registerCommand<CmdDumpPath>("dump-path"); +static auto rDumpPath = registerCommand2<CmdDumpPath>({"store", "dump-path"}); + +struct CmdDumpPath2 : Command +{ + Path path; + + CmdDumpPath2() + { + expectArgs({ + .label = "path", + .handler = {&path}, + .completer = completePath + }); + } + + std::string description() override + { + return "serialise a path to stdout in NAR format"; + } + + std::string doc() override + { + return + #include "nar-dump-path.md" + ; + } + + void run() override + { + FdSink sink(STDOUT_FILENO); + dumpPath(path, sink); + sink.flush(); + } +}; + +static auto rDumpPath2 = registerCommand2<CmdDumpPath2>({"nar", "dump-path"}); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 067d3a973..fc48db0d7 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -15,14 +15,11 @@ struct CmdEdit : InstallableCommand return "open the Nix expression of a Nix package in $EDITOR"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To open the Nix expression of the GNU Hello package:", - "nix edit nixpkgs.hello" - }, - }; + return + #include "edit.md" + ; } Category category() override { return catSecondary; } @@ -34,7 +31,7 @@ struct CmdEdit : InstallableCommand auto [v, pos] = installable->toValue(*state); try { - pos = findDerivationFilename(*state, *v, installable->what()); + pos = findPackageFilename(*state, *v, installable->what()); } catch (NoPositionInfo &) { } @@ -45,6 +42,8 @@ struct CmdEdit : InstallableCommand auto args = editorFor(pos); + restoreProcessContext(); + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); std::string command; @@ -53,4 +52,4 @@ struct CmdEdit : InstallableCommand } }; -static auto r1 = registerCommand<CmdEdit>("edit"); +static auto rCmdEdit = registerCommand<CmdEdit>("edit"); diff --git a/src/nix/edit.md b/src/nix/edit.md new file mode 100644 index 000000000..80563d06b --- /dev/null +++ b/src/nix/edit.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Open the Nix expression of the GNU Hello package: + + ```console + # nix edit nixpkgs#hello + ``` + +* Get the filename and line number used by `nix edit`: + + ```console + # nix eval --raw nixpkgs#hello.meta.position + /nix/store/fvafw0gvwayzdan642wrv84pzm5bgpmy-source/pkgs/applications/misc/hello/default.nix:15 + ``` + +# Description + +This command opens the Nix expression of a derivation in an +editor. The filename and line number of the derivation are taken from +its `meta.position` attribute. Nixpkgs' `stdenv.mkDerivation` sets +this attribute to the location of the definition of the +`meta.description`, `version` or `name` derivation attributes. + +The editor to invoke is specified by the `EDITOR` environment +variable. It defaults to `cat`. If the editor is `emacs`, `nano` or +`vim`, it is passed the line number of the derivation using the +argument `+<lineno>`. + +)"" diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 26e98ac2a..65d61e005 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -3,6 +3,7 @@ #include "shared.hh" #include "store-api.hh" #include "eval.hh" +#include "eval-inline.hh" #include "json.hh" #include "value-to-json.hh" #include "progress-bar.hh" @@ -12,10 +13,30 @@ using namespace nix; struct CmdEval : MixJSON, InstallableCommand { bool raw = false; + std::optional<std::string> apply; + std::optional<Path> writeTo; CmdEval() { - mkFlag(0, "raw", "print strings unquoted", &raw); + addFlag({ + .longName = "raw", + .description = "Print strings without quotes or escaping.", + .handler = {&raw, true}, + }); + + addFlag({ + .longName = "apply", + .description = "Apply the function *expr* to each argument.", + .labels = {"expr"}, + .handler = {&apply}, + }); + + addFlag({ + .longName = "write-to", + .description = "Write a string or attrset of strings to *path*.", + .labels = {"path"}, + .handler = {&writeTo}, + }); } std::string description() override @@ -23,26 +44,11 @@ struct CmdEval : MixJSON, InstallableCommand return "evaluate a Nix expression"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To evaluate a Nix expression given on the command line:", - "nix eval '(1 + 2)'" - }, - Example{ - "To evaluate a Nix expression from a file or URI:", - "nix eval -f channel:nixos-17.09 hello.name" - }, - Example{ - "To get the current version of Nixpkgs:", - "nix eval --raw nixpkgs.lib.version" - }, - Example{ - "To print the store path of the Hello package:", - "nix eval --raw nixpkgs.hello" - }, - }; + return + #include "eval.md" + ; } Category category() override { return catSecondary; } @@ -54,20 +60,66 @@ struct CmdEval : MixJSON, InstallableCommand auto state = getEvalState(); - auto v = installable->toValue(*state).first; + auto [v, pos] = installable->toValue(*state); PathSet context; - if (raw) { + if (apply) { + auto vApply = state->allocValue(); + state->eval(state->parseExprFromString(*apply, absPath(".")), *vApply); + auto vRes = state->allocValue(); + state->callFunction(*vApply, *v, *vRes, noPos); + v = vRes; + } + + if (writeTo) { + stopProgressBar(); + + if (pathExists(*writeTo)) + throw Error("path '%s' already exists", *writeTo); + + std::function<void(Value & v, const Pos & pos, const Path & path)> recurse; + + recurse = [&](Value & v, const Pos & pos, const Path & path) + { + state->forceValue(v); + if (v.type() == nString) + // FIXME: disallow strings with contexts? + writeFile(path, v.string.s); + else if (v.type() == nAttrs) { + if (mkdir(path.c_str(), 0777) == -1) + throw SysError("creating directory '%s'", path); + for (auto & attr : *v.attrs) + try { + if (attr.name == "." || attr.name == "..") + throw Error("invalid file name '%s'", attr.name); + recurse(*attr.value, *attr.pos, path + "/" + std::string(attr.name)); + } catch (Error & e) { + e.addTrace(*attr.pos, hintfmt("while evaluating the attribute '%s'", attr.name)); + throw; + } + } + else + throw TypeError("value at '%s' is not a string or an attribute set", pos); + }; + + recurse(*v, pos, *writeTo); + } + + else if (raw) { stopProgressBar(); std::cout << state->coerceToString(noPos, *v, context); - } else if (json) { + } + + else if (json) { JSONPlaceholder jsonOut(std::cout); printValueAsJSON(*state, true, *v, jsonOut, context); - } else { + } + + else { state->forceValueDeep(*v); - logger->stdout("%s", *v); + logger->cout("%s", *v); } } }; -static auto r1 = registerCommand<CmdEval>("eval"); +static auto rCmdEval = registerCommand<CmdEval>("eval"); diff --git a/src/nix/eval.md b/src/nix/eval.md new file mode 100644 index 000000000..61334cde1 --- /dev/null +++ b/src/nix/eval.md @@ -0,0 +1,74 @@ +R""( + +# Examples + +* Evaluate a Nix expression given on the command line: + + ```console + # nix eval --expr '1 + 2' + ``` + +* Evaluate a Nix expression to JSON: + + ```console + # nix eval --json --expr '{ x = 1; }' + {"x":1} + ``` + +* Evaluate a Nix expression from a file: + + ```console + # nix eval -f ./my-nixpkgs hello.name + ``` + +* Get the current version of the `nixpkgs` flake: + + ```console + # nix eval --raw nixpkgs#lib.version + ``` + +* Print the store path of the Hello package: + + ```console + # nix eval --raw nixpkgs#hello + ``` + +* Get a list of checks in the `nix` flake: + + ```console + # nix eval nix#checks.x86_64-linux --apply builtins.attrNames + ``` + +* Generate a directory with the specified contents: + + ```console + # nix eval --write-to ./out --expr '{ foo = "bar"; subdir.bla = "123"; }' + # cat ./out/foo + bar + # cat ./out/subdir/bla + 123 + +# Description + +This command evaluates the Nix expression *installable* and prints the +result on standard output. + +# Output format + +`nix eval` can produce output in several formats: + +* By default, the evaluation result is printed as a Nix expression. + +* With `--json`, the evaluation result is printed in JSON format. Note + that this fails if the result contains values that are not + representable as JSON, such as functions. + +* With `--raw`, the evaluation result must be a string, which is + printed verbatim, without any quoting. + +* With `--write-to` *path*, the evaluation result must be a string or + a nested attribute set whose leaf values are strings. These strings + are written to files named *path*/*attrpath*. *path* must not + already exist. + +)"" diff --git a/src/nix/flake-archive.md b/src/nix/flake-archive.md new file mode 100644 index 000000000..85bbeeb16 --- /dev/null +++ b/src/nix/flake-archive.md @@ -0,0 +1,29 @@ +R""( + +# Examples + +* Copy the `dwarffs` flake and its dependencies to a binary cache: + + ```console + # nix flake archive --to file:///tmp/my-cache dwarffs + ``` + +* Fetch the `dwarffs` flake and its dependencies to the local Nix + store: + + ```console + # nix flake archive dwarffs + ``` + +* Print the store paths of the flake sources of NixOps without + fetching them: + + ```console + # nix flake archive --json --dry-run nixops + ``` + +# Description + +FIXME + +)"" diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md new file mode 100644 index 000000000..d995d6274 --- /dev/null +++ b/src/nix/flake-check.md @@ -0,0 +1,73 @@ +R""( + +# Examples + +* Evaluate the flake in the current directory, and build its checks: + + ```console + # nix flake check + ``` + +* Verify that the `patchelf` flake evaluates, but don't build its + checks: + + ```console + # nix flake check --no-build github:NixOS/patchelf + ``` + +# Description + +This command verifies that the flake specified by flake reference +*flake-url* can be evaluated successfully (as detailed below), and +that the derivations specified by the flake's `checks` output can be +built successfully. + +If the `keep-going` option is set to `true`, Nix will keep evaluating as much +as it can and report the errors as it encounters them. Otherwise it will stop +at the first error. + +# Evaluation checks + +The following flake output attributes must be derivations: + +* `checks.`*system*`.`*name* +* `defaultPackage.`*system*` +* `devShell.`*system*` +* `devShells.`*system*`.`*name*` +* `nixosConfigurations.`*name*`.config.system.build.toplevel +* `packages.`*system*`.`*name* + +The following flake output attributes must be [app +definitions](./nix3-run.md): + +* `apps.`*system*`.`*name* +* `defaultApp.`*system*` + +The following flake output attributes must be [template +definitions](./nix3-flake-init.md): + +* `defaultTemplate` +* `templates`.`*name* + +The following flake output attributes must be *Nixpkgs overlays*: + +* `overlay` +* `overlays`.`*name* + +The following flake output attributes must be *NixOS modules*: + +* `nixosModule` +* `nixosModules`.`*name* + +The following flake output attributes must be +[bundlers](./nix3-bundle.md): + +* `bundlers`.`*name* +* `defaultBundler` + +In addition, the `hydraJobs` output is evaluated in the same way as +Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested +attribute set of derivations). Similarly, the +`legacyPackages`.*system* output is evaluated like `nix-env -qa`. + +)"" diff --git a/src/nix/flake-clone.md b/src/nix/flake-clone.md new file mode 100644 index 000000000..36cb96051 --- /dev/null +++ b/src/nix/flake-clone.md @@ -0,0 +1,18 @@ +R""( + +# Examples + +* Check out the source code of the `dwarffs` flake and build it: + + ```console + # nix flake clone dwarffs --dest dwarffs + # cd dwarffs + # nix build + ``` + +# Description + +This command performs a Git or Mercurial clone of the repository +containing the source code of the flake *flake-url*. + +)"" diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md new file mode 100644 index 000000000..890038016 --- /dev/null +++ b/src/nix/flake-init.md @@ -0,0 +1,54 @@ +R""( + +# Examples + +* Create a flake using the default template: + + ```console + # nix flake init + ``` + +* List available templates: + + ```console + # nix flake show templates + ``` + +* Create a flake from a specific template: + + ```console + # nix flake init -t templates#simpleContainer + ``` + +# Description + +This command creates a flake in the current directory by copying the +files of a template. It will not overwrite existing files. The default +template is `templates#defaultTemplate`, but this can be overridden +using `-t`. + +# Template definitions + +A flake can declare templates through its `templates` and +`defaultTemplate` output attributes. A template has two attributes: + +* `description`: A one-line description of the template, in CommonMark + syntax. + +* `path`: The path of the directory to be copied. + +Here is an example: + +``` +outputs = { self }: { + + templates.rust = { + path = ./rust; + description = "A simple Rust/Cargo project"; + }; + + templates.defaultTemplate = self.templates.rust; +} +``` + +)"" diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md new file mode 100644 index 000000000..2af0ad81e --- /dev/null +++ b/src/nix/flake-lock.md @@ -0,0 +1,38 @@ +R""( + +# Examples + +* Update the `nixpkgs` and `nix` inputs of the flake in the current + directory: + + ```console + # nix flake lock --update-input nixpkgs --update-input nix + * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' + * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + ``` + +# Description + +This command updates the lock file of a flake (`flake.lock`) so that +it contains a lock for every flake input specified in +`flake.nix`. Existing lock file entries are not updated unless +required by a flag such as `--update-input`. + +Note that every command that operates on a flake will also update the +lock file if needed, and supports the same flags. Therefore, + +```console +# nix flake lock --update-input nixpkgs +# nix build +``` + +is equivalent to: + +```console +# nix build --update-input nixpkgs +``` + +Thus, this command is only useful if you want to update the lock file +separately from any other action such as building. + +)"" diff --git a/src/nix/flake-metadata.md b/src/nix/flake-metadata.md new file mode 100644 index 000000000..5a009409b --- /dev/null +++ b/src/nix/flake-metadata.md @@ -0,0 +1,110 @@ +R""( + +# Examples + +* Show what `nixpkgs` resolves to: + + ```console + # nix flake metadata nixpkgs + Resolved URL: github:edolstra/dwarffs + Locked URL: github:edolstra/dwarffs/f691e2c991e75edb22836f1dbe632c40324215c5 + Description: A filesystem that fetches DWARF debug info from the Internet on demand + Path: /nix/store/769s05vjydmc2lcf6b02az28wsa9ixh1-source + Revision: f691e2c991e75edb22836f1dbe632c40324215c5 + Last modified: 2021-01-21 15:41:26 + Inputs: + ├───nix: github:NixOS/nix/6254b1f5d298ff73127d7b0f0da48f142bdc753c + │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f + │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31 + └───nixpkgs follows input 'nix/nixpkgs' + ``` + +* Show information about `dwarffs` in JSON format: + + ```console + # nix flake metadata dwarffs --json | jq . + { + "description": "A filesystem that fetches DWARF debug info from the Internet on demand", + "lastModified": 1597153508, + "locked": { + "lastModified": 1597153508, + "narHash": "sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=", + "owner": "edolstra", + "repo": "dwarffs", + "rev": "d181d714fd36eb06f4992a1997cd5601e26db8f5", + "type": "github" + }, + "locks": { ... }, + "original": { + "id": "dwarffs", + "type": "indirect" + }, + "originalUrl": "flake:dwarffs", + "path": "/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source", + "resolved": { + "owner": "edolstra", + "repo": "dwarffs", + "type": "github" + }, + "resolvedUrl": "github:edolstra/dwarffs", + "revision": "d181d714fd36eb06f4992a1997cd5601e26db8f5", + "url": "github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5" + } + ``` + +# Description + +This command shows information about the flake specified by the flake +reference *flake-url*. It resolves the flake reference using the +[flake registry](./nix3-registry.md), fetches it, and prints some meta +data. This includes: + +* `Resolved URL`: If *flake-url* is a flake identifier, then this is + the flake reference that specifies its actual location, looked up in + the flake registry. + +* `Locked URL`: A flake reference that contains a commit or content + hash and thus uniquely identifies a specific flake version. + +* `Description`: A one-line description of the flake, taken from the + `description` field in `flake.nix`. + +* `Path`: The store path containing the source code of the flake. + +* `Revision`: The Git or Mercurial commit hash of the locked flake. + +* `Revisions`: The number of ancestors of the Git or Mercurial commit + of the locked flake. Note that this is not available for `github` + flakes. + +* `Last modified`: For Git or Mercurial flakes, this is the commit + time of the commit of the locked flake; for tarball flakes, it's the + most recent timestamp of any file inside the tarball. + +* `Inputs`: The flake inputs with their corresponding lock file + entries. + +With `--json`, the output is a JSON object with the following fields: + +* `original` and `originalUrl`: The flake reference specified by the + user (*flake-url*) in attribute set and URL representation. + +* `resolved` and `resolvedUrl`: The resolved flake reference (see + above) in attribute set and URL representation. + +* `locked` and `lockedUrl`: The locked flake reference (see above) in + attribute set and URL representation. + +* `description`: See `Description` above. + +* `path`: See `Path` above. + +* `revision`: See `Revision` above. + +* `revCount`: See `Revisions` above. + +* `lastModified`: See `Last modified` above. + +* `locks`: The contents of `flake.lock`. + +)"" diff --git a/src/nix/flake-new.md b/src/nix/flake-new.md new file mode 100644 index 000000000..725695c01 --- /dev/null +++ b/src/nix/flake-new.md @@ -0,0 +1,34 @@ +R""( + +# Examples + +* Create a flake using the default template in the directory `hello`: + + ```console + # nix flake new hello + ``` + +* List available templates: + + ```console + # nix flake show templates + ``` + +* Create a flake from a specific template in the directory `hello`: + + ```console + # nix flake new hello -t templates#trivial + ``` + +# Description + +This command creates a flake in the directory `dest-dir`, which must +not already exist. It's equivalent to: + +```console +# mkdir dest-dir +# cd dest-dir +# nix flake init +``` + +)"" diff --git a/src/nix/flake-prefetch.md b/src/nix/flake-prefetch.md new file mode 100644 index 000000000..a1cf0289a --- /dev/null +++ b/src/nix/flake-prefetch.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix flake prefetch https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz?narHash=sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download the `dwarffs` flake (looked up in the flake registry): + + ```console + # nix flake prefetch dwarffs --json + {"hash":"sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=" + ,"storePath":"/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source"} + ``` + +# Description + +This command downloads the source tree denoted by flake reference +*flake-url*. Note that this does not need to be a flake (i.e. it does +not have to contain a `flake.nix` file). + +)"" diff --git a/src/nix/flake-show.md b/src/nix/flake-show.md new file mode 100644 index 000000000..e484cf47e --- /dev/null +++ b/src/nix/flake-show.md @@ -0,0 +1,41 @@ +R""( + +# Examples + +* Show the output attributes provided by the `patchelf` flake: + + ```console + github:NixOS/patchelf/f34751b88bd07d7f44f5cd3200fb4122bf916c7e + ├───checks + │ ├───aarch64-linux + │ │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + │ ├───i686-linux + │ │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + │ └───x86_64-linux + │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + ├───defaultPackage + │ ├───aarch64-linux: package 'patchelf-0.12.20201207.f34751b' + │ ├───i686-linux: package 'patchelf-0.12.20201207.f34751b' + │ └───x86_64-linux: package 'patchelf-0.12.20201207.f34751b' + ├───hydraJobs + │ ├───build + │ │ ├───aarch64-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ │ ├───i686-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ │ └───x86_64-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ ├───coverage: derivation 'patchelf-coverage-0.12.20201207.f34751b' + │ ├───release: derivation 'patchelf-0.12.20201207.f34751b' + │ └───tarball: derivation 'patchelf-tarball-0.12.20201207.f34751b' + └───overlay: Nixpkgs overlay + ``` + +# Description + +This command shows the output attributes provided by the flake +specified by flake reference *flake-url*. These are the top-level +attributes in the `outputs` of the flake, as well as lower-level +attributes for some standard outputs (e.g. `packages` or `checks`). + +With `--json`, the output is in a JSON representation suitable for automatic +processing by other tools. + +)"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md new file mode 100644 index 000000000..03b50e38e --- /dev/null +++ b/src/nix/flake-update.md @@ -0,0 +1,34 @@ +R""( + +# Examples + +* Recreate the lock file (i.e. update all inputs) and commit the new + lock file: + + ```console + # nix flake update + * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' + * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + … + warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' + ``` + +# Description + +This command recreates the lock file of a flake (`flake.lock`), thus +updating the lock for every mutable input (like `nixpkgs`) to its +current version. This is equivalent to passing `--recreate-lock-file` +to any command that operates on a flake. That is, + +```console +# nix flake update +# nix build +``` + +is equivalent to: + +```console +# nix build --recreate-lock-file +``` + +)"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc new file mode 100644 index 000000000..a127c3ac0 --- /dev/null +++ b/src/nix/flake.cc @@ -0,0 +1,1126 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "flake/flake.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "path-with-outputs.hh" +#include "attr-path.hh" +#include "fetchers.hh" +#include "registry.hh" +#include "json.hh" +#include "eval-cache.hh" + +#include <nlohmann/json.hpp> +#include <queue> +#include <iomanip> + +using namespace nix; +using namespace nix::flake; + +class FlakeCommand : virtual Args, public MixFlakeOptions +{ + std::string flakeUrl = "."; + +public: + + FlakeCommand() + { + expectArgs({ + .label = "flake-url", + .optional = true, + .handler = {&flakeUrl}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); + } + + FlakeRef getFlakeRef() + { + return parseFlakeRef(flakeUrl, absPath(".")); //FIXME + } + + LockedFlake lockFlake() + { + return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); + } + + std::optional<FlakeRef> getFlakeRefForCompletion() override + { + return getFlakeRef(); + } +}; + +struct CmdFlakeUpdate : FlakeCommand +{ + std::string description() override + { + return "update flake lock file"; + } + + CmdFlakeUpdate() + { + /* Remove flags that don't make sense. */ + removeFlag("recreate-lock-file"); + removeFlag("update-input"); + removeFlag("no-update-lock-file"); + removeFlag("no-write-lock-file"); + } + + std::string doc() override + { + return + #include "flake-update.md" + ; + } + + void run(nix::ref<nix::Store> store) override + { + settings.tarballTtl = 0; + + lockFlags.recreateLockFile = true; + lockFlags.writeLockFile = true; + lockFlags.applyNixConfig = true; + + lockFlake(); + } +}; + +struct CmdFlakeLock : FlakeCommand +{ + std::string description() override + { + return "create missing lock file entries"; + } + + CmdFlakeLock() + { + /* Remove flags that don't make sense. */ + removeFlag("no-write-lock-file"); + } + + std::string doc() override + { + return + #include "flake-lock.md" + ; + } + + void run(nix::ref<nix::Store> store) override + { + settings.tarballTtl = 0; + + lockFlags.writeLockFile = true; + lockFlags.applyNixConfig = true; + + lockFlake(); + } +}; + +static void enumerateOutputs(EvalState & state, Value & vFlake, + std::function<void(const std::string & name, Value & vProvide, const Pos & pos)> callback) +{ + state.forceAttrs(vFlake); + + auto aOutputs = vFlake.attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceAttrs(*aOutputs->value); + + for (auto & attr : *aOutputs->value->attrs) + callback(attr.name, *attr.value, *attr.pos); +} + +struct CmdFlakeMetadata : FlakeCommand, MixJSON +{ + std::string description() override + { + return "show flake metadata"; + } + + std::string doc() override + { + return + #include "flake-metadata.md" + ; + } + + void run(nix::ref<nix::Store> store) override + { + auto lockedFlake = lockFlake(); + auto & flake = lockedFlake.flake; + + if (json) { + nlohmann::json j; + if (flake.description) + j["description"] = *flake.description; + j["originalUrl"] = flake.originalRef.to_string(); + j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs()); + j["resolvedUrl"] = flake.resolvedRef.to_string(); + j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); + j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl + j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); + if (auto rev = flake.lockedRef.input.getRev()) + j["revision"] = rev->to_string(Base16, false); + if (auto revCount = flake.lockedRef.input.getRevCount()) + j["revCount"] = *revCount; + if (auto lastModified = flake.lockedRef.input.getLastModified()) + j["lastModified"] = *lastModified; + j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["locks"] = lockedFlake.lockFile.toJSON(); + logger->cout("%s", j.dump()); + } else { + logger->cout( + ANSI_BOLD "Resolved URL:" ANSI_NORMAL " %s", + flake.resolvedRef.to_string()); + logger->cout( + ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", + flake.lockedRef.to_string()); + if (flake.description) + logger->cout( + ANSI_BOLD "Description:" ANSI_NORMAL " %s", + *flake.description); + logger->cout( + ANSI_BOLD "Path:" ANSI_NORMAL " %s", + store->printStorePath(flake.sourceInfo->storePath)); + if (auto rev = flake.lockedRef.input.getRev()) + logger->cout( + ANSI_BOLD "Revision:" ANSI_NORMAL " %s", + rev->to_string(Base16, false)); + if (auto revCount = flake.lockedRef.input.getRevCount()) + logger->cout( + ANSI_BOLD "Revisions:" ANSI_NORMAL " %s", + *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) + logger->cout( + ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", + std::put_time(std::localtime(&*lastModified), "%F %T")); + + logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); + + std::unordered_set<std::shared_ptr<Node>> visited; + + std::function<void(const Node & node, const std::string & prefix)> recurse; + + recurse = [&](const Node & node, const std::string & prefix) + { + for (const auto & [i, input] : enumerate(node.inputs)) { + bool last = i + 1 == node.inputs.size(); + + if (auto lockedNode = std::get_if<0>(&input.second)) { + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + prefix + (last ? treeLast : treeConn), input.first, + *lockedNode ? (*lockedNode)->lockedRef : flake.lockedRef); + + bool firstVisit = visited.insert(*lockedNode).second; + + if (firstVisit) recurse(**lockedNode, prefix + (last ? treeNull : treeLine)); + } else if (auto follows = std::get_if<1>(&input.second)) { + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'", + prefix + (last ? treeLast : treeConn), input.first, + printInputPath(*follows)); + } + } + }; + + visited.insert(lockedFlake.lockFile.root); + recurse(*lockedFlake.lockFile.root, ""); + } + } +}; + +struct CmdFlakeInfo : CmdFlakeMetadata +{ + void run(nix::ref<nix::Store> store) override + { + warn("'nix flake info' is a deprecated alias for 'nix flake metadata'"); + CmdFlakeMetadata::run(store); + } +}; + +struct CmdFlakeCheck : FlakeCommand +{ + bool build = true; + + CmdFlakeCheck() + { + addFlag({ + .longName = "no-build", + .description = "Do not build checks.", + .handler = {&build, false} + }); + } + + std::string description() override + { + return "check whether the flake evaluates and run its tests"; + } + + std::string doc() override + { + return + #include "flake-check.md" + ; + } + + void run(nix::ref<nix::Store> store) override + { + settings.readOnlyMode = !build; + + auto state = getEvalState(); + + lockFlags.applyNixConfig = true; + auto flake = lockFlake(); + + bool hasErrors = false; + auto reportError = [&](const Error & e) { + try { + throw e; + } catch (Error & e) { + if (settings.keepGoing) { + ignoreException(); + hasErrors = true; + } + else + throw; + } + }; + + // FIXME: rewrite to use EvalCache. + + auto checkSystemName = [&](const std::string & system, const Pos & pos) { + // FIXME: what's the format of "system"? + if (system.find('-') == std::string::npos) + reportError(Error("'%s' is not a valid system type, at %s", system, pos)); + }; + + auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) -> std::optional<StorePath> { + try { + auto drvInfo = getDerivation(*state, v, false); + if (!drvInfo) + throw Error("flake attribute '%s' is not a derivation", attrPath); + // FIXME: check meta attributes + return std::make_optional(store->parseStorePath(drvInfo->queryDrvPath())); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the derivation '%s'", attrPath)); + reportError(e); + } + return std::nullopt; + }; + + std::vector<DerivedPath> drvPaths; + + auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + #if 0 + // FIXME + auto app = App(*state, v); + for (auto & i : app.context) { + auto [drvPathS, outputName] = decodeContext(i); + store->parseStorePath(drvPathS); + } + #endif + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the app definition '%s'", attrPath)); + reportError(e); + } + }; + + auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") + throw Error("overlay does not take an argument named 'final'"); + auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body); + if (!body || body->matchAttrs || std::string(body->arg) != "prev") + throw Error("overlay does not take an argument named 'prev'"); + // FIXME: if we have a 'nixpkgs' input, use it to + // evaluate the overlay. + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the overlay '%s'", attrPath)); + reportError(e); + } + }; + + auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.isLambda()) { + if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) + throw Error("module must match an open attribute set ('{ config, ... }')"); + } else if (v.type() == nAttrs) { + for (auto & attr : *v.attrs) + try { + state->forceValue(*attr.value, *attr.pos); + } catch (Error & e) { + e.addTrace(*attr.pos, hintfmt("while evaluating the option '%s'", attr.name)); + throw; + } + } else + throw Error("module must be a function or an attribute set"); + // FIXME: if we have a 'nixpkgs' input, use it to + // check the module. + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the NixOS module '%s'", attrPath)); + reportError(e); + } + }; + + std::function<void(const std::string & attrPath, Value & v, const Pos & pos)> checkHydraJobs; + + checkHydraJobs = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceAttrs(v, pos); + + if (state->isDerivation(v)) + throw Error("jobset should not be a derivation at top-level"); + + for (auto & attr : *v.attrs) { + state->forceAttrs(*attr.value, *attr.pos); + if (!state->isDerivation(*attr.value)) + checkHydraJobs(attrPath + "." + (std::string) attr.name, + *attr.value, *attr.pos); + } + + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the Hydra jobset '%s'", attrPath)); + reportError(e); + } + }; + + auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking NixOS configuration '%s'", attrPath)); + Bindings & bindings(*state->allocBindings(0)); + auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; + state->forceAttrs(*vToplevel, pos); + if (!state->isDerivation(*vToplevel)) + throw Error("attribute 'config.system.build.toplevel' is not a derivation"); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the NixOS configuration '%s'", attrPath)); + reportError(e); + } + }; + + auto checkTemplate = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking template '%s'", attrPath)); + + state->forceAttrs(v, pos); + + if (auto attr = v.attrs->get(state->symbols.create("path"))) { + if (attr->name == state->symbols.create("path")) { + PathSet context; + auto path = state->coerceToPath(*attr->pos, *attr->value, context); + if (!store->isInStore(path)) + throw Error("template '%s' has a bad 'path' attribute"); + // TODO: recursively check the flake in 'path'. + } + } else + throw Error("template '%s' lacks attribute 'path'", attrPath); + + if (auto attr = v.attrs->get(state->symbols.create("description"))) + state->forceStringNoCtx(*attr->value, *attr->pos); + else + throw Error("template '%s' lacks attribute 'description'", attrPath); + + for (auto & attr : *v.attrs) { + std::string name(attr.name); + if (name != "path" && name != "description") + throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); + } + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); + reportError(e); + } + }; + + auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (!v.isLambda()) + throw Error("bundler must be a function"); + if (!v.lambda.fun->formals || + v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || + v.lambda.fun->formals->argNames.find(state->symbols.create("system")) == v.lambda.fun->formals->argNames.end()) + throw Error("bundler must take formal arguments 'program' and 'system'"); + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking the template '%s'", attrPath)); + reportError(e); + } + }; + + { + Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); + + auto vFlake = state->allocValue(); + flake::callFlake(*state, flake, *vFlake); + + enumerateOutputs(*state, + *vFlake, + [&](const std::string & name, Value & vOutput, const Pos & pos) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking flake output '%s'", name)); + + try { + state->forceValue(vOutput, pos); + + if (name == "checks") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) { + auto drvPath = checkDerivation( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + if (drvPath && (std::string) attr.name == settings.thisSystem.get()) + drvPaths.push_back(DerivedPath::Built{*drvPath}); + } + } + } + + else if (name == "packages" || name == "devShells") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkDerivation( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } + } + + else if (name == "apps") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkApp( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } + } + + else if (name == "defaultPackage" || name == "devShell") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkDerivation( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } + + else if (name == "defaultApp") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkApp( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } + + else if (name == "legacyPackages") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + // FIXME: do getDerivations? + } + } + + else if (name == "overlay") + checkOverlay(name, vOutput, pos); + + else if (name == "overlays") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkOverlay(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "nixosModule") + checkModule(name, vOutput, pos); + + else if (name == "nixosModules") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkModule(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "nixosConfigurations") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkNixOSConfiguration(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "hydraJobs") + checkHydraJobs(name, vOutput, pos); + + else if (name == "defaultTemplate") + checkTemplate(name, vOutput, pos); + + else if (name == "templates") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkTemplate(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "defaultBundler") + checkBundler(name, vOutput, pos); + + else if (name == "bundlers") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkBundler(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else + warn("unknown flake output '%s'", name); + + } catch (Error & e) { + e.addTrace(pos, hintfmt("while checking flake output '%s'", name)); + reportError(e); + } + }); + } + + if (build && !drvPaths.empty()) { + Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); + store->buildPaths(drvPaths); + } + if (hasErrors) + throw Error("Some errors were encountered during the evaluation"); + } +}; + +struct CmdFlakeInitCommon : virtual Args, EvalCommand +{ + std::string templateUrl = "templates"; + Path destDir; + + const Strings attrsPathPrefixes{"templates."}; + const LockFlags lockFlags{ .writeLockFile = false }; + + CmdFlakeInitCommon() + { + addFlag({ + .longName = "template", + .shortName = 't', + .description = "The template to use.", + .labels = {"template"}, + .handler = {&templateUrl}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRefWithFragment( + getEvalState(), + lockFlags, + attrsPathPrefixes, + {"defaultTemplate"}, + prefix); + }} + }); + } + + void run(nix::ref<nix::Store> store) override + { + auto flakeDir = absPath(destDir); + + auto evalState = getEvalState(); + + auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); + + auto installable = InstallableFlake(nullptr, + evalState, std::move(templateFlakeRef), + Strings{templateName == "" ? "defaultTemplate" : templateName}, + Strings(attrsPathPrefixes), lockFlags); + + auto [cursor, attrPath] = installable.getCursor(*evalState); + + auto templateDir = cursor->getAttr("path")->getString(); + + assert(store->isInStore(templateDir)); + + std::vector<Path> files; + + std::function<void(const Path & from, const Path & to)> copyDir; + copyDir = [&](const Path & from, const Path & to) + { + createDirs(to); + + for (auto & entry : readDirectory(from)) { + auto from2 = from + "/" + entry.name; + auto to2 = to + "/" + entry.name; + auto st = lstat(from2); + if (S_ISDIR(st.st_mode)) + copyDir(from2, to2); + else if (S_ISREG(st.st_mode)) { + auto contents = readFile(from2); + if (pathExists(to2)) { + auto contents2 = readFile(to2); + if (contents != contents2) + throw Error("refusing to overwrite existing file '%s'", to2); + } else + writeFile(to2, contents); + } + else if (S_ISLNK(st.st_mode)) { + auto target = readLink(from2); + if (pathExists(to2)) { + if (readLink(to2) != target) + throw Error("refusing to overwrite existing symlink '%s'", to2); + } else + createSymlink(target, to2); + } + else + throw Error("file '%s' has unsupported type", from2); + files.push_back(to2); + } + }; + + copyDir(templateDir, flakeDir); + + if (pathExists(flakeDir + "/.git")) { + Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; + for (auto & s : files) args.push_back(s); + runProgram("git", true, args); + } + } +}; + +struct CmdFlakeInit : CmdFlakeInitCommon +{ + std::string description() override + { + return "create a flake in the current directory from a template"; + } + + std::string doc() override + { + return + #include "flake-init.md" + ; + } + + CmdFlakeInit() + { + destDir = "."; + } +}; + +struct CmdFlakeNew : CmdFlakeInitCommon +{ + std::string description() override + { + return "create a flake in the specified directory from a template"; + } + + std::string doc() override + { + return + #include "flake-new.md" + ; + } + + CmdFlakeNew() + { + expectArgs({ + .label = "dest-dir", + .handler = {&destDir}, + .completer = completePath + }); + } +}; + +struct CmdFlakeClone : FlakeCommand +{ + Path destDir; + + std::string description() override + { + return "clone flake repository"; + } + + std::string doc() override + { + return + #include "flake-clone.md" + ; + } + + CmdFlakeClone() + { + addFlag({ + .longName = "dest", + .shortName = 'f', + .description = "Clone the flake to path *dest*.", + .labels = {"path"}, + .handler = {&destDir} + }); + } + + void run(nix::ref<nix::Store> store) override + { + if (destDir.empty()) + throw Error("missing flag '--dest'"); + + getFlakeRef().resolve(store).input.clone(destDir); + } +}; + +struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun +{ + std::string dstUri; + + CmdFlakeArchive() + { + addFlag({ + .longName = "to", + .description = "URI of the destination Nix store", + .labels = {"store-uri"}, + .handler = {&dstUri} + }); + } + + std::string description() override + { + return "copy a flake and all its inputs to a store"; + } + + std::string doc() override + { + return + #include "flake-archive.md" + ; + } + + void run(nix::ref<nix::Store> store) override + { + auto flake = lockFlake(); + + auto jsonRoot = json ? std::optional<JSONObject>(std::cout) : std::nullopt; + + StorePathSet sources; + + sources.insert(flake.flake.sourceInfo->storePath); + if (jsonRoot) + jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath)); + + // FIXME: use graph output, handle cycles. + std::function<void(const Node & node, std::optional<JSONObject> & jsonObj)> traverse; + traverse = [&](const Node & node, std::optional<JSONObject> & jsonObj) + { + auto jsonObj2 = jsonObj ? jsonObj->object("inputs") : std::optional<JSONObject>(); + for (auto & [inputName, input] : node.inputs) { + if (auto inputNode = std::get_if<0>(&input)) { + auto jsonObj3 = jsonObj2 ? jsonObj2->object(inputName) : std::optional<JSONObject>(); + auto storePath = + dryRun + ? (*inputNode)->lockedRef.input.computeStorePath(*store) + : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + if (jsonObj3) + jsonObj3->attr("path", store->printStorePath(storePath)); + sources.insert(std::move(storePath)); + traverse(**inputNode, jsonObj3); + } + } + }; + + traverse(*flake.lockFile.root, jsonRoot); + + if (!dryRun && !dstUri.empty()) { + ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + copyPaths(*store, *dstStore, sources); + } + } +}; + +struct CmdFlakeShow : FlakeCommand, MixJSON +{ + bool showLegacy = false; + + CmdFlakeShow() + { + addFlag({ + .longName = "legacy", + .description = "Show the contents of the `legacyPackages` output.", + .handler = {&showLegacy, true} + }); + } + + std::string description() override + { + return "show the outputs provided by a flake"; + } + + std::string doc() override + { + return + #include "flake-show.md" + ; + } + + void run(nix::ref<nix::Store> store) override + { + auto state = getEvalState(); + auto flake = std::make_shared<LockedFlake>(lockFlake()); + + std::function<nlohmann::json( + eval_cache::AttrCursor & visitor, + const std::vector<Symbol> & attrPath, + const std::string & headerPrefix, + const std::string & nextPrefix)> visit; + + visit = [&]( + eval_cache::AttrCursor & visitor, + const std::vector<Symbol> & attrPath, + const std::string & headerPrefix, + const std::string & nextPrefix) + -> nlohmann::json + { + auto j = nlohmann::json::object(); + + Activity act(*logger, lvlInfo, actUnknown, + fmt("evaluating '%s'", concatStringsSep(".", attrPath))); + try { + auto recurse = [&]() + { + if (!json) + logger->cout("%s", headerPrefix); + auto attrs = visitor.getAttrs(); + for (const auto & [i, attr] : enumerate(attrs)) { + bool last = i + 1 == attrs.size(); + auto visitor2 = visitor.getAttr(attr); + auto attrPath2(attrPath); + attrPath2.push_back(attr); + auto j2 = visit(*visitor2, attrPath2, + fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, nextPrefix, last ? treeLast : treeConn, attr), + nextPrefix + (last ? treeNull : treeLine)); + if (json) j.emplace(attr, std::move(j2)); + } + }; + + auto showDerivation = [&]() + { + auto name = visitor.getAttr(state->sName)->getString(); + if (json) { + std::optional<std::string> description; + if (auto aMeta = visitor.maybeGetAttr("meta")) { + if (auto aDescription = aMeta->maybeGetAttr("description")) + description = aDescription->getString(); + } + j.emplace("type", "derivation"); + j.emplace("name", name); + if (description) + j.emplace("description", *description); + } else { + logger->cout("%s: %s '%s'", + headerPrefix, + attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" : + attrPath.size() >= 2 && attrPath[0] == "devShells" ? "development environment" : + attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" : + attrPath.size() >= 1 && attrPath[0] == "hydraJobs" ? "derivation" : + "package", + name); + } + }; + + if (attrPath.size() == 0 + || (attrPath.size() == 1 && ( + attrPath[0] == "defaultPackage" + || attrPath[0] == "devShell" + || attrPath[0] == "nixosConfigurations" + || attrPath[0] == "nixosModules" + || attrPath[0] == "defaultApp" + || attrPath[0] == "templates" + || attrPath[0] == "overlays")) + || ((attrPath.size() == 1 || attrPath.size() == 2) + && (attrPath[0] == "checks" + || attrPath[0] == "packages" + || attrPath[0] == "devShells" + || attrPath[0] == "apps")) + ) + { + recurse(); + } + + else if ( + (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell")) + || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages" || attrPath[0] == "devShells")) + ) + { + if (visitor.isDerivation()) + showDerivation(); + else + throw Error("expected a derivation"); + } + + else if (attrPath.size() > 0 && attrPath[0] == "hydraJobs") { + if (visitor.isDerivation()) + showDerivation(); + else + recurse(); + } + + else if (attrPath.size() > 0 && attrPath[0] == "legacyPackages") { + if (attrPath.size() == 1) + recurse(); + else if (!showLegacy) + logger->warn(fmt("%s: " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); + else { + if (visitor.isDerivation()) + showDerivation(); + else if (attrPath.size() <= 2) + // FIXME: handle recurseIntoAttrs + recurse(); + } + } + + else if ( + (attrPath.size() == 2 && attrPath[0] == "defaultApp") || + (attrPath.size() == 3 && attrPath[0] == "apps")) + { + auto aType = visitor.maybeGetAttr("type"); + if (!aType || aType->getString() != "app") + throw EvalError("not an app definition"); + if (json) { + j.emplace("type", "app"); + } else { + logger->cout("%s: app", headerPrefix); + } + } + + else if ( + (attrPath.size() == 1 && attrPath[0] == "defaultTemplate") || + (attrPath.size() == 2 && attrPath[0] == "templates")) + { + auto description = visitor.getAttr("description")->getString(); + if (json) { + j.emplace("type", "template"); + j.emplace("description", description); + } else { + logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); + } + } + + else { + auto [type, description] = + (attrPath.size() == 1 && attrPath[0] == "overlay") + || (attrPath.size() == 2 && attrPath[0] == "overlays") ? std::make_pair("nixpkgs-overlay", "Nixpkgs overlay") : + attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? std::make_pair("nixos-configuration", "NixOS configuration") : + attrPath.size() == 2 && attrPath[0] == "nixosModules" ? std::make_pair("nixos-module", "NixOS module") : + std::make_pair("unknown", "unknown"); + if (json) { + j.emplace("type", type); + } else { + logger->cout("%s: " ANSI_WARNING "%s" ANSI_NORMAL, headerPrefix, description); + } + } + } catch (EvalError & e) { + if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) + throw; + } + + return j; + }; + + auto cache = openEvalCache(*state, flake); + + auto j = visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); + if (json) + logger->cout("%s", j.dump()); + } +}; + +struct CmdFlakePrefetch : FlakeCommand, MixJSON +{ + CmdFlakePrefetch() + { + } + + std::string description() override + { + return "download the source tree denoted by a flake reference into the Nix store"; + } + + std::string doc() override + { + return + #include "flake-prefetch.md" + ; + } + + void run(ref<Store> store) override + { + auto originalRef = getFlakeRef(); + auto resolvedRef = originalRef.resolve(store); + auto [tree, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(tree.storePath)->narHash; + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(tree.storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + lockedRef.to_string(), + store->printStorePath(tree.storePath), + hash.to_string(SRI, true)); + } + } +}; + +struct CmdFlake : NixMultiCommand +{ + CmdFlake() + : MultiCommand({ + {"update", []() { return make_ref<CmdFlakeUpdate>(); }}, + {"lock", []() { return make_ref<CmdFlakeLock>(); }}, + {"metadata", []() { return make_ref<CmdFlakeMetadata>(); }}, + {"info", []() { return make_ref<CmdFlakeInfo>(); }}, + {"check", []() { return make_ref<CmdFlakeCheck>(); }}, + {"init", []() { return make_ref<CmdFlakeInit>(); }}, + {"new", []() { return make_ref<CmdFlakeNew>(); }}, + {"clone", []() { return make_ref<CmdFlakeClone>(); }}, + {"archive", []() { return make_ref<CmdFlakeArchive>(); }}, + {"show", []() { return make_ref<CmdFlakeShow>(); }}, + {"prefetch", []() { return make_ref<CmdFlakePrefetch>(); }}, + }) + { + } + + std::string description() override + { + return "manage Nix flakes"; + } + + std::string doc() override + { + return + #include "flake.md" + ; + } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + settings.requireExperimentalFeature("flakes"); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdFlake = registerCommand<CmdFlake>("flake"); diff --git a/src/nix/flake.md b/src/nix/flake.md new file mode 100644 index 000000000..3d273100b --- /dev/null +++ b/src/nix/flake.md @@ -0,0 +1,566 @@ +R""( + +# Description + +`nix flake` provides subcommands for creating, modifying and querying +*Nix flakes*. Flakes are the unit for packaging Nix code in a +reproducible and discoverable way. They can have dependencies on other +flakes, making it possible to have multi-repository Nix projects. + +A flake is a filesystem tree (typically fetched from a Git repository +or a tarball) that contains a file named `flake.nix` in the root +directory. `flake.nix` specifies some metadata about the flake such as +dependencies (called *inputs*), as well as its *outputs* (the Nix +values such as packages or NixOS modules provided by the flake). + +# Flake references + +Flake references (*flakerefs*) are a way to specify the location of a +flake. These have two different forms: + +* An attribute set representation, e.g. + + ```nix + { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + } + ``` + + The only required attribute is `type`. The supported types are + listed below. + +* A URL-like syntax, e.g. + + ``` + github:NixOS/nixpkgs + ``` + + These are used on the command line as a more convenient alternative + to the attribute set representation. For instance, in the command + + ```console + # nix build github:NixOS/nixpkgs#hello + ``` + + `github:NixOS/nixpkgs` is a flake reference (while `hello` is an + output attribute). They are also allowed in the `inputs` attribute + of a flake, e.g. + + ```nix + inputs.nixpkgs.url = github:NixOS/nixpkgs; + ``` + + is equivalent to + + ```nix + inputs.nixpkgs = { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + }; + ``` + +## Examples + +Here are some examples of flake references in their URL-like representation: + +* `.`: The flake in the current directory. +* `/home/alice/src/patchelf`: A flake in some other directory. +* `nixpkgs`: The `nixpkgs` entry in the flake registry. +* `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` + entry in the flake registry, with its Git revision overridden to a + specific value. +* `github:NixOS/nixpkgs`: The `master` branch of the `NixOS/nixpkgs` + repository on GitHub. +* `github:NixOS/nixpkgs/nixos-20.09`: The `nixos-20.09` branch of the + `nixpkgs` repository. +* `github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: A + specific revision of the `nixpkgs` repository. +* `github:edolstra/nix-warez?dir=blender`: A flake in a subdirectory + of a GitHub repository. +* `git+https://github.com/NixOS/patchelf`: A Git repository. +* `git+https://github.com/NixOS/patchelf?ref=master`: A specific + branch of a Git repository. +* `git+https://github.com/NixOS/patchelf?ref=master&rev=f34751b88bd07d7f44f5cd3200fb4122bf916c7e`: + A specific branch *and* revision of a Git repository. +* `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball + flake. + +## Flake reference attributes + +The following generic flake reference attributes are supported: + +* `dir`: The subdirectory of the flake in which `flake.nix` is + located. This parameter enables having multiple flakes in a + repository or tarball. The default is the root directory of the + flake. + +* `narHash`: The hash of the NAR serialisation (in SRI format) of the + contents of the flake. This is useful for flake types such as + tarballs that lack a unique content identifier such as a Git commit + hash. + +In addition, the following attributes are common to several flake +reference types: + +* `rev`: A Git or Mercurial commit hash. + +* `ref`: A Git or Mercurial branch or tag name. + +Finally, some attribute are typically not specified by the user, but +can occur in *locked* flake references and are available to Nix code: + +* `revCount`: The number of ancestors of the commit `rev`. + +* `lastModified`: The timestamp (in seconds since the Unix epoch) of + the last modification of this version of the flake. For + Git/Mercurial flakes, this is the commit time of commit *rev*, while + for tarball flakes, it's the most recent timestamp of any file + inside the tarball. + +## Types + +Currently the `type` attribute can be one of the following: + +* `path`: arbitrary local directories, or local Git trees. The + required attribute `path` specifies the path of the flake. The URL + form is + + ``` + [path:]<path>(\?<params)? + ``` + + where *path* is an absolute path. + + *path* must be a directory in the file system containing a file + named `flake.nix`. + + If the directory or any of its parents is a Git repository, then + this is essentially equivalent to `git+file://<path>` (see below), + except that the `dir` parameter is derived automatically. For + example, if `/foo/bar` is a Git repository, then the flake reference + `/foo/bar/flake` is equivalent to `/foo/bar?dir=flake`. + + If the directory is not inside a Git repository, then the flake + contents is the entire contents of *path*. + + *path* generally must be an absolute path. However, on the command + line, it can be a relative path (e.g. `.` or `./foo`) which is + interpreted as relative to the current directory. In this case, it + must start with `.` to avoid ambiguity with registry lookups + (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative + path). + +* `git`: Git repositories. The location of the repository is specified + by the attribute `url`. + + They have the URL form + + ``` + git(+http|+https|+ssh|+git|+file|):(//<server>)?<path>(\?<params>)? + ``` + + The `ref` attribute defaults to `master`. + + The `rev` attribute must denote a commit that exists in the branch + or tag specified by the `ref` attribute, since Nix doesn't do a full + clone of the remote repository by default (and the Git protocol + doesn't allow fetching a `rev` without a known `ref`). The default + is the commit currently pointed to by `ref`. + + For example, the following are valid Git flake references: + + * `git+https://example.org/my/repo` + * `git+https://example.org/my/repo?dir=flake1` + * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` + * `git://github.com/edolstra/dwarffs?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4` + * `git+file:///home/my-user/some-repo/some-repo` + +* `mercurial`: Mercurial repositories. The URL form is similar to the + `git` type, except that the URL schema must be one of `hg+http`, + `hg+https`, `hg+ssh` or `hg+file`. + +* `tarball`: Tarballs. The location of the tarball is specified by the + attribute `url`. + + In URL form, the schema must be `http://`, `https://` or `file://` + URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz`, + `.tar.bz2` or `.tar.zst`. + +* `github`: A more efficient way to fetch repositories from + GitHub. The following attributes are required: + + * `owner`: The owner of the repository. + + * `repo`: The name of the repository. + + These are downloaded as tarball archives, rather than + through Git. This is often much faster and uses less disk space + since it doesn't require fetching the entire history of the + repository. On the other hand, it doesn't allow incremental fetching + (but full downloads are often faster than incremental fetches!). + + The URL syntax for `github` flakes is: + + ``` + github:<owner>/<repo>(/<rev-or-ref>)?(\?<params>)? + ``` + + `<rev-or-ref>` specifies the name of a branch or tag (`ref`), or a + commit hash (`rev`). Note that unlike Git, GitHub allows fetching by + commit hash without specifying a branch or tag. + + Some examples: + + * `github:edolstra/dwarffs` + * `github:edolstra/dwarffs/unstable` + * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31` + +* `indirect`: Indirections through the flake registry. These have the + form + + ``` + [flake:]<flake-id>(/<rev-or-ref>(/rev)?)? + ``` + + These perform a lookup of `<flake-id>` in the flake registry. or + example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake + references. The specified `rev` and/or `ref` are merged with the + entry in the registry; see [nix registry](./nix3-registry.md) for + details. + +# Flake format + +As an example, here is a simple `flake.nix` that depends on the +Nixpkgs flake and provides a single package (i.e. an installable +derivation): + +```nix +{ + description = "A flake for building Hello World"; + + inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03; + + outputs = { self, nixpkgs }: { + + defaultPackage.x86_64-linux = + # Notice the reference to nixpkgs here. + with import nixpkgs { system = "x86_64-linux"; }; + stdenv.mkDerivation { + name = "hello"; + src = self; + buildPhase = "gcc -o hello ./hello.c"; + installPhase = "mkdir -p $out/bin; install -t $out/bin hello"; + }; + + }; +} +``` + +The following attributes are supported in `flake.nix`: + +* `description`: A short, one-line description of the flake. + +* `inputs`: An attrset specifying the dependencies of the flake + (described below). + +* `outputs`: A function that, given an attribute set containing the + outputs of each of the input flakes keyed by their identifier, + yields the Nix values provided by this flake. Thus, in the example + above, `inputs.nixpkgs` contains the result of the call to the + `outputs` function of the `nixpkgs` flake. + + In addition to the outputs of each input, each input in `inputs` + also contains some metadata about the inputs. These are: + + * `outPath`: The path in the Nix store of the flake's source tree. + + * `rev`: The commit hash of the flake's repository, if applicable. + + * `revCount`: The number of ancestors of the revision `rev`. This is + not available for `github` repositories, since they're fetched as + tarballs rather than as Git repositories. + + * `lastModifiedDate`: The commit time of the revision `rev`, in the + format `%Y%m%d%H%M%S` (e.g. `20181231100934`). Unlike `revCount`, + this is available for both Git and GitHub repositories, so it's + useful for generating (hopefully) monotonically increasing version + strings. + + * `lastModified`: The commit time of the revision `rev` as an integer + denoting the number of seconds since 1970. + + * `narHash`: The SHA-256 (in SRI format) of the NAR serialization of + the flake's source tree. + + The value returned by the `outputs` function must be an attribute + set. The attributes can have arbitrary values; however, various + `nix` subcommands require specific attributes to have a specific + value (e.g. `packages.x86_64-linux` must be an attribute set of + derivations built for the `x86_64-linux` platform). + +## Flake inputs + +The attribute `inputs` specifies the dependencies of a flake, as an +attrset mapping input names to flake references. For example, the +following specifies a dependency on the `nixpkgs` and `import-cargo` +repositories: + +```nix +# A GitHub repository. +inputs.import-cargo = { + type = "github"; + owner = "edolstra"; + repo = "import-cargo"; +}; + +# An indirection through the flake registry. +inputs.nixpkgs = { + type = "indirect"; + id = "nixpkgs"; +}; +``` + +Alternatively, you can use the URL-like syntax: + +```nix +inputs.import-cargo.url = github:edolstra/import-cargo; +inputs.nixpkgs.url = "nixpkgs"; +``` + +Each input is fetched, evaluated and passed to the `outputs` function +as a set of attributes with the same name as the corresponding +input. The special input named `self` refers to the outputs and source +tree of *this* flake. Thus, a typical `outputs` function looks like +this: + +```nix +outputs = { self, nixpkgs, import-cargo }: { + ... outputs ... +}; +``` + +It is also possible to omit an input entirely and *only* list it as +expected function argument to `outputs`. Thus, + +```nix +outputs = { self, nixpkgs }: ...; +``` + +without an `inputs.nixpkgs` attribute is equivalent to + +```nix +inputs.nixpkgs = { + type = "indirect"; + id = "nixpkgs"; +}; +``` + +Repositories that don't contain a `flake.nix` can also be used as +inputs, by setting the input's `flake` attribute to `false`: + +```nix +inputs.grcov = { + type = "github"; + owner = "mozilla"; + repo = "grcov"; + flake = false; +}; + +outputs = { self, nixpkgs, grcov }: { + packages.x86_64-linux.grcov = stdenv.mkDerivation { + src = grcov; + ... + }; +}; +``` + +Transitive inputs can be overridden from a `flake.nix` file. For +example, the following overrides the `nixpkgs` input of the `nixops` +input: + +```nix +inputs.nixops.inputs.nixpkgs = { + type = "github"; + owner = "my-org"; + repo = "nixpkgs"; +}; +``` + +It is also possible to "inherit" an input from another input. This is +useful to minimize flake dependencies. For example, the following sets +the `nixpkgs` input of the top-level flake to be equal to the +`nixpkgs` input of the `dwarffs` input of the top-level flake: + +```nix +inputs.nixpkgs.follows = "dwarffs/nixpkgs"; +``` + +The value of the `follows` attribute is a `/`-separated sequence of +input names denoting the path of inputs to be followed from the root +flake. + +Overrides and `follows` can be combined, e.g. + +```nix +inputs.nixops.inputs.nixpkgs.follows = "dwarffs/nixpkgs"; +``` + +sets the `nixpkgs` input of `nixops` to be the same as the `nixpkgs` +input of `dwarffs`. It is worth noting, however, that it is generally +not useful to eliminate transitive `nixpkgs` flake inputs in this +way. Most flakes provide their functionality through Nixpkgs overlays +or NixOS modules, which are composed into the top-level flake's +`nixpkgs` input; so their own `nixpkgs` input is usually irrelevant. + +# Lock files + +Inputs specified in `flake.nix` are typically "unlocked" in the sense +that they don't specify an exact revision. To ensure reproducibility, +Nix will automatically generate and use a *lock file* called +`flake.lock` in the flake's directory. The lock file contains a graph +structure isomorphic to the graph of dependencies of the root +flake. Each node in the graph (except the root node) maps the +(usually) unlocked input specifications in `flake.nix` to locked input +specifications. Each node also contains some metadata, such as the +dependencies (outgoing edges) of the node. + +For example, if `flake.nix` has the inputs in the example above, then +the resulting lock file might be: + +```json +{ + "version": 7, + "root": "n1", + "nodes": { + "n1": { + "inputs": { + "nixpkgs": "n2", + "import-cargo": "n3", + "grcov": "n4" + } + }, + "n2": { + "inputs": {}, + "locked": { + "owner": "edolstra", + "repo": "nixpkgs", + "rev": "7f8d4b088e2df7fdb6b513bc2d6941f1d422a013", + "type": "github", + "lastModified": 1580555482, + "narHash": "sha256-OnpEWzNxF/AU4KlqBXM2s5PWvfI5/BS6xQrPvkF5tO8=" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "n3": { + "inputs": {}, + "locked": { + "owner": "edolstra", + "repo": "import-cargo", + "rev": "8abf7b3a8cbe1c8a885391f826357a74d382a422", + "type": "github", + "lastModified": 1567183309, + "narHash": "sha256-wIXWOpX9rRjK5NDsL6WzuuBJl2R0kUCnlpZUrASykSc=" + }, + "original": { + "owner": "edolstra", + "repo": "import-cargo", + "type": "github" + } + }, + "n4": { + "inputs": {}, + "locked": { + "owner": "mozilla", + "repo": "grcov", + "rev": "989a84bb29e95e392589c4e73c29189fd69a1d4e", + "type": "github", + "lastModified": 1580729070, + "narHash": "sha256-235uMxYlHxJ5y92EXZWAYEsEb6mm+b069GAd+BOIOxI=" + }, + "original": { + "owner": "mozilla", + "repo": "grcov", + "type": "github" + }, + "flake": false + } + } +} +``` + +This graph has 4 nodes: the root flake, and its 3 dependencies. The +nodes have arbitrary labels (e.g. `n1`). The label of the root node of +the graph is specified by the `root` attribute. Nodes contain the +following fields: + +* `inputs`: The dependencies of this node, as a mapping from input + names (e.g. `nixpkgs`) to node labels (e.g. `n2`). + +* `original`: The original input specification from `flake.lock`, as a + set of `builtins.fetchTree` arguments. + +* `locked`: The locked input specification, as a set of + `builtins.fetchTree` arguments. Thus, in the example above, when we + build this flake, the input `nixpkgs` is mapped to revision + `7f8d4b088e2df7fdb6b513bc2d6941f1d422a013` of the `edolstra/nixpkgs` + repository on GitHub. + + It also includes the attribute `narHash`, specifying the expected + contents of the tree in the Nix store (as computed by `nix + hash-path`), and may include input-type-specific attributes such as + the `lastModified` or `revCount`. The main reason for these + attributes is to allow flake inputs to be substituted from a binary + cache: `narHash` allows the store path to be computed, while the + other attributes are necessary because they provide information not + stored in the store path. + +* `flake`: A Boolean denoting whether this is a flake or non-flake + dependency. Corresponds to the `flake` attribute in the `inputs` + attribute in `flake.nix`. + +The `original` and `locked` attributes are omitted for the root +node. This is because we cannot record the commit hash or content hash +of the root flake, since modifying `flake.lock` will invalidate these. + +The graph representation of lock files allows circular dependencies +between flakes. For example, here are two flakes that reference each +other: + +```nix +{ + inputs.b = ... location of flake B ...; + # Tell the 'b' flake not to fetch 'a' again, to ensure its 'a' is + # *this* 'a'. + inputs.b.inputs.a.follows = ""; + outputs = { self, b }: { + foo = 123 + b.bar; + xyzzy = 1000; + }; +} +``` + +and + +```nix +{ + inputs.a = ... location of flake A ...; + inputs.a.inputs.b.follows = ""; + outputs = { self, a }: { + bar = 456 + a.xyzzy; + }; +} +``` + +Lock files transitively lock direct as well as indirect +dependencies. That is, if a lock file exists and is up to date, Nix +will not look at the lock files of dependencies. However, lock file +generation itself *does* use the lock files of dependencies by +default. + +)"" diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index a25ec43a9..42c806450 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,9 +1,130 @@ set -e if [ -e .attrs.sh ]; then source .attrs.sh; fi + export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 + if [[ -n $stdenv ]]; then source $stdenv/setup fi -export > $out -set >> $out + +# Better to use compgen, but stdenv bash doesn't have it. +__vars="$(declare -p)" +__functions="$(declare -F)" + +__dumpEnv() { + printf '{\n' + + printf ' "bashFunctions": {\n' + local __first=1 + while read __line; do + if ! [[ $__line =~ ^declare\ -f\ (.*) ]]; then continue; fi + __fun_name="${BASH_REMATCH[1]}" + __fun_body="$(type $__fun_name)" + if [[ $__fun_body =~ \{(.*)\} ]]; then + if [[ -z $__first ]]; then printf ',\n'; else __first=; fi + __fun_body="${BASH_REMATCH[1]}" + printf " " + __escapeString "$__fun_name" + printf ':' + __escapeString "$__fun_body" + else + printf "Cannot parse definition of function '%s'.\n" "$__fun_name" >&2 + return 1 + fi + done < <(printf "%s\n" "$__functions") + printf '\n },\n' + + printf ' "variables": {\n' + local __first=1 + while read __line; do + if ! [[ $__line =~ ^declare\ (-[^ ])\ ([^=]*) ]]; then continue; fi + local type="${BASH_REMATCH[1]}" + local __var_name="${BASH_REMATCH[2]}" + + if [[ $__var_name =~ ^BASH_ || \ + $__var_name = _ || \ + $__var_name = DIRSTACK || \ + $__var_name = EUID || \ + $__var_name = FUNCNAME || \ + $__var_name = HISTCMD || \ + $__var_name = HOSTNAME || \ + $__var_name = GROUPS || \ + $__var_name = PIPESTATUS || \ + $__var_name = PWD || \ + $__var_name = RANDOM || \ + $__var_name = SHLVL || \ + $__var_name = SECONDS \ + ]]; then continue; fi + + if [[ -z $__first ]]; then printf ',\n'; else __first=; fi + + printf " " + __escapeString "$__var_name" + printf ': {' + + # FIXME: handle -i, -r, -n. + if [[ $type == -x ]]; then + printf '"type": "exported", "value": ' + __escapeString "${!__var_name}" + elif [[ $type == -- ]]; then + printf '"type": "var", "value": ' + __escapeString "${!__var_name}" + elif [[ $type == -a ]]; then + printf '"type": "array", "value": [' + local __first2=1 + __var_name="$__var_name[@]" + for __i in "${!__var_name}"; do + if [[ -z $__first2 ]]; then printf ', '; else __first2=; fi + __escapeString "$__i" + printf ' ' + done + printf ']' + elif [[ $type == -A ]]; then + printf '"type": "associative", "value": {\n' + local __first2=1 + declare -n __var_name2="$__var_name" + for __i in "${!__var_name2[@]}"; do + if [[ -z $__first2 ]]; then printf ',\n'; else __first2=; fi + printf " " + __escapeString "$__i" + printf ": " + __escapeString "${__var_name2[$__i]}" + done + printf '\n }' + else + printf '"type": "unknown"' + fi + + printf "}" + done < <(printf "%s\n" "$__vars") + printf '\n }\n}' +} + +__escapeString() { + local __s="$1" + __s="${__s//\\/\\\\}" + __s="${__s//\"/\\\"}" + __s="${__s//$'\n'/\\n}" + __s="${__s//$'\r'/\\r}" + __s="${__s//$'\t'/\\t}" + printf '"%s"' "$__s" +} + +# In case of `__structuredAttrs = true;` the list of outputs is an associative +# array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` +# must contain the array's keys (hence `${!...[@]}`) in this case. +if [ -e .attrs.sh ]; then + __olist="${!outputs[@]}" +else + __olist=$outputs +fi + +for __output in $__olist; do + if [[ -z $__done ]]; then + __dumpEnv > ${!__output} + __done=1 + else + echo -n >> "${!__output}" + fi +done diff --git a/src/nix/hash.cc b/src/nix/hash.cc index b97c6d21f..4535e4ab0 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -8,7 +8,7 @@ using namespace nix; -struct CmdHash : Command +struct CmdHashBase : Command { FileIngestionMethod mode; Base base = SRI; @@ -17,37 +17,62 @@ struct CmdHash : Command std::vector<std::string> paths; std::optional<std::string> modulus; - CmdHash(FileIngestionMethod mode) : mode(mode) + CmdHashBase(FileIngestionMethod mode) : mode(mode) { - mkFlag(0, "sri", "print hash in SRI format", &base, SRI); - mkFlag(0, "base64", "print hash in base-64", &base, Base64); - mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); - mkFlag(0, "base16", "print hash in base-16", &base, Base16); + addFlag({ + .longName = "sri", + .description = "Print the hash in SRI format.", + .handler = {&base, SRI}, + }); + + addFlag({ + .longName = "base64", + .description = "Print the hash in base-64 format.", + .handler = {&base, Base64}, + }); + + addFlag({ + .longName = "base32", + .description = "Print the hash in base-32 (Nix-specific) format.", + .handler = {&base, Base32}, + }); + + addFlag({ + .longName = "base16", + .description = "Print the hash in base-16 format.", + .handler = {&base, Base16}, + }); + addFlag(Flag::mkHashTypeFlag("type", &ht)); + #if 0 - mkFlag() - .longName("modulo") - .description("compute hash modulo specified string") - .labels({"modulus"}) - .dest(&modulus); - #endif - expectArgs("paths", &paths); + addFlag({ + .longName = "modulo", + .description = "Compute the hash modulo the specified string.", + .labels = {"modulus"}, + .handler = {&modulus}, + }); + #endif\ + + expectArgs({ + .label = "paths", + .handler = {&paths}, + .completer = completePath + }); } std::string description() override { - const char* d; switch (mode) { case FileIngestionMethod::Flat: - d = "print cryptographic hash of a regular file"; + return "print cryptographic hash of a regular file"; case FileIngestionMethod::Recursive: - d = "print cryptographic hash of the NAR serialisation of a path"; + return "print cryptographic hash of the NAR serialisation of a path"; + default: + assert(false); }; - return d; } - Category category() override { return catUtility; } - void run() override { for (auto path : paths) { @@ -69,14 +94,11 @@ struct CmdHash : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->stdout(h.to_string(base, base == SRI)); + logger->cout(h.to_string(base, base == SRI)); } } }; -static RegisterCommand r1("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); }); -static RegisterCommand r2("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); }); - struct CmdToBase : Command { Base base; @@ -98,19 +120,43 @@ struct CmdToBase : Command "SRI"); } + void run() override + { + for (auto s : args) + logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI)); + } +}; + +struct CmdHash : NixMultiCommand +{ + CmdHash() + : MultiCommand({ + {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }}, + {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }}, + {"to-base16", []() { return make_ref<CmdToBase>(Base16); }}, + {"to-base32", []() { return make_ref<CmdToBase>(Base32); }}, + {"to-base64", []() { return make_ref<CmdToBase>(Base64); }}, + {"to-sri", []() { return make_ref<CmdToBase>(SRI); }}, + }) + { } + + std::string description() override + { + return "compute and convert cryptographic hashes"; + } + Category category() override { return catUtility; } void run() override { - for (auto s : args) - logger->stdout(Hash(s, ht).to_string(base, base == SRI)); + if (!command) + throw UsageError("'nix hash' requires a sub-command."); + command->second->prepare(); + command->second->run(); } }; -static RegisterCommand r3("to-base16", [](){ return make_ref<CmdToBase>(Base16); }); -static RegisterCommand r4("to-base32", [](){ return make_ref<CmdToBase>(Base32); }); -static RegisterCommand r5("to-base64", [](){ return make_ref<CmdToBase>(Base64); }); -static RegisterCommand r6("to-sri", [](){ return make_ref<CmdToBase>(SRI); }); +static auto rCmdHash = registerCommand<CmdHash>("hash"); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) @@ -144,7 +190,7 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); + CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); cmd.ht = ht; cmd.base = base32 ? Base32 : Base16; cmd.truncate = truncate; @@ -162,4 +208,4 @@ static int compatNixHash(int argc, char * * argv) return 0; } -static RegisterLegacyCommand s1("nix-hash", compatNixHash); +static RegisterLegacyCommand r_nix_hash("nix-hash", compatNixHash); diff --git a/src/nix/help.md b/src/nix/help.md new file mode 100644 index 000000000..734f35028 --- /dev/null +++ b/src/nix/help.md @@ -0,0 +1,17 @@ +R""( + +# Examples + +* Show help about `nix` in general: + + ```console + # nix help + ``` + +* Show help about a particular subcommand: + + ```console + # nix help flake info + ``` + +)"" diff --git a/src/nix/installables.cc b/src/nix/installables.cc deleted file mode 100644 index 708a0dc88..000000000 --- a/src/nix/installables.cc +++ /dev/null @@ -1,346 +0,0 @@ -#include "command.hh" -#include "attr-path.hh" -#include "common-eval-args.hh" -#include "derivations.hh" -#include "eval-inline.hh" -#include "eval.hh" -#include "get-drvs.hh" -#include "store-api.hh" -#include "shared.hh" - -#include <regex> - -namespace nix { - - -SourceExprCommand::SourceExprCommand() -{ - addFlag({ - .longName = "file", - .shortName = 'f', - .description = "evaluate FILE rather than the default", - .labels = {"file"}, - .handler = {&file} - }); -} - -Value * SourceExprCommand::getSourceExpr(EvalState & state) -{ - if (vSourceExpr) return *vSourceExpr; - - auto sToplevel = state.symbols.create("_toplevel"); - - vSourceExpr = allocRootValue(state.allocValue()); - - if (file != "") - state.evalFile(lookupFileArg(state, file), **vSourceExpr); - - else { - - /* Construct the installation source from $NIX_PATH. */ - - auto searchPath = state.getSearchPath(); - - state.mkAttrs(**vSourceExpr, 1024); - - mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true); - - std::unordered_set<std::string> seen; - - auto addEntry = [&](const std::string & name) { - if (name == "") return; - if (!seen.insert(name).second) return; - Value * v1 = state.allocValue(); - mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); - Value * v2 = state.allocValue(); - mkApp(*v2, *v1, mkString(*state.allocValue(), name)); - mkApp(*state.allocAttr(**vSourceExpr, state.symbols.create(name)), - state.getBuiltin("import"), *v2); - }; - - for (auto & i : searchPath) - /* Hack to handle channels. */ - if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { - for (auto & j : readDirectory(i.second)) - if (j.name != "manifest.nix" - && pathExists(fmt("%s/%s/default.nix", i.second, j.name))) - addEntry(j.name); - } else - addEntry(i.first); - - (*vSourceExpr)->attrs->sort(); - } - - return *vSourceExpr; -} - -ref<EvalState> SourceExprCommand::getEvalState() -{ - if (!evalState) - evalState = std::make_shared<EvalState>(searchPath, getStore()); - return ref<EvalState>(evalState); -} - -Buildable Installable::toBuildable() -{ - auto buildables = toBuildables(); - if (buildables.size() != 1) - throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); - return std::move(buildables[0]); -} - -struct InstallableStorePath : Installable -{ - ref<Store> store; - StorePath storePath; - - InstallableStorePath(ref<Store> store, const Path & storePath) - : store(store), storePath(store->parseStorePath(storePath)) { } - - std::string what() override { return store->printStorePath(storePath); } - - Buildables toBuildables() override - { - std::map<std::string, StorePath> outputs; - outputs.insert_or_assign("out", storePath); - Buildable b{ - .drvPath = storePath.isDerivation() ? storePath : std::optional<StorePath>(), - .outputs = std::move(outputs) - }; - Buildables bs; - bs.push_back(std::move(b)); - return bs; - } - - std::optional<StorePath> getStorePath() override - { - return storePath; - } -}; - -struct InstallableValue : Installable -{ - SourceExprCommand & cmd; - - InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } - - Buildables toBuildables() override - { - auto state = cmd.getEvalState(); - - auto v = toValue(*state).first; - - Bindings & autoArgs = *cmd.getAutoArgs(*state); - - DrvInfos drvs; - getDerivations(*state, *v, "", autoArgs, drvs, false); - - Buildables res; - - StorePathSet drvPaths; - - for (auto & drv : drvs) { - Buildable b{.drvPath = state->store->parseStorePath(drv.queryDrvPath())}; - drvPaths.insert(*b.drvPath); - - auto outputName = drv.queryOutputName(); - if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); - - b.outputs.emplace(outputName, state->store->parseStorePath(drv.queryOutPath())); - - res.push_back(std::move(b)); - } - - // Hack to recognize .all: if all drvs have the same drvPath, - // merge the buildables. - if (drvPaths.size() == 1) { - Buildable b{.drvPath = *drvPaths.begin()}; - for (auto & b2 : res) - for (auto & output : b2.outputs) - b.outputs.insert_or_assign(output.first, output.second); - Buildables bs; - bs.push_back(std::move(b)); - return bs; - } else - return res; - } -}; - -struct InstallableExpr : InstallableValue -{ - std::string text; - - InstallableExpr(SourceExprCommand & cmd, const std::string & text) - : InstallableValue(cmd), text(text) { } - - std::string what() override { return text; } - - std::pair<Value *, Pos> toValue(EvalState & state) override - { - auto v = state.allocValue(); - state.eval(state.parseExprFromString(text, absPath(".")), *v); - return {v, noPos}; - } -}; - -struct InstallableAttrPath : InstallableValue -{ - std::string attrPath; - - InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath) - : InstallableValue(cmd), attrPath(attrPath) - { } - - std::string what() override { return attrPath; } - - std::pair<Value *, Pos> toValue(EvalState & state) override - { - auto source = cmd.getSourceExpr(state); - - Bindings & autoArgs = *cmd.getAutoArgs(state); - - auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first; - state.forceValue(*v); - - return {v, noPos}; - } -}; - -// FIXME: extend -std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; -static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); - -static std::vector<std::shared_ptr<Installable>> parseInstallables( - SourceExprCommand & cmd, ref<Store> store, std::vector<std::string> ss, bool useDefaultInstallables) -{ - std::vector<std::shared_ptr<Installable>> result; - - if (ss.empty() && useDefaultInstallables) { - if (cmd.file == "") - cmd.file = "."; - ss = {""}; - } - - for (auto & s : ss) { - - if (s.compare(0, 1, "(") == 0) - result.push_back(std::make_shared<InstallableExpr>(cmd, s)); - - else if (s.find("/") != std::string::npos) { - - auto path = store->toStorePath(store->followLinksToStore(s)); - - if (store->isStorePath(path)) - result.push_back(std::make_shared<InstallableStorePath>(store, path)); - } - - else if (s == "" || std::regex_match(s, attrPathRegex)) - result.push_back(std::make_shared<InstallableAttrPath>(cmd, s)); - - else - throw UsageError("don't know what to do with argument '%s'", s); - } - - return result; -} - -std::shared_ptr<Installable> parseInstallable( - SourceExprCommand & cmd, ref<Store> store, const std::string & installable, - bool useDefaultInstallables) -{ - auto installables = parseInstallables(cmd, store, {installable}, false); - assert(installables.size() == 1); - return installables.front(); -} - -Buildables build(ref<Store> store, RealiseMode mode, - std::vector<std::shared_ptr<Installable>> installables) -{ - if (mode != Build) - settings.readOnlyMode = true; - - Buildables buildables; - - std::vector<StorePathWithOutputs> pathsToBuild; - - for (auto & i : installables) { - for (auto & b : i->toBuildables()) { - if (b.drvPath) { - StringSet outputNames; - for (auto & output : b.outputs) - outputNames.insert(output.first); - pathsToBuild.push_back({*b.drvPath, outputNames}); - } else - for (auto & output : b.outputs) - pathsToBuild.push_back({output.second}); - buildables.push_back(std::move(b)); - } - } - - if (mode == DryRun) - printMissing(store, pathsToBuild, lvlError); - else if (mode == Build) - store->buildPaths(pathsToBuild); - - return buildables; -} - -StorePathSet toStorePaths(ref<Store> store, RealiseMode mode, - std::vector<std::shared_ptr<Installable>> installables) -{ - StorePathSet outPaths; - - for (auto & b : build(store, mode, installables)) - for (auto & output : b.outputs) - outPaths.insert(output.second); - - return outPaths; -} - -StorePath toStorePath(ref<Store> store, RealiseMode mode, - std::shared_ptr<Installable> installable) -{ - auto paths = toStorePaths(store, mode, {installable}); - - if (paths.size() != 1) - throw Error("argument '%s' should evaluate to one store path", installable->what()); - - return *paths.begin(); -} - -StorePathSet toDerivations(ref<Store> store, - std::vector<std::shared_ptr<Installable>> installables, bool useDeriver) -{ - StorePathSet drvPaths; - - for (auto & i : installables) - for (auto & b : i->toBuildables()) { - if (!b.drvPath) { - if (!useDeriver) - throw Error("argument '%s' did not evaluate to a derivation", i->what()); - for (auto & output : b.outputs) { - auto derivers = store->queryValidDerivers(output.second); - if (derivers.empty()) - throw Error("'%s' does not have a known deriver", i->what()); - // FIXME: use all derivers? - drvPaths.insert(*derivers.begin()); - } - } else - drvPaths.insert(*b.drvPath); - } - - return drvPaths; -} - -void InstallablesCommand::prepare() -{ - installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables()); -} - -void InstallableCommand::prepare() -{ - installable = parseInstallable(*this, getStore(), _installable, false); -} - -} diff --git a/src/nix/installables.hh b/src/nix/installables.hh deleted file mode 100644 index 503984220..000000000 --- a/src/nix/installables.hh +++ /dev/null @@ -1,45 +0,0 @@ -#pragma once - -#include "util.hh" -#include "path.hh" -#include "eval.hh" - -#include <optional> - -namespace nix { - -struct Buildable -{ - std::optional<StorePath> drvPath; - std::map<std::string, StorePath> outputs; -}; - -typedef std::vector<Buildable> Buildables; - -struct Installable -{ - virtual ~Installable() { } - - virtual std::string what() = 0; - - virtual Buildables toBuildables() - { - throw Error("argument '%s' cannot be built", what()); - } - - Buildable toBuildable(); - - virtual std::pair<Value *, Pos> toValue(EvalState & state) - { - throw Error("argument '%s' cannot be evaluated", what()); - } - - /* Return a value only if this installable is a store path or a - symlink to it. */ - virtual std::optional<StorePath> getStorePath() - { - return {}; - } -}; - -} diff --git a/src/nix/key-convert-secret-to-public.md b/src/nix/key-convert-secret-to-public.md new file mode 100644 index 000000000..3adc18502 --- /dev/null +++ b/src/nix/key-convert-secret-to-public.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Convert a secret key to a public key: + + ```console + # echo cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== \ + | nix key convert-secret-to-public + cache.example.org-0:tVeHVMaFaKi8lieGBErmScEOuxlSJLA7eO6IPrT1VGc= + ``` + +# Description + +This command reads a Ed25519 secret key from standard input, and +writes the corresponding public key to standard output. For more +details, see [nix key generate-secret](./nix3-key-generate-secret.md). + +)"" diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md new file mode 100644 index 000000000..4938f637c --- /dev/null +++ b/src/nix/key-generate-secret.md @@ -0,0 +1,48 @@ +R""( + +# Examples + +* Generate a new secret key: + + ```console + # nix key generate-secret --key-name cache.example.org-1 > ./secret-key + ``` + + We can then use this key to sign the closure of the Hello package: + + ```console + # nix build nixpkgs#hello + # nix store sign --key-file ./secret-key --recursive ./result + ``` + + Finally, we can verify the store paths using the corresponding + public key: + + ``` + # nix store verify --trusted-public-keys $(nix key convert-secret-to-public < ./secret-key) ./result + ``` + +# Description + +This command generates a new Ed25519 secret key for signing store +paths and prints it on standard output. Use `nix key +convert-secret-to-public` to get the corresponding public key for +verifying signed store paths. + +The mandatory argument `--key-name` specifies a key name (such as +`cache.example.org-1). It is used to look up keys on the client when +it verifies signatures. It can be anything, but it’s suggested to use +the host name of your cache (e.g. `cache.example.org`) with a suffix +denoting the number of the key (to be incremented every time you need +to revoke a key). + +# Format + +Both secret and public keys are represented as the key name followed +by a base-64 encoding of the Ed25519 key data, e.g. + +``` +cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== +``` + +)"" diff --git a/src/nix/legacy.cc b/src/nix/legacy.cc deleted file mode 100644 index 6df09ee37..000000000 --- a/src/nix/legacy.cc +++ /dev/null @@ -1,7 +0,0 @@ -#include "legacy.hh" - -namespace nix { - -RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0; - -} diff --git a/src/nix/legacy.hh b/src/nix/legacy.hh deleted file mode 100644 index f503b0da3..000000000 --- a/src/nix/legacy.hh +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include <functional> -#include <map> -#include <string> - -namespace nix { - -typedef std::function<void(int, char * *)> MainFunction; - -struct RegisterLegacyCommand -{ - typedef std::map<std::string, MainFunction> Commands; - static Commands * commands; - - RegisterLegacyCommand(const std::string & name, MainFunction fun) - { - if (!commands) commands = new Commands; - (*commands)[name] = fun; - } -}; - -} diff --git a/src/nix/local.mk b/src/nix/local.mk index b057b7cc6..e4ec7634d 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -12,14 +12,13 @@ nix_SOURCES := \ $(wildcard src/nix-daemon/*.cc) \ $(wildcard src/nix-env/*.cc) \ $(wildcard src/nix-instantiate/*.cc) \ - $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ -nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain +nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -I src/libcmd -I doc/manual -nix_LIBS = libexpr libmain libfetchers libstore libutil +nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd -nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ @@ -29,3 +28,7 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh src/nix/develop.cc: src/nix/get-env.sh.gen.hh + +src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh + +src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh diff --git a/src/nix/log.cc b/src/nix/log.cc index 3fe22f6c2..962c47525 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -13,22 +13,11 @@ struct CmdLog : InstallableCommand return "show the build log of the specified packages or paths, if available"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get the build log of GNU Hello:", - "nix log nixpkgs.hello" - }, - Example{ - "To get the build log of a specific path:", - "nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1" - }, - Example{ - "To get a build log from a specific binary cache:", - "nix log --store https://cache.nixos.org nixpkgs.hello" - }, - }; + return + #include "log.md" + ; } Category category() override { return catSecondary; } @@ -41,15 +30,18 @@ struct CmdLog : InstallableCommand subs.push_front(store); - auto b = installable->toBuildable(); + auto b = installable->toDerivedPath(); RunPager pager; for (auto & sub : subs) { - auto log = b.drvPath ? sub->getBuildLog(*b.drvPath) : nullptr; - for (auto & output : b.outputs) { - if (log) break; - log = sub->getBuildLog(output.second); - } + auto log = std::visit(overloaded { + [&](DerivedPath::Opaque bo) { + return sub->getBuildLog(bo.path); + }, + [&](DerivedPath::Built bfd) { + return sub->getBuildLog(bfd.drvPath); + }, + }, b.raw()); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); @@ -61,4 +53,4 @@ struct CmdLog : InstallableCommand } }; -static auto r1 = registerCommand<CmdLog>("log"); +static auto rCmdLog = registerCommand<CmdLog>("log"); diff --git a/src/nix/log.md b/src/nix/log.md new file mode 100644 index 000000000..1c76226a3 --- /dev/null +++ b/src/nix/log.md @@ -0,0 +1,40 @@ +R""( + +# Examples + +* Get the build log of GNU Hello: + + ```console + # nix log nixpkgs#hello + ``` + +* Get the build log of a specific store path: + + ```console + # nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1 + ``` + +* Get a build log from a specific binary cache: + + ```console + # nix log --store https://cache.nixos.org nixpkgs#hello + ``` + +# Description + +This command prints the log of a previous build of the derivation +*installable* on standard output. + +Nix looks for build logs in two places: + +* In the directory `/nix/var/log/nix/drvs`, which contains logs for + locally built derivations. + +* In the binary caches listed in the `substituters` setting. Logs + should be named `<cache>/log/<base-name-of-store-path>`, where + `store-path` is a derivation, + e.g. `https://cache.nixos.org/log/dvmig8jgrdapvbyxb1rprckdmdqx08kv-hello-2.10.drv`. + For non-derivation store paths, Nix will first try to determine the + deriver by fetching the `.narinfo` file for this store path. + +)"" diff --git a/src/nix/ls.cc b/src/nix/ls.cc index d2157f2d4..c1dc9a95b 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -17,9 +17,26 @@ struct MixLs : virtual Args, MixJSON MixLs() { - mkFlag('R', "recursive", "list subdirectories recursively", &recursive); - mkFlag('l', "long", "show more file information", &verbose); - mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); + addFlag({ + .longName = "recursive", + .shortName = 'R', + .description = "List subdirectories recursively.", + .handler = {&recursive, true}, + }); + + addFlag({ + .longName = "long", + .shortName = 'l', + .description = "Show detailed file information.", + .handler = {&verbose, true}, + }); + + addFlag({ + .longName = "directory", + .shortName = 'd', + .description = "Show directories rather than their contents.", + .handler = {&showDirectory, true}, + }); } void listText(ref<FSAccessor> accessor) @@ -37,11 +54,11 @@ struct MixLs : virtual Args, MixJSON auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); if (st.type == FSAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); - logger->stdout(line); + logger->cout(line); if (recursive && st.type == FSAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { - logger->stdout(relPath); + logger->cout(relPath); if (recursive) { auto st = accessor->stat(curPath); if (st.type == FSAccessor::Type::tDirectory) @@ -75,6 +92,8 @@ struct MixLs : virtual Args, MixJSON if (json) { JSONPlaceholder jsonRoot(std::cout); + if (showDirectory) + throw UsageError("'--directory' is useless with '--json'"); listNar(jsonRoot, accessor, path, recursive); } else listText(accessor); @@ -85,17 +104,11 @@ struct CmdLsStore : StoreCommand, MixLs { CmdLsStore() { - expectArg("path", &path); - } - - Examples examples() override - { - return { - Example{ - "To list the contents of a store path in a binary cache:", - "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" - }, - }; + expectArgs({ + .label = "path", + .handler = {&path}, + .completer = completePath + }); } std::string description() override @@ -103,7 +116,12 @@ struct CmdLsStore : StoreCommand, MixLs return "show information about a path in the Nix store"; } - Category category() override { return catUtility; } + std::string doc() override + { + return + #include "store-ls.md" + ; + } void run(ref<Store> store) override { @@ -117,18 +135,19 @@ struct CmdLsNar : Command, MixLs CmdLsNar() { - expectArg("nar", &narPath); + expectArgs({ + .label = "nar", + .handler = {&narPath}, + .completer = completePath + }); expectArg("path", &path); } - Examples examples() override + std::string doc() override { - return { - Example{ - "To list a specific file in a NAR:", - "nix ls-nar -l hello.nar /bin/hello" - }, - }; + return + #include "nar-ls.md" + ; } std::string description() override @@ -136,13 +155,11 @@ struct CmdLsNar : Command, MixLs return "show information about a path inside a NAR file"; } - Category category() override { return catUtility; } - void run() override { list(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); } }; -static auto r1 = registerCommand<CmdLsStore>("ls-store"); -static auto r2 = registerCommand<CmdLsNar>("ls-nar"); +static auto rCmdLsStore = registerCommand2<CmdLsStore>({"store", "ls"}); +static auto rCmdLsNar = registerCommand2<CmdLsNar>({"nar", "ls"}); diff --git a/src/nix/main.cc b/src/nix/main.cc index 203901168..8aaf08813 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -7,10 +7,10 @@ #include "legacy.hh" #include "shared.hh" #include "store-api.hh" -#include "progress-bar.hh" #include "filetransfer.hh" #include "finally.hh" #include "loggers.hh" +#include "markdown.hh" #include <sys/types.h> #include <sys/socket.h> @@ -18,6 +18,8 @@ #include <netdb.h> #include <netinet/in.h> +#include <nlohmann/json.hpp> + extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); @@ -51,14 +53,18 @@ static bool haveInternet() } std::string programPath; +char * * savedArgv; + +struct HelpRequested { }; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { bool printBuildLogs = false; bool useNet = true; bool refresh = false; + bool showVersion = false; - NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") + NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { categories.clear(); categories[Command::catDefault] = "Main commands"; @@ -68,83 +74,174 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "help", - .description = "show usage information", - .handler = {[&]() { showHelpAndExit(); }}, - }); - - addFlag({ - .longName = "help-config", - .description = "show configuration options", - .handler = {[&]() { - std::cout << "The following configuration options are available:\n\n"; - Table2 tbl; - std::map<std::string, Config::SettingInfo> settings; - globalConfig.getSettings(settings); - for (const auto & s : settings) - tbl.emplace_back(s.first, s.second.description); - printTable(std::cout, tbl); - throw Exit(); - }}, + .description = "Show usage information.", + .handler = {[&]() { throw HelpRequested(); }}, }); addFlag({ .longName = "print-build-logs", .shortName = 'L', - .description = "print full build logs on stderr", + .description = "Print full build logs on standard error.", + .category = loggingCategory, .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, }); addFlag({ .longName = "version", - .description = "show version information", - .handler = {[&]() { printVersion(programName); }}, + .description = "Show version information.", + .handler = {[&]() { showVersion = true; }}, }); addFlag({ - .longName = "no-net", - .description = "disable substituters and consider all previously downloaded files up-to-date", + .longName = "offline", + .aliases = {"no-net"}, // FIXME: remove + .description = "Disable substituters and consider all previously downloaded files up-to-date.", .handler = {[&]() { useNet = false; }}, }); addFlag({ .longName = "refresh", - .description = "consider all previously downloaded files out-of-date", + .description = "Consider all previously downloaded files out-of-date.", .handler = {[&]() { refresh = true; }}, }); + } + + std::map<std::string, std::vector<std::string>> aliases = { + {"add-to-store", {"store", "add-path"}}, + {"cat-nar", {"nar", "cat"}}, + {"cat-store", {"store", "cat"}}, + {"copy-sigs", {"store", "copy-sigs"}}, + {"dev-shell", {"develop"}}, + {"diff-closures", {"store", "diff-closures"}}, + {"dump-path", {"store", "dump-path"}}, + {"hash-file", {"hash", "file"}}, + {"hash-path", {"hash", "path"}}, + {"ls-nar", {"nar", "ls"}}, + {"ls-store", {"store", "ls"}}, + {"make-content-addressable", {"store", "make-content-addressable"}}, + {"optimise-store", {"store", "optimise"}}, + {"ping-store", {"store", "ping"}}, + {"sign-paths", {"store", "sign"}}, + {"to-base16", {"hash", "to-base16"}}, + {"to-base32", {"hash", "to-base32"}}, + {"to-base64", {"hash", "to-base64"}}, + {"verify", {"store", "verify"}}, + }; + + bool aliasUsed = false; + + Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) override + { + if (aliasUsed || command || pos == args.end()) return pos; + auto arg = *pos; + auto i = aliases.find(arg); + if (i == aliases.end()) return pos; + warn("'%s' is a deprecated alias for '%s'", + arg, concatStringsSep(" ", i->second)); + pos = args.erase(pos); + for (auto j = i->second.rbegin(); j != i->second.rend(); ++j) + pos = args.insert(pos, *j); + aliasUsed = true; + return pos; + } - deprecatedAliases.insert({"dev-shell", "develop"}); + std::string description() override + { + return "a tool for reproducible and declarative configuration management"; } - void printFlags(std::ostream & out) override + std::string doc() override { - Args::printFlags(out); - std::cout << - "\n" - "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n" - "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n" - "--help-config' for a list of configuration settings.\n"; + return + #include "nix.md" + ; } - void printHelp(const string & programName, std::ostream & out) override + // Plugins may add new subcommands. + void pluginsInited() override { - MultiCommand::printHelp(programName, out); + commands = RegisterCommand::getCommandsFor({}); + } +}; + +/* Render the help for the specified subcommand to stdout using + lowdown. */ +static void showHelp(std::vector<std::string> subcommand, MultiCommand & toplevel) +{ + auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); -#if 0 - out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n"; -#endif + evalSettings.restrictEval = false; + evalSettings.pureEval = false; + EvalState state({}, openStore("dummy://")); - std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n"; + auto vGenerateManpage = state.allocValue(); + state.eval(state.parseExprFromString( + #include "generate-manpage.nix.gen.hh" + , "/"), *vGenerateManpage); + + auto vUtils = state.allocValue(); + state.cacheFile( + "/utils.nix", "/utils.nix", + state.parseExprFromString( + #include "utils.nix.gen.hh" + , "/"), + *vUtils); + + auto vJson = state.allocValue(); + mkString(*vJson, toplevel.toJSON().dump()); + + auto vRes = state.allocValue(); + state.callFunction(*vGenerateManpage, *vJson, *vRes, noPos); + + auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); + if (!attr) + throw UsageError("Nix has no subcommand '%s'", concatStringsSep("", subcommand)); + + auto markdown = state.forceString(*attr->value); + + RunPager pager; + std::cout << renderMarkdownToTerminal(markdown) << "\n"; +} + +struct CmdHelp : Command +{ + std::vector<std::string> subcommand; + + CmdHelp() + { + expectArgs({ + .label = "subcommand", + .handler = {&subcommand}, + }); } - void showHelpAndExit() + std::string description() override { - printHelp(programName, std::cout); - throw Exit(); + return "show help about `nix` or a particular subcommand"; + } + + std::string doc() override + { + return + #include "help.md" + ; + } + + void run() override + { + assert(parent); + MultiCommand * toplevel = parent; + while (toplevel->parent) toplevel = toplevel->parent; + showHelp(subcommand, *toplevel); } }; +static auto rCmdHelp = registerCommand<CmdHelp>("help"); + void mainWrapped(int argc, char * * argv) { + savedArgv = argv; + /* The chroot helper needs to be run before any threads have been started. */ if (argc > 0 && argv[0] == chrootHelperName) { @@ -163,8 +260,9 @@ void mainWrapped(int argc, char * * argv) if (legacy) return legacy(argc, argv); } - verbosity = lvlWarn; + verbosity = lvlNotice; settings.verboseBuild = false; + evalSettings.pureEval = true; setLogFormat("bar"); @@ -172,11 +270,66 @@ void mainWrapped(int argc, char * * argv) NixArgs args; - args.parseCmdline(argvToStrings(argc, argv)); + if (argc == 2 && std::string(argv[1]) == "__dump-args") { + std::cout << args.toJSON().dump() << "\n"; + return; + } + + if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { + evalSettings.pureEval = false; + EvalState state({}, openStore("dummy://")); + auto res = nlohmann::json::object(); + auto builtins = state.baseEnv.values[0]->attrs; + for (auto & builtin : *builtins) { + auto b = nlohmann::json::object(); + if (!builtin.value->isPrimOp()) continue; + auto primOp = builtin.value->primOp; + if (!primOp->doc) continue; + b["arity"] = primOp->arity; + b["args"] = primOp->args; + b["doc"] = trim(stripIndentation(primOp->doc)); + res[(std::string) builtin.name] = std::move(b); + } + std::cout << res.dump() << "\n"; + return; + } + + Finally printCompletions([&]() + { + if (completions) { + std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n"); + for (auto & s : *completions) + std::cout << s.completion << "\t" << s.description << "\n"; + } + }); - initPlugins(); + try { + args.parseCmdline(argvToStrings(argc, argv)); + } catch (HelpRequested &) { + std::vector<std::string> subcommand; + MultiCommand * command = &args; + while (command) { + if (command && command->command) { + subcommand.push_back(command->command->first); + command = dynamic_cast<MultiCommand *>(&*command->command->second); + } else + break; + } + showHelp(subcommand, args); + return; + } catch (UsageError &) { + if (!completions) throw; + } - if (!args.command) args.showHelpAndExit(); + if (completions) return; + + if (args.showVersion) { + printVersion(programName); + return; + } + + if (!args.command) + throw UsageError("no subcommand specified"); if (args.command->first != "repl" && args.command->first != "doctor" @@ -190,18 +343,21 @@ void mainWrapped(int argc, char * * argv) if (!args.useNet) { // FIXME: should check for command line overrides only. - if (!settings.useSubstitutes.overriden) + if (!settings.useSubstitutes.overridden) settings.useSubstitutes = false; - if (!settings.tarballTtl.overriden) + if (!settings.tarballTtl.overridden) settings.tarballTtl = std::numeric_limits<unsigned int>::max(); - if (!fileTransferSettings.tries.overriden) + if (!fileTransferSettings.tries.overridden) fileTransferSettings.tries = 0; - if (!fileTransferSettings.connectTimeout.overriden) + if (!fileTransferSettings.connectTimeout.overridden) fileTransferSettings.connectTimeout = 1; } - if (args.refresh) + if (args.refresh) { settings.tarballTtl = 0; + settings.ttlNegativeNarInfoCache = 0; + settings.ttlPositiveNarInfoCache = 0; + } args.command->second->prepare(); args.command->second->run(); @@ -211,6 +367,10 @@ void mainWrapped(int argc, char * * argv) int main(int argc, char * * argv) { + // Increase the default stack size for the evaluator and for + // libstdc++'s std::regex. + nix::setStackSize(64 * 1024 * 1024); + return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); }); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index fb36fc410..f5bdc7e65 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -10,30 +10,21 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON { CmdMakeContentAddressable() { - realiseMode = Build; + realiseMode = Realise::Outputs; } std::string description() override { - return "rewrite a path or closure to content-addressable form"; + return "rewrite a path or closure to content-addressed form"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To create a content-addressable representation of GNU Hello (but not its dependencies):", - "nix make-content-addressable nixpkgs.hello" - }, - Example{ - "To compute a content-addressable representation of the current NixOS system closure:", - "nix make-content-addressable -r /run/current-system" - }, - }; + return + #include "make-content-addressable.md" + ; } - Category category() override { return catUtility; } - void run(ref<Store> store, StorePaths storePaths) override { auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); @@ -73,14 +64,16 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON *sink.s = rewriteStrings(*sink.s, rewrites); HashModuloSink hashModuloSink(htSHA256, oldHashPart); - hashModuloSink((unsigned char *) sink.s->data(), sink.s->size()); + hashModuloSink(*sink.s); auto narHash = hashModuloSink.finish().first; - ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference)); + ValidPathInfo info { + store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), + narHash, + }; info.references = std::move(references); if (hasSelfReference) info.references.insert(info.path); - info.narHash = narHash; info.narSize = sink.s->size(); info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, @@ -88,11 +81,11 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON }; if (!json) - printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); + notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); auto source = sinkToSource([&](Sink & nextSink) { RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); - rsink2((unsigned char *) sink.s->data(), sink.s->size()); + rsink2(*sink.s); rsink2.flush(); }); @@ -106,4 +99,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand<CmdMakeContentAddressable>("make-content-addressable"); +static auto rCmdMakeContentAddressable = registerCommand2<CmdMakeContentAddressable>({"store", "make-content-addressable"}); diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressable.md new file mode 100644 index 000000000..3dd847edc --- /dev/null +++ b/src/nix/make-content-addressable.md @@ -0,0 +1,59 @@ +R""( + +# Examples + +* Create a content-addressed representation of the closure of GNU Hello: + + ```console + # nix store make-content-addressable -r nixpkgs#hello + … + rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10' + ``` + + Since the resulting paths are content-addressed, they are always + trusted and don't need signatures to copied to another store: + + ```console + # nix copy --to /tmp/nix --trusted-public-keys '' /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 + ``` + + By contrast, the original closure is input-addressed, so it does + need signatures to be trusted: + + ```console + # nix copy --to /tmp/nix --trusted-public-keys '' nixpkgs#hello + cannot add path '/nix/store/zy9wbxwcygrwnh8n2w9qbbcr6zk87m26-libunistring-0.9.10' because it lacks a valid signature + ``` + +* Create a content-addressed representation of the current NixOS + system closure: + + ```console + # nix store make-content-addressable -r /run/current-system + ``` + +# Description + +This command converts the closure of the store paths specified by +*installables* to content-addressed form. Nix store paths are usually +*input-addressed*, meaning that the hash part of the store path is +computed from the contents of the derivation (i.e., the build-time +dependency graph). Input-addressed paths need to be signed by a +trusted key if you want to import them into a store, because we need +to trust that the contents of the path were actually built by the +derivation. + +By contrast, in a *content-addressed* path, the hash part is computed +from the contents of the path. This allows the contents of the path to +be verified without any additional information such as +signatures. This means that a command like + +```console +# nix store build /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 \ + --substituters https://my-cache.example.org +``` + +will succeed even if the binary cache `https://my-cache.example.org` +doesn't present any signatures. + +)"" diff --git a/src/nix/nar-cat.md b/src/nix/nar-cat.md new file mode 100644 index 000000000..55c481a28 --- /dev/null +++ b/src/nix/nar-cat.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* List a file in a NAR and pipe it through `gunzip`: + + ```console + # nix nar cat ./hello.nar /share/man/man1/hello.1.gz | gunzip + .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.4. + .TH HELLO "1" "November 2014" "hello 2.10" "User Commands" + … + ``` + +# Description + +This command prints on standard output the contents of the regular +file *path* inside the NAR file *nar*. + +)"" diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md new file mode 100644 index 000000000..26191ad25 --- /dev/null +++ b/src/nix/nar-dump-path.md @@ -0,0 +1,17 @@ +R""( + +# Examples + +* To serialise directory `foo` as a NAR: + + ```console + # nix nar dump-path ./foo > foo.nar + ``` + +# Description + +This command generates a NAR file containing the serialisation of +*path*, which must contain only regular files, directories and +symbolic links. The NAR is written to standard output. + +)"" diff --git a/src/nix/nar-ls.md b/src/nix/nar-ls.md new file mode 100644 index 000000000..d373f9715 --- /dev/null +++ b/src/nix/nar-ls.md @@ -0,0 +1,24 @@ +R""( + +# Examples + +* To list a specific file in a NAR: + + ```console + # nix nar ls -l ./hello.nar /bin/hello + -r-xr-xr-x 38184 hello + ``` + +* To recursively list the contents of a directory inside a NAR, in JSON + format: + + ```console + # nix nar ls --json -R ./hello.nar /bin + {"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}} + ``` + +# Description + +This command shows information about a *path* inside NAR file *nar*. + +)"" diff --git a/src/nix/nar.cc b/src/nix/nar.cc new file mode 100644 index 000000000..dbb043d9b --- /dev/null +++ b/src/nix/nar.cc @@ -0,0 +1,33 @@ +#include "command.hh" + +using namespace nix; + +struct CmdNar : NixMultiCommand +{ + CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"})) + { } + + std::string description() override + { + return "create or inspect NAR files"; + } + + std::string doc() override + { + return + #include "nar.md" + ; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix nar' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdNar = registerCommand<CmdNar>("nar"); diff --git a/src/nix/nar.md b/src/nix/nar.md new file mode 100644 index 000000000..a83b5c764 --- /dev/null +++ b/src/nix/nar.md @@ -0,0 +1,13 @@ +R""( + +# Description + +`nix nar` provides several subcommands for creating and inspecting +*Nix Archives* (NARs). + +# File format + +For the definition of the NAR file format, see Figure 5.2 in +https://edolstra.github.io/pubs/phd-thesis.pdf. + +)"" diff --git a/src/nix/nix.md b/src/nix/nix.md new file mode 100644 index 000000000..d10de7c01 --- /dev/null +++ b/src/nix/nix.md @@ -0,0 +1,119 @@ +R""( + +# Examples + +* Create a new flake: + + ```console + # nix flake new hello + # cd hello + ``` + +* Build the flake in the current directory: + + ```console + # nix build + # ./result/bin/hello + Hello, world! + ``` + +* Run the flake in the current directory: + + ```console + # nix run + Hello, world! + ``` + +* Start a development shell for hacking on this flake: + + ```console + # nix develop + # unpackPhase + # cd hello-* + # configurePhase + # buildPhase + # ./hello + Hello, world! + # installPhase + # ../outputs/out/bin/hello + Hello, world! + ``` + +# Description + +Nix is a tool for building software, configurations and other +artifacts in a reproducible and declarative way. For more information, +see the [Nix homepage](https://nixos.org/) or the [Nix +manual](https://nixos.org/manual/nix/stable/). + +# Installables + +Many `nix` subcommands operate on one or more *installables*. These are +command line arguments that represent something that can be built in +the Nix store. Here are the recognised types of installables: + +* **Flake output attributes**: `nixpkgs#hello` + + These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a + flake reference and *attrpath* is an optional attribute path. For + more information on flakes, see [the `nix flake` manual + page](./nix3-flake.md). Flake references are most commonly a flake + identifier in the flake registry (e.g. `nixpkgs`) or a path + (e.g. `/path/to/my-flake` or `.`). + + If *attrpath* is omitted, Nix tries some default values; for most + subcommands, the default is `defaultPackage.`*system* + (e.g. `defaultPackage.x86_64-linux`), but some subcommands have + other defaults. If *attrpath* *is* specified, *attrpath* is + interpreted as relative to one or more prefixes; for most + subcommands, these are `packages.`*system*, + `legacyPackages.*system*` and the empty prefix. Thus, on + `x86_64-linux` `nix build nixpkgs#hello` will try to build the + attributes `packages.x86_64-linux.hello`, + `legacyPackages.x86_64-linux.hello` and `hello`. + +* **Store paths**: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` + + These are paths inside the Nix store, or symlinks that resolve to a + path in the Nix store. + +* **Store derivations**: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` + + Store derivations are store paths with extension `.drv` and are a + low-level representation of a build-time dependency graph used + internally by Nix. By default, if you pass a store derivation to a + `nix` subcommand, it will operate on the *output paths* of the + derivation. For example, `nix path-info` prints information about + the output paths: + + ```console + # nix path-info --json /nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv + [{"path":"/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10",…}] + ``` + + If you want to operate on the store derivation itself, pass the + `--derivation` flag. + +* **Nix attributes**: `--file /path/to/nixpkgs hello` + + When the `-f` / `--file` *path* option is given, installables are + interpreted as attribute paths referencing a value returned by + evaluating the Nix file *path*. + +* **Nix expressions**: `--expr '(import <nixpkgs> {}).hello.overrideDerivation (prev: { name = "my-hello"; })'`. + + When the `--expr` option is given, all installables are interpreted + as Nix expressions. You may need to specify `--impure` if the + expression references impure inputs (such as `<nixpkgs>`). + +For most commands, if no installable is specified, the default is `.`, +i.e. Nix will operate on the default flake output attribute of the +flake in the current directory. + +# Nix stores + +Most `nix` subcommands operate on a *Nix store*. + +TODO: list store types, options + +)"" diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index b45951879..985006e5a 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -13,22 +13,17 @@ struct CmdOptimiseStore : StoreCommand return "replace identical files in the store by hard links"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To optimise the Nix store:", - "nix optimise-store" - }, - }; + return + #include "optimise-store.md" + ; } - Category category() override { return catUtility; } - void run(ref<Store> store) override { store->optimiseStore(); } }; -static auto r1 = registerCommand<CmdOptimiseStore>("optimise-store"); +static auto rCmdOptimiseStore = registerCommand2<CmdOptimiseStore>({"store", "optimise"}); diff --git a/src/nix/optimise-store.md b/src/nix/optimise-store.md new file mode 100644 index 000000000..f6fb66f97 --- /dev/null +++ b/src/nix/optimise-store.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* Optimise the Nix store: + + ```console + nix store optimise + ``` + +# Description + +This command deduplicates the Nix store: it scans the store for +regular files with identical contents, and replaces them with hard +links to a single instance. + +Note that you can also set `auto-optimise-store` to `true` in +`nix.conf` to perform this optimisation incrementally whenever a new +path is added to the Nix store. To make this efficient, Nix maintains +a content-addressed index of all the files in the Nix store in the +directory `/nix/store/.links/`. + +)"" diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index b89a44f83..518cd5568 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -18,10 +18,32 @@ struct CmdPathInfo : StorePathsCommand, MixJSON CmdPathInfo() { - mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); - mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); - mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); - mkFlag(0, "sigs", "show signatures", &showSigs); + addFlag({ + .longName = "size", + .shortName = 's', + .description = "Print the size of the NAR serialisation of each path.", + .handler = {&showSize, true}, + }); + + addFlag({ + .longName = "closure-size", + .shortName = 'S', + .description = "Print the sum of the sizes of the NAR serialisations of the closure of each path.", + .handler = {&showClosureSize, true}, + }); + + addFlag({ + .longName = "human-readable", + .shortName = 'h', + .description = "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", + .handler = {&humanReadable, true}, + }); + + addFlag({ + .longName = "sigs", + .description = "Show signatures.", + .handler = {&showSigs, true}, + }); } std::string description() override @@ -29,39 +51,16 @@ struct CmdPathInfo : StorePathsCommand, MixJSON return "query information about store paths"; } - Category category() override { return catSecondary; } - - Examples examples() override + std::string doc() override { - return { - Example{ - "To show the closure sizes of every path in the current NixOS system closure, sorted by size:", - "nix path-info -rS /run/current-system | sort -nk2" - }, - Example{ - "To show a package's closure size and all its dependencies with human readable sizes:", - "nix path-info -rsSh nixpkgs.rust" - }, - Example{ - "To check the existence of a path in a binary cache:", - "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" - }, - Example{ - "To print the 10 most recently added paths (using --json and the jq(1) command):", - "nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'" - }, - Example{ - "To show the size of the entire Nix store:", - "nix path-info --json --all | jq 'map(.narSize) | add'" - }, - Example{ - "To show every path whose closure is bigger than 1 GB, sorted by closure size:", - "nix path-info --json --all -S | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'" - }, - }; + return + #include "path-info.md" + ; } - void printSize(unsigned long long value) + Category category() override { return catSecondary; } + + void printSize(uint64_t value) { if (!humanReadable) { std::cout << fmt("\t%11d", value); @@ -127,4 +126,4 @@ struct CmdPathInfo : StorePathsCommand, MixJSON } }; -static auto r1 = registerCommand<CmdPathInfo>("path-info"); +static auto rCmdPathInfo = registerCommand<CmdPathInfo>("path-info"); diff --git a/src/nix/path-info.md b/src/nix/path-info.md new file mode 100644 index 000000000..7a1714ba4 --- /dev/null +++ b/src/nix/path-info.md @@ -0,0 +1,94 @@ +R""( + +# Examples + +* Print the store path produced by `nixpkgs#hello`: + + ```console + # nix path-info nixpkgs#hello + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + ``` + +* Show the closure sizes of every path in the current NixOS system + closure, sorted by size: + + ```console + # nix path-info -rS /run/current-system | sort -nk2 + /nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty 96 + /nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers 112 + … + /nix/store/539jkw9a8dyry7clcv60gk6na816j7y8-etc 5783255504 + /nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65 5887562256 + ``` + +* Show a package's closure size and all its dependencies with human + readable sizes: + + ```console + # nix path-info -rsSh nixpkgs#rustc + /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7K 69.1M + /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9M 37.4M + … + ``` + +* Check the existence of a path in a binary cache: + + ```console + # nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/ + path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid + + ``` + +* Print the 10 most recently added paths (using --json and the jq(1) + command): + + ```console + # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path' + ``` + +* Show the size of the entire Nix store: + + ```console + # nix path-info --json --all | jq 'map(.narSize) | add' + 49812020936 + ``` + +* Show every path whose closure is bigger than 1 GB, sorted by closure + size: + + ```console + # nix path-info --json --all -S \ + | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' + [ + …, + [ + "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", + 5887562256 + ] + ] + ``` + +* Print the path of the store derivation produced by `nixpkgs#hello`: + + ```console + # nix path-info --derivation nixpkgs#hello + /nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv + ``` + +# Description + +This command shows information about the store paths produced by +*installables*, or about all paths in the store if you pass `--all`. + +By default, this command only prints the store paths. You can get +additional information by passing flags such as `--closure-size`, +`--size`, `--sigs` or `--json`. + +> **Warning** +> +> Note that `nix path-info` does not build or substitute the +> *installables* you specify. Thus, if the corresponding store paths +> don't already exist, this command will fail. You can use `nix build` +> to ensure that they exist. + +)"" diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 127397a29..62b645b06 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -8,25 +8,20 @@ struct CmdPingStore : StoreCommand { std::string description() override { - return "test whether a store can be opened"; + return "test whether a store can be accessed"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To test whether connecting to a remote Nix store via SSH works:", - "nix ping-store --store ssh://mac1" - }, - }; + return + #include "ping-store.md" + ; } - Category category() override { return catUtility; } - void run(ref<Store> store) override { store->connect(); } }; -static auto r1 = registerCommand<CmdPingStore>("ping-store"); +static auto rCmdPingStore = registerCommand2<CmdPingStore>({"store", "ping"}); diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md new file mode 100644 index 000000000..8c846791b --- /dev/null +++ b/src/nix/ping-store.md @@ -0,0 +1,33 @@ +R""( + +# Examples + +* Test whether connecting to a remote Nix store via SSH works: + + ```console + # nix store ping --store ssh://mac1 + ``` + +* Test whether a URL is a valid binary cache: + + ```console + # nix store ping --store https://cache.nixos.org + ``` + +* Test whether the Nix daemon is up and running: + + ```console + # nix store ping --store daemon + ``` + +# Description + +This command tests whether a particular Nix store (specified by the +argument `--store` *url*) can be accessed. What this means is +dependent on the type of the store. For instance, for an SSH store it +means that Nix can connect to the specified machine. + +If the command succeeds, Nix returns a exit code of 0 and does not +print any output. + +)"" diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc new file mode 100644 index 000000000..768d37595 --- /dev/null +++ b/src/nix/prefetch.cc @@ -0,0 +1,313 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "filetransfer.hh" +#include "finally.hh" +#include "progress-bar.hh" +#include "tarfile.hh" +#include "attr-path.hh" +#include "eval-inline.hh" +#include "legacy.hh" + +#include <nlohmann/json.hpp> + +using namespace nix; + +/* If ‘url’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +string resolveMirrorUrl(EvalState & state, string url) +{ + if (url.substr(0, 9) != "mirror://") return url; + + std::string s(url, 9); + auto p = s.find('/'); + if (p == std::string::npos) throw Error("invalid mirror URL '%s'", url); + std::string mirrorName(s, 0, p); + + Value vMirrors; + // FIXME: use nixpkgs flake + state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error("unknown mirror name '%s'", mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error("mirror URL '%s' did not expand to anything", url); + + auto mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} + +std::tuple<StorePath, Hash> prefetchFile( + ref<Store> store, + std::string_view url, + std::optional<std::string> name, + HashType hashType, + std::optional<Hash> expectedHash, + bool unpack, + bool executable) +{ + auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + + /* Figure out a name in the Nix store. */ + if (!name) { + name = baseNameOf(url); + if (name->empty()) + throw Error("cannot figure out file name for '%s'", url); + } + + std::optional<StorePath> storePath; + std::optional<Hash> hash; + + /* If an expected hash is given, the file may already exist in + the store. */ + if (expectedHash) { + hashType = expectedHash->type; + storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + if (store->isValidPath(*storePath)) + hash = expectedHash; + else + storePath.reset(); + } + + if (!storePath) { + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/tmp"; + + /* Download the file. */ + { + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + FileTransferRequest req(url); + req.decompress = false; + getFileTransfer()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("unpacking '%s'", url)); + Path unpacked = (Path) tmpDir + "/unpacked"; + createDirs(unpacked); + unpackTarfile(tmpFile, unpacked); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + Activity act(*logger, lvlChatty, actUnknown, + fmt("adding '%s' to the store", url)); + + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); + } + + return {storePath.value(), hash.value()}; +} + +static int main_nix_prefetch_url(int argc, char * * argv) +{ + { + HashType ht = htSHA256; + std::vector<string> args; + bool printPath = getEnv("PRINT_PATH") == "1"; + bool fromExpr = false; + string attrPath; + bool unpack = false; + bool executable = false; + std::optional<std::string> name; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + } + else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } + else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--executable") + executable = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + if (args.size() > 2) + throw UsageError("too many arguments"); + + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + + auto store = openStore(); + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + /* If -A is given, get the URL from the specified Nix + expression. */ + string url; + if (!fromExpr) { + if (args.empty()) + throw UsageError("you must specify a URL"); + url = args[0]; + } else { + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); + state->forceAttrs(v); + + /* Extract the URL. */ + auto & attr = v.attrs->need(state->symbols.create("urls")); + state->forceList(*attr.value); + if (attr.value->listSize() < 1) + throw Error("'urls' list is empty"); + url = state->forceString(*attr.value->listElems()[0]); + + /* Extract the hash mode. */ + auto attr2 = v.attrs->get(state->symbols.create("outputHashMode")); + if (!attr2) + printInfo("warning: this does not look like a fetchurl call"); + else + unpack = state->forceString(*attr2->value) == "recursive"; + + /* Extract the name. */ + if (!name) { + auto attr3 = v.attrs->get(state->symbols.create("name")); + if (!attr3) + name = state->forceString(*attr3->value); + } + } + + std::optional<Hash> expectedHash; + if (args.size() == 2) + expectedHash = Hash::parseAny(args[1], ht); + + auto [storePath, hash] = prefetchFile( + store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + + stopProgressBar(); + + if (!printPath) + printInfo("path is '%s'", store->printStorePath(storePath)); + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) + std::cout << store->printStorePath(storePath) << std::endl; + + return 0; + } +} + +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); + +struct CmdStorePrefetchFile : StoreCommand, MixJSON +{ + std::string url; + bool executable = false; + std::optional<std::string> name; + HashType hashType = htSHA256; + std::optional<Hash> expectedHash; + + CmdStorePrefetchFile() + { + addFlag({ + .longName = "name", + .description = "Override the name component of the resulting store path. It defaults to the base name of *url*.", + .labels = {"name"}, + .handler = {&name} + }); + + addFlag({ + .longName = "expected-hash", + .description = "The expected hash of the file.", + .labels = {"hash"}, + .handler = {[&](std::string s) { + expectedHash = Hash::parseAny(s, hashType); + }} + }); + + addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + + addFlag({ + .longName = "executable", + .description = + "Make the resulting file executable. Note that this causes the " + "resulting hash to be a NAR hash rather than a flat file hash.", + .handler = {&executable, true}, + }); + + expectArg("url", &url); + } + + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } + void run(ref<Store> store) override + { + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + url, + store->printStorePath(storePath), + hash.to_string(SRI, true)); + } + } +}; + +static auto rCmdStorePrefetchFile = registerCommand2<CmdStorePrefetchFile>({"store", "prefetch-file"}); diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md new file mode 100644 index 000000000..2aad491de --- /dev/null +++ b/src/nix/print-dev-env.md @@ -0,0 +1,50 @@ +R""( + +# Examples + +* Apply the build environment of GNU hello to the current shell: + + ```console + # . <(nix print-dev-env nixpkgs#hello) + ``` + +* Get the build environment in JSON format: + + ```console + # nix print-dev-env nixpkgs#hello --json + ``` + + The output will look like this: + + ```json + { + "bashFunctions": { + "buildPhase": " \n runHook preBuild;\n...", + ... + }, + "variables": { + "src": { + "type": "exported", + "value": "/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz" + }, + "postUnpackHooks": { + "type": "array", + "value": ["_updateSourceDateEpochFromSourceRoot"] + }, + ... + } + } + ``` + +# Description + +This command prints a shell script that can be sourced by `bash` and +that sets the variables and shell functions defined by the build +process of *installable*. This allows you to get a similar build +environment in your current shell rather than in a subshell (as with +`nix develop`). + +With `--json`, the output is a JSON serialisation of the variables and +functions defined by the build process. + +)"" diff --git a/src/nix/profile-diff-closures.md b/src/nix/profile-diff-closures.md new file mode 100644 index 000000000..295d1252b --- /dev/null +++ b/src/nix/profile-diff-closures.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Show what changed between each version of the NixOS system + profile: + + ```console + # nix profile diff-closures --profile /nix/var/nix/profiles/system + Version 13 -> 14: + acpi-call: 2020-04-07-5.8.13 → 2020-04-07-5.8.14 + aws-sdk-cpp: -6723.1 KiB + … + + Version 14 -> 15: + acpi-call: 2020-04-07-5.8.14 → 2020-04-07-5.8.16 + attica: -996.2 KiB + breeze-icons: -78713.5 KiB + brotli: 1.0.7 → 1.0.9, +44.2 KiB + ``` + +# Description + +This command shows the difference between the closures of subsequent +versions of a profile. See [`nix store +diff-closures`](nix3-store-diff-closures.md) for details. + +)"" diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md new file mode 100644 index 000000000..f0bfe5037 --- /dev/null +++ b/src/nix/profile-history.md @@ -0,0 +1,26 @@ +R""( + +# Examples + +* Show the changes between each version of your default profile: + + ```console + # nix profile history + Version 508 (2020-04-10): + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13 + + Version 509 (2020-05-16) <- 508: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211 + ``` + +# Description + +This command shows what packages were added, removed or upgraded +between subsequent versions of a profile. It only shows top-level +packages, not dependencies; for that, use [`nix profile +diff-closures`](./nix3-profile-diff-closures.md). + +The addition of a package to a profile is denoted by the string `∅ ->` +*version*, whereas the removal is denoted by *version* `-> ∅`. + +)"" diff --git a/src/nix/profile-install.md b/src/nix/profile-install.md new file mode 100644 index 000000000..e3009491e --- /dev/null +++ b/src/nix/profile-install.md @@ -0,0 +1,27 @@ +R""( + +# Examples + +* Install a package from Nixpkgs: + + ```console + # nix profile install nixpkgs#hello + ``` + +* Install a package from a specific branch of Nixpkgs: + + ```console + # nix profile install nixpkgs/release-20.09#hello + ``` + +* Install a package from a specific revision of Nixpkgs: + + ```console + # nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello + ``` + +# Description + +This command adds *installables* to a Nix profile. + +)"" diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md new file mode 100644 index 000000000..5c29c0b02 --- /dev/null +++ b/src/nix/profile-list.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Show what packages are installed in the default profile: + + ```console + # nix profile list + 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 + 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 + 2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 + ``` + +# Description + +This command shows what packages are currently installed in a +profile. The output consists of one line per package, with the +following fields: + +* An integer that can be used to unambiguously identify the package in + invocations of `nix profile remove` and `nix profile upgrade`. + +* The original ("mutable") flake reference and output attribute path + used at installation time. + +* The immutable flake reference to which the mutable flake reference + was resolved. + +* The store path(s) of the package. + +)"" diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md new file mode 100644 index 000000000..ba85441d8 --- /dev/null +++ b/src/nix/profile-remove.md @@ -0,0 +1,33 @@ +R""( + +# Examples + +* Remove a package by position: + + ```console + # nix profile remove 3 + ``` + +* Remove a package by attribute path: + + ```console + # nix profile remove packages.x86_64-linux.hello + ``` + +* Remove all packages: + + ```console + # nix profile remove '.*' + ``` + +* Remove a package by store path: + + ```console + # nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10 + ``` + +# Description + +This command removes a package from a profile. + +)"" diff --git a/src/nix/profile-rollback.md b/src/nix/profile-rollback.md new file mode 100644 index 000000000..6bb75aa5e --- /dev/null +++ b/src/nix/profile-rollback.md @@ -0,0 +1,26 @@ +R""( + +# Examples + +* Roll back your default profile to the previous version: + + ```console + # nix profile rollback + switching profile from version 519 to 518 + ``` + +* Switch your default profile to version 510: + + ```console + # nix profile rollback --to 510 + switching profile from version 518 to 510 + ``` + +# Description + +This command switches a profile to the most recent version older +than the currently active version, or if `--to` *N* is given, to +version *N* of the profile. To see the available versions of a +profile, use `nix profile history`. + +)"" diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md new file mode 100644 index 000000000..e06e74abe --- /dev/null +++ b/src/nix/profile-upgrade.md @@ -0,0 +1,41 @@ +R""( + +# Examples + +* Upgrade all packages that were installed using a mutable flake + reference: + + ```console + # nix profile upgrade '.*' + ``` + +* Upgrade a specific package: + + ```console + # nix profile upgrade packages.x86_64-linux.hello + ``` + +* Upgrade a specific profile element by number: + + ```console + # nix profile list + 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … + + # nix profile upgrade 0 + ``` + +# Description + +This command upgrades a previously installed package in a Nix profile, +by fetching and evaluating the latest version of the flake from which +the package was installed. + +> **Warning** +> +> This only works if you used a *mutable* flake reference at +> installation time, e.g. `nixpkgs#hello`. It does not work if you +> used an *immutable* flake reference +> (e.g. `github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a#hello`), +> since in that case the "latest version" is always the same. + +)"" diff --git a/src/nix/profile-wipe-history.md b/src/nix/profile-wipe-history.md new file mode 100644 index 000000000..b4b262864 --- /dev/null +++ b/src/nix/profile-wipe-history.md @@ -0,0 +1,20 @@ +R""( + +# Examples + +* Delete all versions of the default profile older than 100 days: + + ```console + # nix profile wipe-history --profile /tmp/profile --older-than 100d + removing profile version 515 + removing profile version 514 + ``` + +# Description + +This command deletes non-current versions of a profile, making it +impossible to roll back to these versions. By default, all non-current +versions are deleted. With `--older-than` *N*`d`, all non-current +versions older than *N* days are deleted. + +)"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc new file mode 100644 index 000000000..a1cb3fc76 --- /dev/null +++ b/src/nix/profile.cc @@ -0,0 +1,654 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "archive.hh" +#include "builtins/buildenv.hh" +#include "flake/flakeref.hh" +#include "../nix-env/user-env.hh" +#include "profiles.hh" +#include "names.hh" + +#include <nlohmann/json.hpp> +#include <regex> +#include <iomanip> + +using namespace nix; + +struct ProfileElementSource +{ + FlakeRef originalRef; + // FIXME: record original attrpath. + FlakeRef resolvedRef; + std::string attrPath; + // FIXME: output names + + bool operator < (const ProfileElementSource & other) const + { + return + std::pair(originalRef.to_string(), attrPath) < + std::pair(other.originalRef.to_string(), other.attrPath); + } +}; + +struct ProfileElement +{ + StorePathSet storePaths; + std::optional<ProfileElementSource> source; + bool active = true; + // FIXME: priority + + std::string describe() const + { + if (source) + return fmt("%s#%s", source->originalRef, source->attrPath); + StringSet names; + for (auto & path : storePaths) + names.insert(DrvName(path.name()).name); + return concatStringsSep(", ", names); + } + + std::string versions() const + { + StringSet versions; + for (auto & path : storePaths) + versions.insert(DrvName(path.name()).version); + return showVersions(versions); + } + + bool operator < (const ProfileElement & other) const + { + return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); + } +}; + +struct ProfileManifest +{ + std::vector<ProfileElement> elements; + + ProfileManifest() { } + + ProfileManifest(EvalState & state, const Path & profile) + { + auto manifestPath = profile + "/manifest.json"; + + if (pathExists(manifestPath)) { + auto json = nlohmann::json::parse(readFile(manifestPath)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + + for (auto & e : json["elements"]) { + ProfileElement element; + for (auto & p : e["storePaths"]) + element.storePaths.insert(state.store->parseStorePath((std::string) p)); + element.active = e["active"]; + if (e.value("uri", "") != "") { + element.source = ProfileElementSource{ + parseFlakeRef(e["originalUri"]), + parseFlakeRef(e["uri"]), + e["attrPath"] + }; + } + elements.emplace_back(std::move(element)); + } + } + + else if (pathExists(profile + "/manifest.nix")) { + // FIXME: needed because of pure mode; ugly. + if (state.allowedPaths) { + state.allowedPaths->insert(state.store->followLinksToStore(profile)); + state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix")); + } + + auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + + for (auto & drvInfo : drvInfos) { + ProfileElement element; + element.storePaths = {state.store->parseStorePath(drvInfo.queryOutPath())}; + elements.emplace_back(std::move(element)); + } + } + } + + std::string toJSON(Store & store) const + { + auto array = nlohmann::json::array(); + for (auto & element : elements) { + auto paths = nlohmann::json::array(); + for (auto & path : element.storePaths) + paths.push_back(store.printStorePath(path)); + nlohmann::json obj; + obj["storePaths"] = paths; + obj["active"] = element.active; + if (element.source) { + obj["originalUri"] = element.source->originalRef.to_string(); + obj["uri"] = element.source->resolvedRef.to_string(); + obj["attrPath"] = element.source->attrPath; + } + array.push_back(obj); + } + nlohmann::json json; + json["version"] = 1; + json["elements"] = array; + return json.dump(); + } + + StorePath build(ref<Store> store) + { + auto tempDir = createTempDir(); + + StorePathSet references; + + Packages pkgs; + for (auto & element : elements) { + for (auto & path : element.storePaths) { + if (element.active) + pkgs.emplace_back(store->printStorePath(path), true, 5); + references.insert(path); + } + } + + buildProfile(tempDir, std::move(pkgs)); + + writeFile(tempDir + "/manifest.json", toJSON(*store)); + + /* Add the symlink tree to the store. */ + StringSink sink; + dumpPath(tempDir, sink); + + auto narHash = hashString(htSHA256, *sink.s); + + ValidPathInfo info { + store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "profile", references), + narHash, + }; + info.references = std::move(references); + info.narSize = sink.s->size(); + info.ca = FixedOutputHash { .method = FileIngestionMethod::Recursive, .hash = info.narHash }; + + auto source = StringSource { *sink.s }; + store->addToStore(info, source); + + return std::move(info.path); + } + + static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent) + { + auto prevElems = prev.elements; + std::sort(prevElems.begin(), prevElems.end()); + + auto curElems = cur.elements; + std::sort(curElems.begin(), curElems.end()); + + auto i = prevElems.begin(); + auto j = curElems.begin(); + + bool changes = false; + + while (i != prevElems.end() || j != curElems.end()) { + if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) { + std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions()); + changes = true; + ++j; + } + else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) { + std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions()); + changes = true; + ++i; + } + else { + auto v1 = i->versions(); + auto v2 = j->versions(); + if (v1 != v2) { + std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2); + changes = true; + } + ++i; + ++j; + } + } + + if (!changes) + std::cout << fmt("%sNo changes.\n", indent); + } +}; + +struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile +{ + std::string description() override + { + return "install a package into a profile"; + } + + std::string doc() override + { + return + #include "profile-install.md" + ; + } + + void run(ref<Store> store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + std::vector<DerivedPath> pathsToBuild; + + for (auto & installable : installables) { + if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) { + auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + + ProfileElement element; + if (!drv.outPath) + throw UnimplementedError("CA derivations are not yet supported by 'nix profile'"); + element.storePaths = {*drv.outPath}; // FIXME + element.source = ProfileElementSource{ + installable2->flakeRef, + resolvedRef, + attrPath, + }; + + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, StringSet{drv.outputName}}); + + manifest.elements.emplace_back(std::move(element)); + } else { + auto buildables = build(getEvalStore(), store, Realise::Outputs, {installable}, bmNormal); + + for (auto & buildable : buildables) { + ProfileElement element; + + std::visit(overloaded { + [&](BuiltPath::Opaque bo) { + pathsToBuild.push_back(bo); + element.storePaths.insert(bo.path); + }, + [&](BuiltPath::Built bfd) { + // TODO: Why are we querying if we know the output + // names already? Is it just to figure out what the + // default one is? + for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { + pathsToBuild.push_back(DerivedPath::Built{bfd.drvPath, {output.first}}); + element.storePaths.insert(output.second); + } + }, + }, buildable.raw()); + + manifest.elements.emplace_back(std::move(element)); + } + } + } + + store->buildPaths(pathsToBuild); + + updateProfile(manifest.build(store)); + } +}; + +class MixProfileElementMatchers : virtual Args +{ + std::vector<std::string> _matchers; + +public: + + MixProfileElementMatchers() + { + expectArgs("elements", &_matchers); + } + + typedef std::variant<size_t, Path, std::regex> Matcher; + + std::vector<Matcher> getMatchers(ref<Store> store) + { + std::vector<Matcher> res; + + for (auto & s : _matchers) { + if (auto n = string2Int<size_t>(s)) + res.push_back(*n); + else if (store->isStorePath(s)) + res.push_back(s); + else + res.push_back(std::regex(s, std::regex::extended | std::regex::icase)); + } + + return res; + } + + bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector<Matcher> & matchers) + { + for (auto & matcher : matchers) { + if (auto n = std::get_if<size_t>(&matcher)) { + if (*n == pos) return true; + } else if (auto path = std::get_if<Path>(&matcher)) { + if (element.storePaths.count(store.parseStorePath(*path))) return true; + } else if (auto regex = std::get_if<std::regex>(&matcher)) { + if (element.source + && std::regex_match(element.source->attrPath, *regex)) + return true; + } + } + + return false; + } +}; + +struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElementMatchers +{ + std::string description() override + { + return "remove packages from a profile"; + } + + std::string doc() override + { + return + #include "profile-remove.md" + ; + } + + void run(ref<Store> store) override + { + ProfileManifest oldManifest(*getEvalState(), *profile); + + auto matchers = getMatchers(store); + + ProfileManifest newManifest; + + for (size_t i = 0; i < oldManifest.elements.size(); ++i) { + auto & element(oldManifest.elements[i]); + if (!matches(*store, element, i, matchers)) + newManifest.elements.push_back(std::move(element)); + } + + // FIXME: warn about unused matchers? + + printInfo("removed %d packages, kept %d packages", + oldManifest.elements.size() - newManifest.elements.size(), + newManifest.elements.size()); + + updateProfile(newManifest.build(store)); + } +}; + +struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProfileElementMatchers +{ + std::string description() override + { + return "upgrade packages using their most recent flake"; + } + + std::string doc() override + { + return + #include "profile-upgrade.md" + ; + } + + void run(ref<Store> store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + auto matchers = getMatchers(store); + + // FIXME: code duplication + std::vector<DerivedPath> pathsToBuild; + + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + if (element.source + && !element.source->originalRef.input.isImmutable() + && matches(*store, element, i, matchers)) + { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking '%s' for updates", element.source->attrPath)); + + InstallableFlake installable( + this, + getEvalState(), + FlakeRef(element.source->originalRef), + {element.source->attrPath}, + {}, + lockFlags); + + auto [attrPath, resolvedRef, drv] = installable.toDerivation(); + + if (element.source->resolvedRef == resolvedRef) continue; + + printInfo("upgrading '%s' from flake '%s' to '%s'", + element.source->attrPath, element.source->resolvedRef, resolvedRef); + + if (!drv.outPath) + throw UnimplementedError("CA derivations are not yet supported by 'nix profile'"); + element.storePaths = {*drv.outPath}; // FIXME + element.source = ProfileElementSource{ + installable.flakeRef, + resolvedRef, + attrPath, + }; + + pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {drv.outputName}}); + } + } + + store->buildPaths(pathsToBuild); + + updateProfile(manifest.build(store)); + } +}; + +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +{ + std::string description() override + { + return "list installed packages"; + } + + std::string doc() override + { + return + #include "profile-list.md" + ; + } + + void run(ref<Store> store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + logger->cout("%d %s %s %s", i, + element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", + element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", + concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + } + } +}; + +struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile +{ + std::string description() override + { + return "show the closure difference between each version of a profile"; + } + + std::string doc() override + { + return + #include "profile-diff-closures.md" + ; + } + + void run(ref<Store> store) override + { + auto [gens, curGen] = findGenerations(*profile); + + std::optional<Generation> prevGen; + bool first = true; + + for (auto & gen : gens) { + if (prevGen) { + if (!first) std::cout << "\n"; + first = false; + std::cout << fmt("Version %d -> %d:\n", prevGen->number, gen.number); + printClosureDiff(store, + store->followLinksToStorePath(prevGen->path), + store->followLinksToStorePath(gen.path), + " "); + } + + prevGen = gen; + } + } +}; + +struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile +{ + std::string description() override + { + return "show all versions of a profile"; + } + + std::string doc() override + { + return + #include "profile-history.md" + ; + } + + void run(ref<Store> store) override + { + auto [gens, curGen] = findGenerations(*profile); + + std::optional<std::pair<Generation, ProfileManifest>> prevGen; + bool first = true; + + for (auto & gen : gens) { + ProfileManifest manifest(*getEvalState(), gen.path); + + if (!first) std::cout << "\n"; + first = false; + + std::cout << fmt("Version %s%d" ANSI_NORMAL " (%s)%s:\n", + gen.number == curGen ? ANSI_GREEN : ANSI_BOLD, + gen.number, + std::put_time(std::gmtime(&gen.creationTime), "%Y-%m-%d"), + prevGen ? fmt(" <- %d", prevGen->first.number) : ""); + + ProfileManifest::printDiff( + prevGen ? prevGen->second : ProfileManifest(), + manifest, + " "); + + prevGen = {gen, std::move(manifest)}; + } + } +}; + +struct CmdProfileRollback : virtual StoreCommand, MixDefaultProfile, MixDryRun +{ + std::optional<GenerationNumber> version; + + CmdProfileRollback() + { + addFlag({ + .longName = "to", + .description = "The profile version to roll back to.", + .labels = {"version"}, + .handler = {&version}, + }); + } + + std::string description() override + { + return "roll back to the previous version or a specified version of a profile"; + } + + std::string doc() override + { + return + #include "profile-rollback.md" + ; + } + + void run(ref<Store> store) override + { + switchGeneration(*profile, version, dryRun); + } +}; + +struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRun +{ + std::optional<std::string> minAge; + + CmdProfileWipeHistory() + { + addFlag({ + .longName = "older-than", + .description = + "Delete versions older than the specified age. *age* " + "must be in the format *N*`d`, where *N* denotes a number " + "of days.", + .labels = {"age"}, + .handler = {&minAge}, + }); + } + + std::string description() override + { + return "delete non-current versions of a profile"; + } + + std::string doc() override + { + return + #include "profile-wipe-history.md" + ; + } + + void run(ref<Store> store) override + { + if (minAge) + deleteGenerationsOlderThan(*profile, *minAge, dryRun); + else + deleteOldGenerations(*profile, dryRun); + } +}; + +struct CmdProfile : NixMultiCommand +{ + CmdProfile() + : MultiCommand({ + {"install", []() { return make_ref<CmdProfileInstall>(); }}, + {"remove", []() { return make_ref<CmdProfileRemove>(); }}, + {"upgrade", []() { return make_ref<CmdProfileUpgrade>(); }}, + {"list", []() { return make_ref<CmdProfileList>(); }}, + {"diff-closures", []() { return make_ref<CmdProfileDiffClosures>(); }}, + {"history", []() { return make_ref<CmdProfileHistory>(); }}, + {"rollback", []() { return make_ref<CmdProfileRollback>(); }}, + {"wipe-history", []() { return make_ref<CmdProfileWipeHistory>(); }}, + }) + { } + + std::string description() override + { + return "manage Nix profiles"; + } + + std::string doc() override + { + return + #include "profile.md" + ; + } + + void run() override + { + if (!command) + throw UsageError("'nix profile' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdProfile = registerCommand<CmdProfile>("profile"); diff --git a/src/nix/profile.md b/src/nix/profile.md new file mode 100644 index 000000000..d3ddcd3d1 --- /dev/null +++ b/src/nix/profile.md @@ -0,0 +1,107 @@ +R""( + +# Description + +`nix profile` allows you to create and manage *Nix profiles*. A Nix +profile is a set of packages that can be installed and upgraded +independently from each other. Nix profiles are versioned, allowing +them to be rolled back easily. + +# Default profile + +The default profile used by `nix profile` is `$HOME/.nix-profile`, +which, if it does not exist, is created as a symlink to +`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the +`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. + +You can specify another profile location using `--profile` *path*. + +# Filesystem layout + +Profiles are versioned as follows. When using profile *path*, *path* +is a symlink to *path*`-`*N*, where *N* is the current *version* of +the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix +store. For example: + +```console +$ ls -l /nix/var/nix/profiles/per-user/alice/profile* +lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile -> profile-7-link +lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /nix/var/nix/profiles/per-user/alice/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile +lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /nix/var/nix/profiles/per-user/alice/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile +lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile +``` + +Each of these symlinks is a root for the Nix garbage collector. + +The contents of the store path corresponding to each version of the +profile is a tree of symlinks to the files of the installed packages, +e.g. + +```console +$ ll -R /nix/var/nix/profiles/per-user/eelco/profile-7-link/ +/nix/var/nix/profiles/per-user/eelco/profile-7-link/: +total 20 +dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin +-r--r--r-- 2 root root 1402 Jan 1 1970 manifest.json +dr-xr-xr-x 4 root root 4096 Jan 1 1970 share + +/nix/var/nix/profiles/per-user/eelco/profile-7-link/bin: +total 20 +lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium +lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify +lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us + +/nix/var/nix/profiles/per-user/eelco/profile-7-link/share/applications: +total 12 +lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop +lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop +lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop + +… +``` + +The file `manifest.json` records the provenance of the packages that +are installed in this version of the profile. It looks like this: + +```json +{ + "version": 1, + "elements": [ + { + "active": true, + "attrPath": "legacyPackages.x86_64-linux.zoom-us", + "originalUri": "flake:nixpkgs", + "storePaths": [ + "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927" + ], + "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a" + }, + … + ] +} +``` + +Each object in the array `elements` denotes an installed package and +has the following fields: + +* `originalUri`: The [flake reference](./nix3-flake.md) specified by + the user at the time of installation (e.g. `nixpkgs`). This is also + the flake reference that will be used by `nix profile upgrade`. + +* `uri`: The immutable flake reference to which `originalUri` + resolved. + +* `attrPath`: The flake output attribute that provided this + package. Note that this is not necessarily the attribute that the + user specified, but the one resulting from applying the default + attribute paths and prefixes; for instance, `hello` might resolve to + `packages.x86_64-linux.hello` and the empty string to + `defaultPackage.x86_64-linux`. + +* `storePath`: The paths in the Nix store containing the package. + +* `active`: Whether the profile contains symlinks to the files of this + package. If set to false, the package is kept in the Nix store, but + is not "visible" in the profile's symlink tree. + +)"" diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc new file mode 100644 index 000000000..d59e594df --- /dev/null +++ b/src/nix/realisation.cc @@ -0,0 +1,85 @@ +#include "command.hh" +#include "common-args.hh" + +#include <nlohmann/json.hpp> + +using namespace nix; + +struct CmdRealisation : virtual NixMultiCommand +{ + CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"})) + { } + + std::string description() override + { + return "manipulate a Nix realisation"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix realisation' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdRealisation = registerCommand<CmdRealisation>("realisation"); + +struct CmdRealisationInfo : BuiltPathsCommand, MixJSON +{ + std::string description() override + { + return "query information about one or several realisations"; + } + + std::string doc() override + { + return + #include "realisation/info.md" + ; + } + + Category category() override { return catSecondary; } + + void run(ref<Store> store, BuiltPaths paths) override + { + settings.requireExperimentalFeature("ca-derivations"); + RealisedPath::Set realisations; + + for (auto & builtPath : paths) { + auto theseRealisations = builtPath.toRealisedPaths(*store); + realisations.insert(theseRealisations.begin(), theseRealisations.end()); + } + + if (json) { + nlohmann::json res = nlohmann::json::array(); + for (auto & path : realisations) { + nlohmann::json currentPath; + if (auto realisation = std::get_if<Realisation>(&path.raw)) + currentPath = realisation->toJSON(); + else + currentPath["opaquePath"] = store->printStorePath(path.path()); + + res.push_back(currentPath); + } + std::cout << res.dump(); + } + else { + for (auto & path : realisations) { + if (auto realisation = std::get_if<Realisation>(&path.raw)) { + std::cout << + realisation->id.to_string() << " " << + store->printStorePath(realisation->outPath); + } else + std::cout << store->printStorePath(path.path()); + + std::cout << std::endl; + } + } + } +}; + +static auto rCmdRealisationInfo = registerCommand2<CmdRealisationInfo>({"realisation", "info"}); diff --git a/src/nix/realisation/info.md b/src/nix/realisation/info.md new file mode 100644 index 000000000..852240f44 --- /dev/null +++ b/src/nix/realisation/info.md @@ -0,0 +1,15 @@ +R"MdBoundary( +# Description + +Display some informations about the given realisation + +# Examples + +Show some information about the realisation of the `hello` package: + +```console +$ nix realisation info nixpkgs#hello --json +[{"id":"sha256:3d382378a00588e064ee30be96dd0fa7e7df7cf3fbcace85a0e7b7dada1eef25!out","outPath":"fd3m7xawvrqcg98kgz5hc2vk3x9q0lh7-hello"}] +``` + +)MdBoundary" diff --git a/src/nix/registry-add.md b/src/nix/registry-add.md new file mode 100644 index 000000000..a947fa0b3 --- /dev/null +++ b/src/nix/registry-add.md @@ -0,0 +1,40 @@ +R""( + +# Examples + +* Set the `nixpkgs` flake identifier to a specific branch of Nixpkgs: + + ```console + # nix registry add nixpkgs github:NixOS/nixpkgs/nixos-20.03 + ``` + +* Pin `nixpkgs` to a specific revision: + + ```console + # nix registry add nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + ``` + +* Add an entry that redirects a specific branch of `nixpkgs` to + another fork: + + ```console + # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs + ``` + +* Add `nixpkgs` pointing to `github:nixos/nixpkgs` to your custom flake + registry: + + ```console + nix registry add --registry ./custom-flake-registry.json nixpkgs github:nixos/nixpkgs + ``` + +# Description + +This command adds an entry to the user registry that maps flake +reference *from-url* to flake reference *to-url*. If an entry for +*from-url* already exists, it is overwritten. + +Entries can be removed using [`nix registry +remove`](./nix3-registry-remove.md). + +)"" diff --git a/src/nix/registry-list.md b/src/nix/registry-list.md new file mode 100644 index 000000000..30b6e29d8 --- /dev/null +++ b/src/nix/registry-list.md @@ -0,0 +1,29 @@ +R""( + +# Examples + +* Show the contents of all registries: + + ```console + # nix registry list + user flake:dwarffs github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5 + system flake:nixpkgs path:/nix/store/fxl9mrm5xvzam0lxi9ygdmksskx4qq8s-source?lastModified=1605220118&narHash=sha256-Und10ixH1WuW0XHYMxxuHRohKYb45R%2fT8CwZuLd2D2Q=&rev=3090c65041104931adda7625d37fa874b2b5c124 + global flake:blender-bin github:edolstra/nix-warez?dir=blender + global flake:dwarffs github:edolstra/dwarffs + … + ``` + +# Description + +This command displays the contents of all registries on standard +output. Each line represents one registry entry in the format *type* +*from* *to*, where *type* denotes the registry containing the entry: + +* `flags`: entries specified on the command line using `--override-flake`. +* `user`: the user registry. +* `system`: the system registry. +* `global`: the global registry. + +See the [`nix registry` manual page](./nix3-registry.md) for more details. + +)"" diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md new file mode 100644 index 000000000..ebc0e3eff --- /dev/null +++ b/src/nix/registry-pin.md @@ -0,0 +1,45 @@ +R""( + +# Examples + +* Pin `nixpkgs` to its most recent Git revision: + + ```console + # nix registry pin nixpkgs + ``` + + Afterwards the user registry will have an entry like this: + + ```console + nix registry list | grep '^user ' + user flake:nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + ``` + + and `nix flake info` will say: + + ```console + # nix flake info nixpkgs + Resolved URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + Locked URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + … + ``` + +* Pin `nixpkgs` in a custom registry to its most recent Git revision: + + ```console + # nix registry pin --registry ./custom-flake-registry.json nixpkgs + ``` + + +# Description + +This command adds an entry to the user registry that maps flake +reference *url* to the corresponding *locked* flake reference, that +is, a flake reference that specifies an exact revision or content +hash. This ensures that until this registry entry is removed, all uses +of *url* will resolve to exactly the same flake. + +Entries can be removed using [`nix registry +remove`](./nix3-registry-remove.md). + +)"" diff --git a/src/nix/registry-remove.md b/src/nix/registry-remove.md new file mode 100644 index 000000000..eecd4c6e7 --- /dev/null +++ b/src/nix/registry-remove.md @@ -0,0 +1,22 @@ +R""( + +# Examples + +* Remove the entry `nixpkgs` from the user registry: + + ```console + # nix registry remove nixpkgs + ``` + +* Remove the entry `nixpkgs` from a custom registry: + + ```console + # nix registry remove --registry ./custom-flake-registry.json nixpkgs + ``` + +# Description + +This command removes from the user registry any entry for flake +reference *url*. + +)"" diff --git a/src/nix/registry.cc b/src/nix/registry.cc new file mode 100644 index 000000000..6a92576c7 --- /dev/null +++ b/src/nix/registry.cc @@ -0,0 +1,236 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "eval.hh" +#include "flake/flake.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "registry.hh" + +using namespace nix; +using namespace nix::flake; + + +class RegistryCommand : virtual Args +{ + std::string registry_path; + + std::shared_ptr<fetchers::Registry> registry; + +public: + + RegistryCommand() + { + addFlag({ + .longName = "registry", + .description = "The registry to operate on.", + .labels = {"registry"}, + .handler = {®istry_path}, + }); + } + + std::shared_ptr<fetchers::Registry> getRegistry() + { + if (registry) return registry; + if (registry_path.empty()) { + registry = fetchers::getUserRegistry(); + } else { + registry = fetchers::getCustomRegistry(registry_path); + } + return registry; + } + + Path getRegistryPath() + { + if (registry_path.empty()) { + return fetchers::getUserRegistryPath(); + } else { + return registry_path; + } + } +}; + +struct CmdRegistryList : StoreCommand +{ + std::string description() override + { + return "list available Nix flakes"; + } + + std::string doc() override + { + return + #include "registry-list.md" + ; + } + + void run(nix::ref<nix::Store> store) override + { + using namespace fetchers; + + auto registries = getRegistries(store); + + for (auto & registry : registries) { + for (auto & entry : registry->entries) { + // FIXME: format nicely + logger->cout("%s %s %s", + registry->type == Registry::Flag ? "flags " : + registry->type == Registry::User ? "user " : + registry->type == Registry::System ? "system" : + "global", + entry.from.toURLString(), + entry.to.toURLString(attrsToQuery(entry.extraAttrs))); + } + } + } +}; + +struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand +{ + std::string fromUrl, toUrl; + + std::string description() override + { + return "add/replace flake in user flake registry"; + } + + std::string doc() override + { + return + #include "registry-add.md" + ; + } + + CmdRegistryAdd() + { + expectArg("from-url", &fromUrl); + expectArg("to-url", &toUrl); + } + + void run() override + { + auto fromRef = parseFlakeRef(fromUrl); + auto toRef = parseFlakeRef(toUrl); + auto registry = getRegistry(); + fetchers::Attrs extraAttrs; + if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; + registry->remove(fromRef.input); + registry->add(fromRef.input, toRef.input, extraAttrs); + registry->write(getRegistryPath()); + } +}; + +struct CmdRegistryRemove : RegistryCommand, Command +{ + std::string url; + + std::string description() override + { + return "remove flake from user flake registry"; + } + + std::string doc() override + { + return + #include "registry-remove.md" + ; + } + + CmdRegistryRemove() + { + expectArg("url", &url); + } + + void run() override + { + auto registry = getRegistry(); + registry->remove(parseFlakeRef(url).input); + registry->write(getRegistryPath()); + } +}; + +struct CmdRegistryPin : RegistryCommand, EvalCommand +{ + std::string url; + + std::string locked; + + std::string description() override + { + return "pin a flake to its current version or to the current version of a flake URL"; + } + + std::string doc() override + { + return + #include "registry-pin.md" + ; + } + + CmdRegistryPin() + { + expectArg("url", &url); + + expectArgs({ + .label = "locked", + .optional = true, + .handler = {&locked}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); + } + + void run(nix::ref<nix::Store> store) override + { + if (locked.empty()) { + locked = url; + } + auto registry = getRegistry(); + auto ref = parseFlakeRef(url); + auto locked_ref = parseFlakeRef(locked); + registry->remove(ref.input); + auto [tree, resolved] = locked_ref.resolve(store).input.fetch(store); + fetchers::Attrs extraAttrs; + if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; + registry->add(ref.input, resolved, extraAttrs); + registry->write(getRegistryPath()); + } +}; + +struct CmdRegistry : virtual NixMultiCommand +{ + CmdRegistry() + : MultiCommand({ + {"list", []() { return make_ref<CmdRegistryList>(); }}, + {"add", []() { return make_ref<CmdRegistryAdd>(); }}, + {"remove", []() { return make_ref<CmdRegistryRemove>(); }}, + {"pin", []() { return make_ref<CmdRegistryPin>(); }}, + }) + { + } + + std::string description() override + { + return "manage the flake registry"; + } + + std::string doc() override + { + return + #include "registry.md" + ; + } + + Category category() override { return catSecondary; } + + void run() override + { + if (!command) + throw UsageError("'nix registry' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdRegistry = registerCommand<CmdRegistry>("registry"); diff --git a/src/nix/registry.md b/src/nix/registry.md new file mode 100644 index 000000000..a1674bd2e --- /dev/null +++ b/src/nix/registry.md @@ -0,0 +1,98 @@ +R""( + +# Description + +`nix flake` provides subcommands for managing *flake +registries*. Flake registries are a convenience feature that allows +you to refer to flakes using symbolic identifiers such as `nixpkgs`, +rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You +can use these identifiers on the command line (e.g. when you do `nix +run nixpkgs#hello`) or in flake input specifications in `flake.nix` +files. The latter are automatically resolved to full URLs and recorded +in the flake's `flake.lock` file. + +In addition, the flake registry allows you to redirect arbitrary flake +references (e.g. `github:NixOS/patchelf`) to another location, such as +a local fork. + +There are multiple registries. These are, in order from lowest to +highest precedence: + +* The global registry, which is a file downloaded from the URL + specified by the setting `flake-registry`. It is cached locally and + updated automatically when it's older than `tarball-ttl` + seconds. The default global registry is kept in [a GitHub + repository](https://github.com/NixOS/flake-registry). + +* The system registry, which is shared by all users. The default + location is `/etc/nix/registry.json`. On NixOS, the system registry + can be specified using the NixOS option `nix.registry`. + +* The user registry `~/.config/nix/registry.json`. This registry can + be modified by commands such as `nix flake pin`. + +* Overrides specified on the command line using the option + `--override-flake`. + +# Registry format + +A registry is a JSON file with the following format: + +```json +{ + "version": 2, + "flakes": [ + { + "from": { + "type": "indirect", + "id": "nixpkgs" + }, + "to": { + "type": "github", + "owner": "NixOS", + "repo": "nixpkgs" + } + }, + ... + ] +} +``` + +That is, it contains a list of objects with attributes `from` and +`to`, both of which contain a flake reference in attribute +representation. (For example, `{"type": "indirect", "id": "nixpkgs"}` +is the attribute representation of `nixpkgs`, while `{"type": +"github", "owner": "NixOS", "repo": "nixpkgs"}` is the attribute +representation of `github:NixOS/nixpkgs`.) + +Given some flake reference *R*, a registry entry is used if its +`from` flake reference *matches* *R*. *R* is then replaced by the +*unification* of the `to` flake reference with *R*. + +# Matching + +The `from` flake reference in a registry entry *matches* some flake +reference *R* if the attributes in `from` are the same as the +attributes in `R`. For example: + +* `nixpkgs` matches with `nixpkgs`. + +* `nixpkgs` matches with `nixpkgs/nixos-20.09`. + +* `nixpkgs/nixos-20.09` does not match with `nixpkgs`. + +* `nixpkgs` does not match with `git://github.com/NixOS/patchelf`. + +# Unification + +The `to` flake reference in a registry entry is *unified* with some flake +reference *R* by taking `to` and applying the `rev` and `ref` +attributes from *R*, if specified. For example: + +* `github:NixOS/nixpkgs` unified with `nixpkgs` produces `github:NixOS/nixpkgs`. + +* `github:NixOS/nixpkgs` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`. + +* `github:NixOS/nixpkgs/master` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`. + +)"" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fdacf604b..c1233ab46 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -32,13 +32,19 @@ extern "C" { #include "globals.hh" #include "command.hh" #include "finally.hh" +#include "markdown.hh" +#if HAVE_BOEHMGC #define GC_INCLUDE_NEW #include <gc/gc_cpp.h> +#endif namespace nix { -struct NixRepl : gc +struct NixRepl + #if HAVE_BOEHMGC + : gc + #endif { string curDir; std::unique_ptr<EvalState> state; @@ -59,9 +65,10 @@ struct NixRepl : gc void mainLoop(const std::vector<std::string> & files); StringSet completePrefix(string prefix); bool getLine(string & input, const std::string &prompt); - Path getDerivationPath(Value & v); + StorePath getDerivationPath(Value & v); bool processLine(string line); void loadFile(const Path & path); + void loadFlake(const std::string & flakeRef); void initEnv(); void reloadFiles(); void addAttrsToScope(Value & attrs); @@ -98,6 +105,25 @@ NixRepl::~NixRepl() write_history(historyFile.c_str()); } +string runNix(Path program, const Strings & args, + const std::optional<std::string> & input = {}) +{ + auto subprocessEnv = getEnv(); + subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); + + auto res = runProgram(RunOptions { + .program = settings.nixBinDir+ "/" + program, + .args = args, + .environment = subprocessEnv, + .input = input, + }); + + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + + return res.second; +} + static NixRepl * curRepl; // ugly static char * completionCallback(char * s, int *match) { @@ -206,7 +232,7 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) try { if (!removeWhitespace(input).empty() && !processLine(input)) return; } catch (ParseError & e) { - if (e.msg().find("unexpected $end") != std::string::npos) { + if (e.msg().find("unexpected end of file") != std::string::npos) { // For parse errors on incomplete input, we continue waiting for the next line of // input without clearing the input so far. continue; @@ -214,9 +240,9 @@ void NixRepl::mainLoop(const std::vector<std::string> & files) printMsg(lvlError, e.msg()); } } catch (Error & e) { - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Interrupted & e) { - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } // We handled the current input fully, so we should clear it @@ -337,24 +363,6 @@ StringSet NixRepl::completePrefix(string prefix) } -static int runProgram(const string & program, const Strings & args) -{ - Strings args2(args); - args2.push_front(program); - - Pid pid; - pid = fork(); - if (pid == -1) throw SysError("forking"); - if (pid == 0) { - restoreAffinity(); - execvp(program.c_str(), stringsToCharPtrs(args2).data()); - _exit(1); - } - - return pid.wait(); -} - - bool isVarName(const string & s) { if (s.size() == 0) return false; @@ -370,13 +378,16 @@ bool isVarName(const string & s) } -Path NixRepl::getDerivationPath(Value & v) { +StorePath NixRepl::getDerivationPath(Value & v) { auto drvInfo = getDerivation(*state, v, false); if (!drvInfo) throw Error("expression does not evaluate to a derivation, so I can't build it"); - Path drvPath = drvInfo->queryDrvPath(); - if (drvPath == "" || !state->store->isValidPath(state->store->parseStorePath(drvPath))) - throw Error("expression did not evaluate to a valid derivation"); + Path drvPathRaw = drvInfo->queryDrvPath(); + if (drvPathRaw == "") + throw Error("expression did not evaluate to a valid derivation (no drv path)"); + StorePath drvPath = state->store->parseStorePath(drvPathRaw); + if (!state->store->isValidPath(drvPath)) + throw Error("expression did not evaluate to a valid derivation (invalid drv path)"); return drvPath; } @@ -396,6 +407,7 @@ bool NixRepl::processLine(string line) } if (command == ":?" || command == ":help") { + // FIXME: convert to Markdown, include in the 'nix repl' manpage. std::cout << "The following commands are available:\n" << "\n" @@ -403,15 +415,17 @@ bool NixRepl::processLine(string line) << " <x> = <expr> Bind expression to variable\n" << " :a <expr> Add attributes from resulting set to scope\n" << " :b <expr> Build derivation\n" - << " :e <expr> Open the derivation in $EDITOR\n" + << " :e <expr> Open package or function in $EDITOR\n" << " :i <expr> Build derivation, then install result into current profile\n" << " :l <path> Load Nix expression and add it to scope\n" + << " :lf <ref> Load Nix flake and add it to scope\n" << " :p <expr> Evaluate and print expression recursively\n" << " :q Exit nix-repl\n" << " :r Reload all files\n" << " :s <expr> Build dependencies of derivation, then start nix-shell\n" << " :t <expr> Describe result of evaluation\n" - << " :u <expr> Build derivation, then start nix-shell\n"; + << " :u <expr> Build derivation, then start nix-shell\n" + << " :doc <expr> Show documentation of a builtin function\n"; } else if (command == ":a" || command == ":add") { @@ -425,6 +439,10 @@ bool NixRepl::processLine(string line) loadFile(arg); } + else if (command == ":lf" || command == ":load-flake") { + loadFlake(arg); + } + else if (command == ":r" || command == ":reload") { state->resetFileCache(); reloadFiles(); @@ -436,22 +454,22 @@ bool NixRepl::processLine(string line) Pos pos; - if (v.type == tPath || v.type == tString) { + if (v.type() == nPath || v.type() == nString) { PathSet context; auto filename = state->coerceToString(noPos, v, context); pos.file = state->symbols.create(filename); - } else if (v.type == tLambda) { + } else if (v.isLambda()) { pos = v.lambda.fun->pos; } else { // assume it's a derivation - pos = findDerivationFilename(*state, v, arg); + pos = findPackageFilename(*state, v, arg); } // Open in EDITOR auto args = editorFor(pos); auto editor = args.front(); args.pop_front(); - runProgram(editor, args); + runProgram(editor, true, args); // Reload right after exiting the editor state->resetFileCache(); @@ -469,29 +487,32 @@ bool NixRepl::processLine(string line) evalString("drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f); state->callFunction(f, v, result, Pos()); - Path drvPath = getDerivationPath(result); - runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + StorePath drvPath = getDerivationPath(result); + runNix("nix-shell", {state->store->printStorePath(drvPath)}); } else if (command == ":b" || command == ":i" || command == ":s") { Value v; evalString(arg, v); - Path drvPath = getDerivationPath(v); + StorePath drvPath = getDerivationPath(v); + Path drvPathRaw = state->store->printStorePath(drvPath); if (command == ":b") { /* We could do the build in this process using buildPaths(), but doing it in a child makes it easier to recover from problems / SIGINT. */ - if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { - auto drv = readDerivation(*state->store, drvPath); + try { + runNix("nix", {"build", "--no-link", drvPathRaw}); + auto drv = state->store->readDerivation(drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; - for (auto & i : drv.outputs) - std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(i.second.path)); + for (auto & i : drv.outputsAndOptPaths(*state->store)) + std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(*i.second.second)); + } catch (ExecError &) { } } else if (command == ":i") { - runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); + runNix("nix-env", {"-i", drvPathRaw}); } else { - runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + runNix("nix-shell", {drvPathRaw}); } } @@ -504,6 +525,29 @@ bool NixRepl::processLine(string line) else if (command == ":q" || command == ":quit") return false; + else if (command == ":doc") { + Value v; + evalString(arg, v); + if (auto doc = state->getDoc(v)) { + std::string markdown; + + if (!doc->args.empty() && doc->name) { + auto args = doc->args; + for (auto & arg : args) + arg = "*" + arg + "*"; + + markdown += + "**Synopsis:** `builtins." + (std::string) (*doc->name) + "` " + + concatStringsSep(" ", args) + "\n\n"; + } + + markdown += trim(stripIndentation(doc->doc)); + + std::cout << renderMarkdownToTerminal(markdown); + } else + throw Error("value does not have documentation"); + } + else if (command != "") throw Error("unknown command '%1%'", command); @@ -517,9 +561,7 @@ bool NixRepl::processLine(string line) { Expr * e = parseString(string(line, p + 1)); Value & v(*state->allocValue()); - v.type = tThunk; - v.thunk.env = env; - v.thunk.expr = e; + v.mkThunk(env, e); addVarToScope(state->symbols.create(name), v); } else { Value v; @@ -542,6 +584,25 @@ void NixRepl::loadFile(const Path & path) addAttrsToScope(v2); } +void NixRepl::loadFlake(const std::string & flakeRefS) +{ + auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + if (evalSettings.pureEval && !flakeRef.input.isImmutable()) + throw Error("cannot use ':load-flake' on mutable flake reference '%s' (use --impure to override)", flakeRefS); + + Value v; + + flake::callFlake(*state, + flake::lockFlake(*state, flakeRef, + flake::LockFlags { + .updateLockFile = false, + .useRegistries = !evalSettings.pureEval, + .allowMutable = !evalSettings.pureEval, + }), + v); + addAttrsToScope(v); +} + void NixRepl::initEnv() { @@ -635,31 +696,31 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m state->forceValue(v); - switch (v.type) { + switch (v.type()) { - case tInt: + case nInt: str << ANSI_CYAN << v.integer << ANSI_NORMAL; break; - case tBool: + case nBool: str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; break; - case tString: - str << ANSI_YELLOW; + case nString: + str << ANSI_WARNING; printStringValue(str, v.string.s); str << ANSI_NORMAL; break; - case tPath: + case nPath: str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping? break; - case tNull: + case nNull: str << ANSI_CYAN "null" ANSI_NORMAL; break; - case tAttrs: { + case nAttrs: { seen.insert(&v); bool isDrv = state->isDerivation(v); @@ -704,9 +765,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; } - case tList1: - case tList2: - case tListN: + case nList: seen.insert(&v); str << "[ "; @@ -727,22 +786,21 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << "]"; break; - case tLambda: { - std::ostringstream s; - s << v.lambda.fun->pos; - str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; - break; - } - - case tPrimOp: - str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; - break; - - case tPrimOpApp: - str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; + case nFunction: + if (v.isLambda()) { + std::ostringstream s; + s << v.lambda.fun->pos; + str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; + } else if (v.isPrimOp()) { + str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; + } else if (v.isPrimOpApp()) { + str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; + } else { + abort(); + } break; - case tFloat: + case nFloat: str << v.fpoint; break; @@ -760,7 +818,11 @@ struct CmdRepl : StoreCommand, MixEvalArgs CmdRepl() { - expectArgs("files", &files); + expectArgs({ + .label = "files", + .handler = {&files}, + .completer = completePath + }); } std::string description() override @@ -768,24 +830,22 @@ struct CmdRepl : StoreCommand, MixEvalArgs return "start an interactive environment for evaluating Nix expressions"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "Display all special commands within the REPL:", - "nix repl\n nix-repl> :?" - } - }; + return + #include "repl.md" + ; } void run(ref<Store> store) override { + evalSettings.pureEval = false; auto repl = std::make_unique<NixRepl>(searchPath, openStore()); repl->autoArgs = getAutoArgs(*repl->state); repl->mainLoop(files); } }; -static auto r1 = registerCommand<CmdRepl>("repl"); +static auto rCmdRepl = registerCommand<CmdRepl>("repl"); } diff --git a/src/nix/repl.md b/src/nix/repl.md new file mode 100644 index 000000000..bba60f871 --- /dev/null +++ b/src/nix/repl.md @@ -0,0 +1,57 @@ +R""( + +# Examples + +* Display all special commands within the REPL: + + ```console + # nix repl + nix-repl> :? + ``` + +* Evaluate some simple Nix expressions: + + ```console + # nix repl + + nix-repl> 1 + 2 + 3 + + nix-repl> map (x: x * 2) [1 2 3] + [ 2 4 6 ] + ``` + +* Interact with Nixpkgs in the REPL: + + ```console + # nix repl '<nixpkgs>' + + Loading '<nixpkgs>'... + Added 12428 variables. + + nix-repl> emacs.name + "emacs-27.1" + + nix-repl> emacs.drvPath + "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv" + + nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out" + + nix-repl> :b x + this derivation produced the following outputs: + out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello + + nix-repl> builtins.readFile drv + "Hello, world!\n" + ``` + +# Description + +This command provides an interactive environment for evaluating Nix +expressions. (REPL stands for 'read–eval–print loop'.) + +On startup, it loads the Nix expressions named *files* and adds them +into the lexical scope. You can load addition files using the `:l +<filename>` command, or reload all files using `:r`. + +)"" diff --git a/src/nix/run.cc b/src/nix/run.cc index 321ee1d11..b01fdebaa 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,3 +1,4 @@ +#include "run.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -20,45 +21,47 @@ using namespace nix; std::string chrootHelperName = "__run_in_chroot"; -struct RunCommon : virtual Command +namespace nix { + +void runProgramInStore(ref<Store> store, + const std::string & program, + const Strings & args) { - void runProgram(ref<Store> store, - const std::string & program, - const Strings & args) - { - stopProgressBar(); + stopProgressBar(); - restoreSignals(); + restoreProcessContext(); - restoreAffinity(); + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded program + (which "nix" is), so we exec() a single-threaded helper program + (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast<LocalStore>(); - /* If this is a diverted store (i.e. its "logical" location - (typically /nix/store) differs from its "physical" location - (e.g. /home/eelco/nix/store), then run the command in a - chroot. For non-root users, this requires running it in new - mount and user namespaces. Unfortunately, - unshare(CLONE_NEWUSER) doesn't work in a multithreaded - program (which "nix" is), so we exec() a single-threaded - helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast<LocalStore>(); + if (store2 && store->storeDir != store2->getRealStoreDir()) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; + for (auto & arg : args) helperArgs.push_back(arg); - if (store2 && store->storeDir != store2->realStoreDir) { - Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, program }; - for (auto & arg : args) helperArgs.push_back(arg); + execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); - execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + throw SysError("could not execute chroot helper"); + } - throw SysError("could not execute chroot helper"); - } + execvp(program.c_str(), stringsToCharPtrs(args).data()); - execvp(program.c_str(), stringsToCharPtrs(args).data()); + throw SysError("unable to execute '%s'", program); +} - throw SysError("unable to execute '%s'", program); - } -}; +} -struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment +struct CmdShell : InstallablesCommand, MixEnvironment { + + using InstallablesCommand::run; + std::vector<std::string> command = { getEnv("SHELL").value_or("bash") }; CmdShell() @@ -66,7 +69,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed; defaults to '$SHELL'", + .description = "Command and arguments to be executed, defaulting to `$SHELL`", .labels = {"command", "args"}, .handler = {[&](std::vector<std::string> ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); @@ -80,35 +83,19 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment return "run a shell in which the specified packages are available"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To start a shell providing GNU Hello from NixOS 17.03:", - "nix shell -f channel:nixos-17.03 hello" - }, - Example{ - "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix shell nixpkgs.youtube-dl" - }, - Example{ - "To run GNU Hello:", - "nix shell nixpkgs.hello -c hello --greeting 'Hi everybody!'" - }, - Example{ - "To run GNU Hello in a chroot store:", - "nix shell --store ~/my-nix nixpkgs.hello -c hello" - }, - }; + return + #include "shell.md" + ; } void run(ref<Store> store) override { - auto outPaths = toStorePaths(store, Build, installables); + auto outPaths = toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); auto accessor = store->getFSAccessor(); - std::unordered_set<StorePath> done; std::queue<StorePath> todo; for (auto & path : outPaths) todo.push(path); @@ -137,11 +124,69 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment Strings args; for (auto & arg : command) args.push_back(arg); - runProgram(store, *command.begin(), args); + runProgramInStore(store, *command.begin(), args); + } +}; + +static auto rCmdShell = registerCommand<CmdShell>("shell"); + +struct CmdRun : InstallableCommand +{ + using InstallableCommand::run; + + std::vector<std::string> args; + + CmdRun() + { + expectArgs({ + .label = "args", + .handler = {&args}, + .completer = completePath + }); + } + + std::string description() override + { + return "run a Nix application"; + } + + std::string doc() override + { + return + #include "run.md" + ; + } + + Strings getDefaultFlakeAttrPaths() override + { + Strings res{"defaultApp." + settings.thisSystem.get()}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPaths()) + res.push_back(s); + return res; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + Strings res{"apps." + settings.thisSystem.get() + "."}; + for (auto & s : SourceExprCommand::getDefaultFlakeAttrPathPrefixes()) + res.push_back(s); + return res; + } + + void run(ref<Store> store) override + { + auto state = getEvalState(); + + auto app = installable->toApp(*state).resolve(getEvalStore(), store); + + Strings allArgs{app.program}; + for (auto & i : args) allArgs.push_back(i); + + runProgramInStore(store, app.program, allArgs); } }; -static auto r1 = registerCommand<CmdShell>("shell"); +static auto rCmdRun = registerCommand<CmdRun>("run"); void chrootHelper(int argc, char * * argv) { @@ -182,14 +227,16 @@ void chrootHelper(int argc, char * * argv) for (auto entry : readDirectory("/")) { auto src = "/" + entry.name; - auto st = lstat(src); - if (!S_ISDIR(st.st_mode)) continue; Path dst = tmpDir + "/" + entry.name; if (pathExists(dst)) continue; - if (mkdir(dst.c_str(), 0700) == -1) - throw SysError("creating directory '%s'", dst); - if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("mounting '%s' on '%s'", src, dst); + auto st = lstat(src); + if (S_ISDIR(st.st_mode)) { + if (mkdir(dst.c_str(), 0700) == -1) + throw SysError("creating directory '%s'", dst); + if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("mounting '%s' on '%s'", src, dst); + } else if (S_ISLNK(st.st_mode)) + createSymlink(readLink(src), dst); } char * cwd = getcwd(0, 0); diff --git a/src/nix/run.hh b/src/nix/run.hh new file mode 100644 index 000000000..6180a87dd --- /dev/null +++ b/src/nix/run.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +void runProgramInStore(ref<Store> store, + const std::string & program, + const Strings & args); + +} diff --git a/src/nix/run.md b/src/nix/run.md new file mode 100644 index 000000000..a76750376 --- /dev/null +++ b/src/nix/run.md @@ -0,0 +1,88 @@ +R""( + +# Examples + +* Run the default app from the `blender-bin` flake: + + ```console + # nix run blender-bin + ``` + +* Run a non-default app from the `blender-bin` flake: + + ```console + # nix run blender-bin#blender_2_83 + ``` + + Tip: you can find apps provided by this flake by running `nix flake + show blender-bin`. + +* Run `vim` from the `nixpkgs` flake: + + ```console + # nix run nixpkgs#vim + ``` + + Note that `vim` (as of the time of writing of this page) is not an + app but a package. Thus, Nix runs the eponymous file from the `vim` + package. + +* Run `vim` with arguments: + + ```console + # nix run nixpkgs#vim -- --help + ``` + +# Description + +`nix run` builds and runs *installable*, which must evaluate to an +*app* or a regular Nix derivation. + +If *installable* evaluates to an *app* (see below), it executes the +program specified by the app definition. + +If *installable* evaluates to a derivation, it will try to execute the +program `<out>/bin/<name>`, where *out* is the primary output store +path of the derivation and *name* is the `meta.mainProgram` attribute +of the derivation if it exists, and otherwise the name part of the +value of the `name` attribute of the derivation (e.g. if `name` is set +to `hello-1.10`, it will run `$out/bin/hello`). + +# Flake output attributes + +If no flake output attribute is given, `nix run` tries the following +flake output attributes: + +* `defaultApp.<system>` + +* `defaultPackage.<system>` + +If an attribute *name* is given, `nix run` tries the following flake +output attributes: + +* `apps.<system>.<name>` + +* `packages.<system>.<name>` + +* `legacyPackages.<system>.<name>` + +# Apps + +An app is specified by a flake output attribute named +`apps.<system>.<name>` or `defaultApp.<system>`. It looks like this: + +```nix +apps.x86_64-linux.blender_2_79 = { + type = "app"; + program = "${self.packages.x86_64-linux.blender_2_79}/bin/blender"; +}; +``` + +The only supported attributes are: + +* `type` (required): Must be set to `app`. + +* `program` (required): The full path of the executable to run. It + must reside in the Nix store. + +)"" diff --git a/src/nix/search.cc b/src/nix/search.cc index 93c3f3f83..c52a48d4e 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -6,8 +6,9 @@ #include "get-drvs.hh" #include "common-args.hh" #include "json.hh" -#include "json-to-value.hh" #include "shared.hh" +#include "eval-cache.hh" +#include "attr-path.hh" #include <regex> #include <fstream> @@ -25,59 +26,36 @@ std::string hilite(const std::string & s, const std::smatch & m, std::string pos m.empty() ? s : std::string(m.prefix()) - + ANSI_RED + std::string(m.str()) + postfix + + ANSI_GREEN + std::string(m.str()) + postfix + std::string(m.suffix()); } -struct CmdSearch : SourceExprCommand, MixJSON +struct CmdSearch : InstallableCommand, MixJSON { std::vector<std::string> res; - bool writeCache = true; - bool useCache = true; - CmdSearch() { expectArgs("regex", &res); - - addFlag({ - .longName = "update-cache", - .shortName = 'u', - .description = "update the package search cache", - .handler = {[&]() { writeCache = true; useCache = false; }} - }); - - addFlag({ - .longName = "no-cache", - .description = "do not use or update the package search cache", - .handler = {[&]() { writeCache = false; useCache = false; }} - }); } std::string description() override { - return "query available packages"; + return "search for packages"; + } + + std::string doc() override + { + return + #include "search.md" + ; } - Examples examples() override + Strings getDefaultFlakeAttrPaths() override { return { - Example{ - "To show all available packages:", - "nix search" - }, - Example{ - "To show any packages containing 'blender' in its name or description:", - "nix search blender" - }, - Example{ - "To search for Firefox or Chromium:", - "nix search 'firefox|chromium'" - }, - Example{ - "To search for git and frontend or gui:", - "nix search git 'frontend|gui'" - } + "packages." + settings.thisSystem.get() + ".", + "legacyPackages." + settings.thisSystem.get() + "." }; } @@ -88,190 +66,111 @@ struct CmdSearch : SourceExprCommand, MixJSON // Empty search string should match all packages // Use "^" here instead of ".*" due to differences in resulting highlighting // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) - if (res.empty()) { + if (res.empty()) res.push_back("^"); - } std::vector<std::regex> regexes; regexes.reserve(res.size()); - for (auto &re : res) { + for (auto & re : res) regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase)); - } auto state = getEvalState(); auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr; - auto sToplevel = state->symbols.create("_toplevel"); - auto sRecurse = state->symbols.create("recurseForDerivations"); - - bool fromCache = false; - - std::map<std::string, std::string> results; + uint64_t results = 0; - std::function<void(Value *, std::string, bool, JSONObject *)> doExpr; - - doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) { - debug("at attribute '%s'", attrPath); + std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit; + visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse) + { + Activity act(*logger, lvlInfo, actUnknown, + fmt("evaluating '%s'", concatStringsSep(".", attrPath))); try { - uint found = 0; + auto recurse = [&]() + { + for (const auto & attr : cursor.getAttrs()) { + auto cursor2 = cursor.getAttr(attr); + auto attrPath2(attrPath); + attrPath2.push_back(attr); + visit(*cursor2, attrPath2, false); + } + }; - state->forceValue(*v); + if (cursor.isDerivation()) { + size_t found = 0; - if (v->type == tLambda && toplevel) { - Value * v2 = state->allocValue(); - state->autoCallFunction(*state->allocBindings(1), *v, *v2); - v = v2; - state->forceValue(*v); - } + DrvName name(cursor.getAttr("name")->getString()); - if (state->isDerivation(*v)) { + auto aMeta = cursor.maybeGetAttr("meta"); + auto aDescription = aMeta ? aMeta->maybeGetAttr("description") : nullptr; + auto description = aDescription ? aDescription->getString() : ""; + std::replace(description.begin(), description.end(), '\n', ' '); + auto attrPath2 = concatStringsSep(".", attrPath); - DrvInfo drv(*state, attrPath, v->attrs); - std::string description; std::smatch attrPathMatch; std::smatch descriptionMatch; std::smatch nameMatch; - std::string name; - DrvName parsed(drv.queryName()); - - for (auto ®ex : regexes) { - std::regex_search(attrPath, attrPathMatch, regex); - - name = parsed.name; - std::regex_search(name, nameMatch, regex); - - description = drv.queryMetaString("description"); - std::replace(description.begin(), description.end(), '\n', ' '); + for (auto & regex : regexes) { + std::regex_search(attrPath2, attrPathMatch, regex); + std::regex_search(name.name, nameMatch, regex); std::regex_search(description, descriptionMatch, regex); - if (!attrPathMatch.empty() || !nameMatch.empty() || !descriptionMatch.empty()) - { found++; - } } if (found == res.size()) { + results++; if (json) { - - auto jsonElem = jsonOut->object(attrPath); - - jsonElem.attr("pkgName", parsed.name); - jsonElem.attr("version", parsed.version); + auto jsonElem = jsonOut->object(attrPath2); + jsonElem.attr("pname", name.name); + jsonElem.attr("version", name.version); jsonElem.attr("description", description); - } else { - auto name = hilite(parsed.name, nameMatch, "\e[0;2m") - + std::string(parsed.fullName, parsed.name.length()); - results[attrPath] = fmt( - "* %s (%s)\n %s\n", - wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), - wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), - hilite(description, descriptionMatch, ANSI_NORMAL)); - } - } - - if (cache) { - cache->attr("type", "derivation"); - cache->attr("name", drv.queryName()); - cache->attr("system", drv.querySystem()); - if (description != "") { - auto meta(cache->object("meta")); - meta.attr("description", description); + auto name2 = hilite(name.name, nameMatch, "\e[0;2m"); + if (results > 1) logger->cout(""); + logger->cout( + "* %s%s", + wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")), + name.version != "" ? " (" + name.version + ")" : ""); + if (description != "") + logger->cout( + " %s", hilite(description, descriptionMatch, ANSI_NORMAL)); } } } - else if (v->type == tAttrs) { + else if ( + attrPath.size() == 0 + || (attrPath[0] == "legacyPackages" && attrPath.size() <= 2) + || (attrPath[0] == "packages" && attrPath.size() <= 2)) + recurse(); - if (!toplevel) { - auto attrs = v->attrs; - Bindings::iterator j = attrs->find(sRecurse); - if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) { - debug("skip attribute '%s'", attrPath); - return; - } - } - - bool toplevel2 = false; - if (!fromCache) { - Bindings::iterator j = v->attrs->find(sToplevel); - toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos); - } + else if (initialRecurse) + recurse(); - for (auto & i : *v->attrs) { - auto cache2 = - cache ? std::make_unique<JSONObject>(cache->object(i.name)) : nullptr; - doExpr(i.value, - attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name, - toplevel2 || fromCache, cache2 ? cache2.get() : nullptr); - } + else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) { + auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations); + if (attr && attr->getBool()) + recurse(); } - } catch (AssertionError & e) { - } catch (Error & e) { - if (!toplevel) { - e.addTrace(std::nullopt, "While evaluating the attribute '%s'", attrPath); + } catch (EvalError & e) { + if (!(attrPath.size() > 0 && attrPath[0] == "legacyPackages")) throw; - } } }; - Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json"; - - if (useCache && pathExists(jsonCacheFileName)) { - - warn("using cached results; pass '-u' to update the cache"); - - Value vRoot; - parseJSON(*state, readFile(jsonCacheFileName), vRoot); - - fromCache = true; - - doExpr(&vRoot, "", true, nullptr); - } - - else { - createDirs(dirOf(jsonCacheFileName)); + for (auto & [cursor, prefix] : installable->getCursors(*state)) + visit(*cursor, parseAttrPath(*state, prefix), true); - Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid()); - - std::ofstream jsonCacheFile; - - try { - // iostream considered harmful - jsonCacheFile.exceptions(std::ofstream::failbit); - jsonCacheFile.open(tmpFile); - - auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr; - - doExpr(getSourceExpr(*state), "", true, cache.get()); - - } catch (std::exception &) { - /* Fun fact: catching std::ios::failure does not work - due to C++11 ABI shenanigans. - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ - if (!jsonCacheFile) - throw Error("error writing to %s", tmpFile); - throw; - } - - if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) - throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); - } - - if (!json && results.size() == 0) + if (!json && !results) throw Error("no results for the given search term(s)!"); - - RunPager pager; - for (auto el : results) std::cout << el.second << "\n"; - } }; -static auto r1 = registerCommand<CmdSearch>("search"); +static auto rCmdSearch = registerCommand<CmdSearch>("search"); diff --git a/src/nix/search.md b/src/nix/search.md new file mode 100644 index 000000000..d182788a6 --- /dev/null +++ b/src/nix/search.md @@ -0,0 +1,72 @@ +R""( + +# Examples + +* Show all packages in the `nixpkgs` flake: + + ```console + # nix search nixpkgs + * legacyPackages.x86_64-linux.AMB-plugins (0.8.1) + A set of ambisonics ladspa plugins + + * legacyPackages.x86_64-linux.ArchiSteamFarm (4.3.1.0) + Application with primary purpose of idling Steam cards from multiple accounts simultaneously + … + ``` + +* Show packages in the `nixpkgs` flake containing `blender` in its + name or description: + + ```console + # nix search nixpkgs blender + * legacyPackages.x86_64-linux.blender (2.91.0) + 3D Creation/Animation/Publishing System + ``` + +* Search for packages underneath the attribute `gnome3` in Nixpkgs: + + ```console + # nix search nixpkgs#gnome3 vala + * legacyPackages.x86_64-linux.gnome3.vala (0.48.9) + Compiler for GObject type system + ``` + +* Show all packages in the flake in the current directory: + + ```console + # nix search + ``` + +* Search for Firefox or Chromium: + + ```console + # nix search nixpkgs 'firefox|chromium' + ``` + +* Search for packages containing `git'`and either `frontend` or `gui`: + + ```console + # nix search nixpkgs git 'frontend|gui' + ``` + +# Description + +`nix search` searches *installable* (which must be evaluatable, e.g. a +flake) for packages whose name or description matches all of the +regular expressions *regex*. For each matching package, It prints the +full attribute name (from the root of the installable), the version +and the `meta.description` field, highlighting the substrings that +were matched by the regular expressions. If no regular expressions are +specified, all packages are shown. + +# Flake output attributes + +If no flake output attribute is given, `nix search` searches for +packages: + +* Directly underneath `packages.<system>`. + +* Underneath `legacyPackages.<system>`, recursing into attribute sets + that contain an attribute `recurseForDerivations = true`. + +)"" diff --git a/src/nix/shell.md b/src/nix/shell.md new file mode 100644 index 000000000..2a379e03f --- /dev/null +++ b/src/nix/shell.md @@ -0,0 +1,48 @@ +R""( + +# Examples + +* Start a shell providing `youtube-dl` from the `nixpkgs` flake: + + ```console + # nix shell nixpkgs#youtube-dl + # youtube-dl --version + 2020.11.01.1 + ``` + +* Start a shell providing GNU Hello from NixOS 20.03: + + ```console + # nix shell nixpkgs/nixos-20.03#hello + ``` + +* Run GNU Hello: + + ```console + # nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!' + Hi everybody! + ``` + +* Run GNU Hello in a chroot store: + + ```console + # nix shell --store ~/my-nix nixpkgs#hello -c hello + ``` + +* Start a shell providing GNU Hello in a chroot store: + + ```console + # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash + ``` + + Note that it's necessary to specify `bash` explicitly because your + default shell (e.g. `/bin/bash`) generally will not exist in the + chroot. + +# Description + +`nix shell` runs a command in an environment in which the `$PATH` +variable provides the specified *installables*. If not command is +specified, it starts the default shell of your user account. + +)"" diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 4fd8886de..29944e748 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -2,7 +2,8 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" -#include "json.hh" + +#include <nlohmann/json.hpp> using namespace nix; @@ -19,15 +20,11 @@ struct CmdShowConfig : Command, MixJSON { if (json) { // FIXME: use appropriate JSON types (bool, ints, etc). - JSONObject jsonObj(std::cout); - globalConfig.toJSON(jsonObj); + logger->cout("%s", globalConfig.toJSON().dump()); } else { - std::map<std::string, Config::SettingInfo> settings; - globalConfig.getSettings(settings); - for (auto & s : settings) - logger->stdout("%s = %s", s.first, s.second.value); + logger->cout("%s", globalConfig.toKeyValue()); } } }; -static auto r1 = registerCommand<CmdShowConfig>("show-config"); +static auto rShowConfig = registerCommand<CmdShowConfig>("show-config"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 5d77cfdca..2588a011d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -1,4 +1,5 @@ // FIXME: integrate this with nix path-info? +// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'? #include "command.hh" #include "common-args.hh" @@ -18,7 +19,7 @@ struct CmdShowDerivation : InstallablesCommand addFlag({ .longName = "recursive", .shortName = 'r', - .description = "include the dependencies of the specified derivations", + .description = "Include the dependencies of the specified derivations.", .handler = {&recursive, true} }); } @@ -28,18 +29,11 @@ struct CmdShowDerivation : InstallablesCommand return "show the contents of a store derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show the store derivation that results from evaluating the Hello package:", - "nix show-derivation nixpkgs.hello" - }, - Example{ - "To show the full derivation graph (if available) that produced your NixOS system:", - "nix show-derivation -r /run/current-system" - }, - }; + return + #include "show-derivation.md" + ; } Category category() override { return catUtility; } @@ -67,13 +61,23 @@ struct CmdShowDerivation : InstallablesCommand { auto outputsObj(drvObj.object("outputs")); - for (auto & output : drv.outputs) { - auto outputObj(outputsObj.object(output.first)); - outputObj.attr("path", store->printStorePath(output.second.path)); - if (output.second.hash) { - outputObj.attr("hashAlgo", output.second.hash->printMethodAlgo()); - outputObj.attr("hash", output.second.hash->hash.to_string(Base16, false)); - } + for (auto & [_outputName, output] : drv.outputs) { + auto & outputName = _outputName; // work around clang bug + auto outputObj { outputsObj.object(outputName) }; + std::visit(overloaded { + [&](DerivationOutputInputAddressed doi) { + outputObj.attr("path", store->printStorePath(doi.path)); + }, + [&](DerivationOutputCAFixed dof) { + outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName))); + outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); + outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); + }, + [&](DerivationOutputCAFloating dof) { + outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); + }, + [&](DerivationOutputDeferred) {}, + }, output.output); } } @@ -92,7 +96,7 @@ struct CmdShowDerivation : InstallablesCommand } } - drvObj.attr("platform", drv.platform); + drvObj.attr("system", drv.platform); drvObj.attr("builder", drv.builder); { @@ -114,4 +118,4 @@ struct CmdShowDerivation : InstallablesCommand } }; -static auto r1 = registerCommand<CmdShowDerivation>("show-derivation"); +static auto rCmdShowDerivation = registerCommand<CmdShowDerivation>("show-derivation"); diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md new file mode 100644 index 000000000..aa863899c --- /dev/null +++ b/src/nix/show-derivation.md @@ -0,0 +1,103 @@ +R""( + +# Examples + +* Show the store derivation that results from evaluating the Hello + package: + + ```console + # nix show-derivation nixpkgs#hello + { + "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { + … + } + } + ``` + +* Show the full derivation graph (if available) that produced your + NixOS system: + + ```console + # nix show-derivation -r /run/current-system + ``` + +* Print all files fetched using `fetchurl` by Firefox's dependency + graph: + + ```console + # nix show-derivation -r nixpkgs#firefox \ + | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ + | uniq | sort + ``` + + Note that `.outputs.out.hash` selects *fixed-output derivations* + (derivations that produce output with a specified content hash), + while `.env.urls` selects derivations with a `urls` attribute. + +# Description + +This command prints on standard output a JSON representation of the +store derivations to which *installables* evaluate. Store derivations +are used internally by Nix. They are store paths with extension `.drv` +that represent the build-time dependency graph to which a Nix +expression evaluates. + +By default, this command only shows top-level derivations, but with +`--recursive`, it also shows their dependencies. + +The JSON output is a JSON object whose keys are the store paths of the +derivations, and whose values are a JSON object with the following +fields: + +* `outputs`: Information about the output paths of the + derivation. This is a JSON object with one member per output, where + the key is the output name and the value is a JSON object with these + fields: + + * `path`: The output path. + * `hashAlgo`: For fixed-output derivations, the hashing algorithm + (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a + NAR hash rather than a flat file hash. + * `hash`: For fixed-output derivations, the expected content hash in + base-16. + + Example: + + ```json + "outputs": { + "out": { + "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", + "hashAlgo": "r:sha256", + "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + } + } + ``` + +* `inputSrcs`: A list of store paths on which this derivation depends. + +* `inputDrvs`: A JSON object specifying the derivations on which this + derivation depends, and what outputs of those derivations. For + example, + + ```json + "inputDrvs": { + "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + } + ``` + + specifies that this derivation depends on the `dev` output of + `curl`, and the `out` output of `unzip`. + +* `system`: The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + +* `builder`: The absolute path of the program to be executed to run + the build. Typically this is the `bash` shell + (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). + +* `args`: The command-line arguments passed to the `builder`. + +* `env`: The environment passed to the `builder`. + +)"" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 6c9b9a792..c64b472b6 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Copy signatures from the specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }}, }); @@ -24,11 +24,9 @@ struct CmdCopySigs : StorePathsCommand std::string description() override { - return "copy path signatures from substituters (like binary caches)"; + return "copy store path signatures from substituters"; } - Category category() override { return catUtility; } - void run(ref<Store> store, StorePaths storePaths) override { if (substituterUris.empty()) @@ -92,30 +90,29 @@ struct CmdCopySigs : StorePathsCommand } }; -static auto r1 = registerCommand<CmdCopySigs>("copy-sigs"); +static auto rCmdCopySigs = registerCommand2<CmdCopySigs>({"store", "copy-sigs"}); -struct CmdSignPaths : StorePathsCommand +struct CmdSign : StorePathsCommand { Path secretKeyFile; - CmdSignPaths() + CmdSign() { addFlag({ .longName = "key-file", .shortName = 'k', - .description = "file containing the secret signing key", + .description = "File containing the secret signing key.", .labels = {"file"}, - .handler = {&secretKeyFile} + .handler = {&secretKeyFile}, + .completer = completePath }); } std::string description() override { - return "sign the specified paths"; + return "sign store paths"; } - Category category() override { return catUtility; } - void run(ref<Store> store, StorePaths storePaths) override { if (secretKeyFile.empty()) @@ -143,4 +140,89 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto r2 = registerCommand<CmdSignPaths>("sign-paths"); +static auto rCmdSign = registerCommand2<CmdSign>({"store", "sign"}); + +struct CmdKeyGenerateSecret : Command +{ + std::optional<std::string> keyName; + + CmdKeyGenerateSecret() + { + addFlag({ + .longName = "key-name", + .description = "Identifier of the key (e.g. `cache.example.org-1`).", + .labels = {"name"}, + .handler = {&keyName}, + }); + } + + std::string description() override + { + return "generate a secret key for signing store paths"; + } + + std::string doc() override + { + return + #include "key-generate-secret.md" + ; + } + + void run() override + { + if (!keyName) + throw UsageError("required argument '--key-name' is missing"); + + std::cout << SecretKey::generate(*keyName).to_string(); + } +}; + +struct CmdKeyConvertSecretToPublic : Command +{ + std::string description() override + { + return "generate a public key for verifying store paths from a secret key read from standard input"; + } + + std::string doc() override + { + return + #include "key-convert-secret-to-public.md" + ; + } + + void run() override + { + SecretKey secretKey(drainFD(STDIN_FILENO)); + std::cout << secretKey.toPublicKey().to_string(); + } +}; + +struct CmdKey : NixMultiCommand +{ + CmdKey() + : MultiCommand({ + {"generate-secret", []() { return make_ref<CmdKeyGenerateSecret>(); }}, + {"convert-secret-to-public", []() { return make_ref<CmdKeyConvertSecretToPublic>(); }}, + }) + { + } + + std::string description() override + { + return "generate and convert Nix signing keys"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + settings.requireExperimentalFeature("flakes"); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdKey = registerCommand<CmdKey>("key"); diff --git a/src/nix/store-cat.md b/src/nix/store-cat.md new file mode 100644 index 000000000..da2073473 --- /dev/null +++ b/src/nix/store-cat.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Show the contents of a file in a binary cache: + + ```console + # nix store cat --store https://cache.nixos.org/ \ + /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello | hexdump -C | head -n1 + 00000000 7f 45 4c 46 02 01 01 00 00 00 00 00 00 00 00 00 |.ELF............| + ``` + +# Description + +This command prints on standard output the contents of the regular +file *path* in a Nix store. *path* can be a top-level store path or +any file inside a store path. + +)"" diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc new file mode 100644 index 000000000..10245978e --- /dev/null +++ b/src/nix/store-delete.cc @@ -0,0 +1,44 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreDelete : StorePathsCommand +{ + GCOptions options { .action = GCOptions::gcDeleteSpecific }; + + CmdStoreDelete() + { + addFlag({ + .longName = "ignore-liveness", + .description = "Do not check whether the paths are reachable from a root.", + .handler = {&options.ignoreLiveness, true} + }); + } + + std::string description() override + { + return "delete paths from the Nix store"; + } + + std::string doc() override + { + return + #include "store-delete.md" + ; + } + + void run(ref<Store> store, std::vector<StorePath> storePaths) override + { + for (auto & path : storePaths) + options.pathsToDelete.insert(path); + + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreDelete = registerCommand2<CmdStoreDelete>({"store", "delete"}); diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md new file mode 100644 index 000000000..db535f87c --- /dev/null +++ b/src/nix/store-delete.md @@ -0,0 +1,24 @@ +R""( + +# Examples + +* Delete a specific store path: + + ```console + # nix store delete /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command deletes the store paths specified by *installables*. , +but only if it is safe to do so; that is, when the path is not +reachable from a root of the garbage collector. This means that you +can only delete paths that would also be deleted by `nix store +gc`. Thus, `nix store delete` is a more targeted version of `nix store +gc`. + +With the option `--ignore-liveness`, reachability from the roots is +ignored. However, the path still won't be deleted if there are other +paths in the store that refer to it (i.e., depend on it). + +)"" diff --git a/src/nix/store-dump-path.md b/src/nix/store-dump-path.md new file mode 100644 index 000000000..4ef563526 --- /dev/null +++ b/src/nix/store-dump-path.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* To get a NAR containing the GNU Hello package: + + ```console + # nix store dump-path nixpkgs#hello > hello.nar + ``` + +* To get a NAR from the binary cache https://cache.nixos.org/: + + ```console + # nix store dump-path --store https://cache.nixos.org/ \ + /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25 > glibc.nar + ``` + +# Description + +This command generates a NAR file containing the serialisation of the +store path *installable*. The NAR is written to standard output. + +)"" diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc new file mode 100644 index 000000000..a2d74066e --- /dev/null +++ b/src/nix/store-gc.cc @@ -0,0 +1,43 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreGC : StoreCommand, MixDryRun +{ + GCOptions options; + + CmdStoreGC() + { + addFlag({ + .longName = "max", + .description = "Stop after freeing *n* bytes of disk space.", + .labels = {"n"}, + .handler = {&options.maxFreed} + }); + } + + std::string description() override + { + return "perform garbage collection on a Nix store"; + } + + std::string doc() override + { + return + #include "store-gc.md" + ; + } + + void run(ref<Store> store) override + { + options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + GCResults results; + PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreGC = registerCommand2<CmdStoreGC>({"store", "gc"}); diff --git a/src/nix/store-gc.md b/src/nix/store-gc.md new file mode 100644 index 000000000..956b3c872 --- /dev/null +++ b/src/nix/store-gc.md @@ -0,0 +1,21 @@ +R""( + +# Examples + +* Delete unreachable paths in the Nix store: + + ```console + # nix store gc + ``` + +* Delete up to 1 gigabyte of garbage: + + ```console + # nix store gc --max 1G + ``` + +# Description + +This command deletes unreachable paths in the Nix store. + +)"" diff --git a/src/nix/store-ls.md b/src/nix/store-ls.md new file mode 100644 index 000000000..836efce42 --- /dev/null +++ b/src/nix/store-ls.md @@ -0,0 +1,27 @@ +R""( + +# Examples + +* To list the contents of a store path in a binary cache: + + ```console + # nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + dr-xr-xr-x 0 ./bin + -r-xr-xr-x 38184 ./bin/hello + dr-xr-xr-x 0 ./share + … + ``` + +* To show information about a specific file in a binary cache: + + ```console + # nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello + -r-xr-xr-x 38184 hello + ``` + +# Description + +This command shows information about *path* in a Nix store. *path* can +be a top-level store path or any file inside a store path. + +)"" diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md new file mode 100644 index 000000000..f9fdcbc57 --- /dev/null +++ b/src/nix/store-prefetch-file.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Download a file to the Nix store: + + ```console + # nix store prefetch-file https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz + Downloaded 'https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz' to + '/nix/store/vbdbi42hgnc4h7pyqzp6h2yf77kw93aw-source' (hash + 'sha256-qKheVd5D0BervxMDbt+1hnTKE2aRWC8XCAwc0SeHt6s='). + ``` + +* Download a file and get the SHA-512 hash: + + ```console + # nix store prefetch-file --json --hash-type sha512 \ + https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz \ + | jq -r .hash + sha512-6XJxfym0TNH9knxeH4ZOvns6wElFy3uahunl2hJgovACCMEMXSy42s69zWVyGJALXTI+86tpDJGlIcAySEKBbA== + ``` + +# Description + +This command downloads the file *url* to the Nix store. It prints out +the resulting store path and the cryptographic hash of the contents of +the file. + +The name component of the store path defaults to the last component of +*url*, but this can be overridden using `--name`. + +)"" diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc new file mode 100644 index 000000000..1c7a4392e --- /dev/null +++ b/src/nix/store-repair.cc @@ -0,0 +1,27 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreRepair : StorePathsCommand +{ + std::string description() override + { + return "repair store paths"; + } + + std::string doc() override + { + return + #include "store-repair.md" + ; + } + + void run(ref<Store> store, std::vector<StorePath> storePaths) override + { + for (auto & path : storePaths) + store->repairPath(path); + } +}; + +static auto rStoreRepair = registerCommand2<CmdStoreRepair>({"store", "repair"}); diff --git a/src/nix/store-repair.md b/src/nix/store-repair.md new file mode 100644 index 000000000..92d2205a9 --- /dev/null +++ b/src/nix/store-repair.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Repair a store path, after determining that it is corrupt: + + ```console + # nix store verify /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + path '/nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10' was + modified! expected hash + 'sha256:1hd5vnh6xjk388gdk841vflicy8qv7qzj2hb7xlyh8lpb43j921l', got + 'sha256:1a25lf78x5wi6pfkrxalf0n13kdaca0bqmjqnp7wfjza2qz5ssgl' + + # nix store repair /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command attempts to "repair" the store paths specified by +*installables* by redownloading them using the available +substituters. If no substitutes are available, then repair is not +possible. + +> **Warning** +> +> During repair, there is a very small time window during which the old +> path (if it exists) is moved out of the way and replaced with the new +> path. If repair is interrupted in between, then the system may be left +> in a broken state (e.g., if the path contains a critical system +> component like the GNU C Library). + +)"" diff --git a/src/nix/store.cc b/src/nix/store.cc new file mode 100644 index 000000000..44e53c7c7 --- /dev/null +++ b/src/nix/store.cc @@ -0,0 +1,26 @@ +#include "command.hh" + +using namespace nix; + +struct CmdStore : virtual NixMultiCommand +{ + CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"})) + { } + + std::string description() override + { + return "manipulate a Nix store"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix store' requires a sub-command."); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdStore = registerCommand<CmdStore>("store"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index a880bdae0..9cd567896 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -19,14 +19,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand addFlag({ .longName = "profile", .shortName = 'p', - .description = "the Nix profile to upgrade", + .description = "The path to the Nix profile to upgrade.", .labels = {"profile-dir"}, .handler = {&profileDir} }); addFlag({ .longName = "nix-store-paths-url", - .description = "URL of the file that contains the store paths of the latest Nix release", + .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, .handler = {&storePathsUrl} }); @@ -37,18 +37,11 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand return "upgrade Nix to the latest stable version"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To upgrade Nix to the latest stable version:", - "nix upgrade-nix" - }, - Example{ - "To upgrade Nix in a specific profile:", - "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile" - }, - }; + return + #include "upgrade-nix.md" + ; } Category category() override { return catNixInstallation; } @@ -68,10 +61,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (dryRun) { stopProgressBar(); - logWarning({ - .name = "Version update", - .hint = hintfmt("would upgrade to version %s", version) - }); + warn("would upgrade to version %s", version); return; } @@ -158,4 +148,4 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand } }; -static auto r1 = registerCommand<CmdUpgradeNix>("upgrade-nix"); +static auto rCmdUpgradeNix = registerCommand<CmdUpgradeNix>("upgrade-nix"); diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md new file mode 100644 index 000000000..4d27daad9 --- /dev/null +++ b/src/nix/upgrade-nix.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Upgrade Nix to the latest stable version: + + ```console + # nix upgrade-nix + ``` + +* Upgrade Nix in a specific profile: + + ```console + # nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile + ``` + +# Description + +This command upgrades Nix to the latest version. By default, it +locates the directory containing the `nix` binary in the `$PATH` +environment variable. If that directory is a Nix profile, it will +upgrade the `nix` package in that profile to the latest stable binary +release. + +You cannot use this command to upgrade Nix in the system profile of a +NixOS system (that is, if `nix` is found in `/run/current-system`). + +)"" diff --git a/src/nix/verify.cc b/src/nix/verify.cc index bb5e4529b..f5a576064 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -18,16 +18,33 @@ struct CmdVerify : StorePathsCommand CmdVerify() { - mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); - mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + addFlag({ + .longName = "no-contents", + .description = "Do not verify the contents of each store path.", + .handler = {&noContents, true}, + }); + + addFlag({ + .longName = "no-trust", + .description = "Do not verify whether each store path is trusted.", + .handler = {&noTrust, true}, + }); + addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from the specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }} }); - mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + + addFlag({ + .longName = "sigs-needed", + .shortName = 'n', + .description = "Require that each path has at least *n* valid signatures.", + .labels = {"n"}, + .handler = {&sigsNeeded} + }); } std::string description() override @@ -35,22 +52,13 @@ struct CmdVerify : StorePathsCommand return "verify the integrity of store paths"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To verify the entire Nix store:", - "nix verify --all" - }, - Example{ - "To check whether each path in the closure of Firefox has at least 2 signatures:", - "nix verify -r -n2 --no-contents $(type -p firefox)" - }, - }; + return + #include "verify.md" + ; } - Category category() override { return catSecondary; } - void run(ref<Store> store, StorePaths storePaths) override { std::vector<ref<Store>> substituters; @@ -77,36 +85,31 @@ struct CmdVerify : StorePathsCommand try { checkInterrupt(); - Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath)); - MaintainCount<std::atomic<size_t>> mcActive(active); update(); auto info = store->queryPathInfo(store->parseStorePath(storePath)); + // Note: info->path can be different from storePath + // for binary cache stores when using --all (since we + // can't enumerate names efficiently). + Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", store->printStorePath(info->path))); + if (!noContents) { - std::unique_ptr<AbstractHashSink> hashSink; - if (!info->ca) - hashSink = std::make_unique<HashSink>(*info->narHash.type); - else - hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart())); + auto hashSink = HashSink(info->narHash.type); - store->narFromPath(info->path, *hashSink); + store->narFromPath(info->path, hashSink); - auto hash = hashSink->finish(); + auto hash = hashSink.finish(); if (hash.first != info->narHash) { corrupted++; act2.result(resCorruptedPath, store->printStorePath(info->path)); - logError({ - .name = "Hash error - path modified", - .hint = hintfmt( - "path '%s' was modified! expected hash '%s', got '%s'", - store->printStorePath(info->path), - info->narHash.to_string(Base32, true), - hash.first.to_string(Base32, true)) - }); + printError("path '%s' was modified! expected hash '%s', got '%s'", + store->printStorePath(info->path), + info->narHash.to_string(Base32, true), + hash.first.to_string(Base32, true)); } } @@ -154,12 +157,7 @@ struct CmdVerify : StorePathsCommand if (!good) { untrusted++; act2.result(resUntrustedPath, store->printStorePath(info->path)); - logError({ - .name = "Untrusted path", - .hint = hintfmt("path '%s' is untrusted", - store->printStorePath(info->path)) - }); - + printError("path '%s' is untrusted", store->printStorePath(info->path)); } } @@ -186,4 +184,4 @@ struct CmdVerify : StorePathsCommand } }; -static auto r1 = registerCommand<CmdVerify>("verify"); +static auto rCmdVerify = registerCommand2<CmdVerify>({"store", "verify"}); diff --git a/src/nix/verify.md b/src/nix/verify.md new file mode 100644 index 000000000..1c43792e7 --- /dev/null +++ b/src/nix/verify.md @@ -0,0 +1,49 @@ +R""( + +# Examples + +* Verify the entire Nix store: + + ```console + # nix store verify --all + ``` + +* Check whether each path in the closure of Firefox has at least 2 + signatures: + + ```console + # nix store verify -r -n2 --no-contents $(type -p firefox) + ``` + +* Verify a store path in the binary cache `https://cache.nixos.org/`: + + ```console + # nix store verify --store https://cache.nixos.org/ \ + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + ``` + +# Description + +This command verifies the integrity of the store paths *installables*, +or, if `--all` is given, the entire Nix store. For each path, it +checks that + +* its contents match the NAR hash recorded in the Nix database; and + +* it is *trusted*, that is, it is signed by at least one trusted + signing key, is content-addressed, or is built locally ("ultimately + trusted"). + +# Exit status + +The exit status of this command is the sum of the following values: + +* **1** if any path is corrupted (i.e. its contents don't match the + recorded NAR hash). + +* **2** if any path is untrusted. + +* **4** if any path couldn't be verified for any other reason (such as + an I/O error). + +)"" diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 167c974ee..2f6b361bb 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -40,7 +40,7 @@ struct CmdWhyDepends : SourceExprCommand addFlag({ .longName = "all", .shortName = 'a', - .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path", + .description = "Show all edges in the dependency graph leading from *package* to *dependency*, rather than just a shortest path.", .handler = {&all, true}, }); } @@ -50,39 +50,30 @@ struct CmdWhyDepends : SourceExprCommand return "show why a package has another package in its closure"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show one path through the dependency graph leading from Hello to Glibc:", - "nix why-depends nixpkgs.hello nixpkgs.glibc" - }, - Example{ - "To show all files and paths in the dependency graph leading from Thunderbird to libX11:", - "nix why-depends --all nixpkgs.thunderbird nixpkgs.xorg.libX11" - }, - Example{ - "To show why Glibc depends on itself:", - "nix why-depends nixpkgs.glibc nixpkgs.glibc" - }, - }; + return + #include "why-depends.md" + ; } Category category() override { return catSecondary; } void run(ref<Store> store) override { - auto package = parseInstallable(*this, store, _package, false); - auto packagePath = toStorePath(store, Build, package); - auto dependency = parseInstallable(*this, store, _dependency, false); - auto dependencyPath = toStorePath(store, NoBuild, dependency); + auto package = parseInstallable(store, _package); + auto packagePath = toStorePath(getEvalStore(), store, Realise::Outputs, operateOn, package); + auto dependency = parseInstallable(store, _dependency); + auto dependencyPath = toStorePath(getEvalStore(), store, Realise::Derivation, operateOn, dependency); auto dependencyPathHash = dependencyPath.hashPart(); StorePathSet closure; store->computeFSClosure({packagePath}, closure, false, false); if (!closure.count(dependencyPath)) { - printError("'%s' does not depend on '%s'", package->what(), dependency->what()); + printError("'%s' does not depend on '%s'", + store->printStorePath(packagePath), + store->printStorePath(dependencyPath)); return; } @@ -106,7 +97,11 @@ struct CmdWhyDepends : SourceExprCommand std::map<StorePath, Node> graph; for (auto & path : closure) - graph.emplace(path, Node { .path = path, .refs = store->queryPathInfo(path)->references }); + graph.emplace(path, Node { + .path = path, + .refs = store->queryPathInfo(path)->references, + .dist = path == dependencyPath ? 0 : inf + }); // Transpose the graph. for (auto & node : graph) @@ -115,8 +110,6 @@ struct CmdWhyDepends : SourceExprCommand /* Run Dijkstra's shortest path algorithm to get the distance of every path in the closure to 'dependency'. */ - graph.emplace(dependencyPath, Node { .path = dependencyPath, .dist = 0 }); - std::priority_queue<Node *> queue; queue.push(&graph.at(dependencyPath)); @@ -152,7 +145,7 @@ struct CmdWhyDepends : SourceExprCommand auto pathS = store->printStorePath(node.path); assert(node.dist != inf); - logger->stdout("%s%s%s%s" ANSI_NORMAL, + logger->cout("%s%s%s%s" ANSI_NORMAL, firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", @@ -259,4 +252,4 @@ struct CmdWhyDepends : SourceExprCommand } }; -static auto r1 = registerCommand<CmdWhyDepends>("why-depends"); +static auto rCmdWhyDepends = registerCommand<CmdWhyDepends>("why-depends"); diff --git a/src/nix/why-depends.md b/src/nix/why-depends.md new file mode 100644 index 000000000..dc13619e1 --- /dev/null +++ b/src/nix/why-depends.md @@ -0,0 +1,80 @@ +R""( + +# Examples + +* Show one path through the dependency graph leading from Hello to + Glibc: + + ```console + # nix why-depends nixpkgs#hello nixpkgs#glibc + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + └───bin/hello: …...................../nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32/lib/ld-linux-x86-64.… + → /nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32 + ``` + +* Show all files and paths in the dependency graph leading from + Thunderbird to libX11: + + ```console + # nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11 + /nix/store/qfc8729nzpdln1h0hvi1ziclsl3m84sr-thunderbird-78.5.1 + ├───lib/thunderbird/libxul.so: …6wrw-libxcb-1.14/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/ssf… + │ → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0 + ├───lib/thunderbird/libxul.so: …pxyc-libXt-1.2.0/lib:/nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5/lib:/nix/store… + │ → /nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5 + │ ├───lib/libXdamage.so.1.1.0: …-libXfixes-5.0.3/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/9l0… + │ │ → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0 + … + ``` + +* Show why Glibc depends on itself: + + ```console + # nix why-depends nixpkgs#glibc nixpkgs#glibc + /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31 + └───lib/ld-2.31.so: …che Do not use /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/etc/ld.so.cache. --… + → /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31 + ``` + +* Show why Geeqie has a build-time dependency on `systemd`: + + ```console + # nix why-depends --derivation nixpkgs#geeqie nixpkgs#systemd + /nix/store/drrpq2fqlrbj98bmazrnww7hm1in3wgj-geeqie-1.4.drv + └───/: …atch.drv",["out"]),("/nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv",["dev"]),("/… + → /nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv + └───/: …16.0.drv",["dev"]),("/nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv",["dev"]),("/nix… + → /nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv + └───/: ….3.1.drv",["out"]),("/nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv",["dev"]),("/… + → /nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv + ``` + +# Description + +Nix automatically determines potential runtime dependencies between +store paths by scanning for the *hash parts* of store paths. For +instance, if there exists a store path +`/nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31`, and a file +inside another store path contains the string `9df65igw…`, then the +latter store path *refers* to the former, and thus might need it at +runtime. Nix always maintains the existence of the transitive closure +of a store path under the references relationship; it is therefore not +possible to install a store path without having all of its references +present. + +Sometimes Nix packages end up with unexpected runtime dependencies; +for instance, a reference to a compiler might accidentally end up in a +binary, causing the former to be in the latter's closure. This kind of +*closure size bloat* is undesirable. + +`nix why-depends` allows you to diagnose the cause of such issues. It +shows why the store path *package* depends on the store path +*dependency*, by showing a shortest sequence in the references graph +from the former to the latter. Also, for each node along this path, it +shows a file fragment containing a reference to the next store path in +the sequence. + +To show why derivation *package* has a build-time rather than runtime +dependency on derivation *dependency*, use `--derivation`. + +)"" |