diff options
45 files changed, 637 insertions, 222 deletions
diff --git a/.gitignore b/.gitignore index 58e7377fb..db363aefd 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ perl/Makefile.config /tests/shell.drv /tests/config.nix /tests/ca/config.nix +/tests/repl-result-out # /tests/lang/ /tests/lang/*.out @@ -1 +1 @@ -2.8.0
\ No newline at end of file +2.9.0
\ No newline at end of file diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index f0f9457d2..860222337 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -72,6 +72,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md) - [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md) - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md) - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md) diff --git a/doc/manual/src/release-notes/rl-2.8.md b/doc/manual/src/release-notes/rl-2.8.md new file mode 100644 index 000000000..9778e8c3a --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.8.md @@ -0,0 +1,53 @@ +# Release 2.8 (2022-04-19) + +* New experimental command: `nix fmt`, which applies a formatter + defined by the `formatter.<system>` flake output to the Nix + expressions in a flake. + +* Various Nix commands can now read expressions from standard input + using `--file -`. + +* New experimental builtin function `builtins.fetchClosure` that + copies a closure from a binary cache at evaluation time and rewrites + it to content-addressed form (if it isn't already). Like + `builtins.storePath`, this allows importing pre-built store paths; + the difference is that it doesn't require the user to configure + binary caches and trusted public keys. + + This function is only available if you enable the experimental + feature `fetch-closure`. + +* New experimental feature: *impure derivations*. These are + derivations that can produce a different result every time they're + built. Here is an example: + + ```nix + stdenv.mkDerivation { + name = "impure"; + __impure = true; # marks this derivation as impure + buildCommand = "date > $out"; + } + ``` + + Running `nix build` twice on this expression will build the + derivation twice, producing two different content-addressed store + paths. Like fixed-output derivations, impure derivations have access + to the network. Only fixed-output derivations and impure derivations + can depend on an impure derivation. + +* `nix store make-content-addressable` has been renamed to `nix store + make-content-addressed`. + +* The `nixosModule` flake output attribute has been renamed consistent + with the `.default` renames in Nix 2.7. + + * `nixosModule` → `nixosModules.default` + + As before, the old output will continue to work, but `nix flake check` will + issue a warning about it. + +* `nix run` is now stricter in what it accepts: members of the `apps` + flake output are now required to be apps (as defined in [the + manual](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-run.html#apps)), + and members of `packages` or `legacyPackages` must be derivations + (not apps). diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 8c8c0fd41..3e2998c6c 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,42 +1,4 @@ # Release X.Y (202?-??-??) -* Various nix commands can now read expressions from stdin with `--file -`. - -* `nix store make-content-addressable` has been renamed to `nix store - make-content-addressed`. - -* New experimental builtin function `builtins.fetchClosure` that - copies a closure from a binary cache at evaluation time and rewrites - it to content-addressed form (if it isn't already). Like - `builtins.storePath`, this allows importing pre-built store paths; - the difference is that it doesn't require the user to configure - binary caches and trusted public keys. - - This function is only available if you enable the experimental - feature `fetch-closure`. - -* New experimental feature: *impure derivations*. These are - derivations that can produce a different result every time they're - built. Here is an example: - - ```nix - stdenv.mkDerivation { - name = "impure"; - __impure = true; # marks this derivation as impure - buildCommand = "date > $out"; - } - ``` - - Running `nix build` twice on this expression will build the - derivation twice, producing two different content-addressed store - paths. Like fixed-output derivations, impure derivations have access - to the network. Only fixed-output derivations and impure derivations - can depend on an impure derivation. - -* The `nixosModule` flake output attribute has been renamed consistent - with the `.default` renames in nix 2.7. - - * `nixosModule` → `nixosModules.default` - - As before, the old output will continue to work, but `nix flake check` will - issue a warning about it. +* `nix repl` has a new build-'n-link (`:bl`) command that builds a derivation + while creating GC root symlinks. diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh index 045053dee..9af695f5a 100644 --- a/misc/bash/completion.sh +++ b/misc/bash/completion.sh @@ -15,7 +15,7 @@ function _complete_nix { else COMPREPLY+=("$completion") fi - done < <(NIX_GET_COMPLETIONS=$cword "${words[@]/#\~/$HOME}" 2>/dev/null) + done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}" 2>/dev/null) __ltrim_colon_completions "$cur" } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 0f6125f11..84bbb5292 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -85,11 +85,12 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions { std::optional<Path> file; std::optional<std::string> expr; + bool readOnlyMode = false; // FIXME: move this; not all commands (e.g. 'nix run') use it. OperateOn operateOn = OperateOn::Output; - SourceExprCommand(); + SourceExprCommand(bool supportReadOnlyMode = false); std::vector<std::shared_ptr<Installable>> parseInstallables( ref<Store> store, std::vector<std::string> ss); @@ -128,13 +129,13 @@ struct InstallableCommand : virtual Args, SourceExprCommand { std::shared_ptr<Installable> installable; - InstallableCommand(); + InstallableCommand(bool supportReadOnlyMode = false); void prepare() override; std::optional<FlakeRef> getFlakeRefForCompletion() override { - return parseFlakeRef(_installable, absPath(".")); + return parseFlakeRefWithFragment(_installable, absPath(".")).first; } private: diff --git a/src/libexpr/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e50ff244c..5b6e82388 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -7,6 +7,7 @@ #include "registry.hh" #include "flake/flakeref.hh" #include "store-api.hh" +#include "command.hh" namespace nix { @@ -59,6 +60,9 @@ MixEvalArgs::MixEvalArgs() fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); + }}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(openStore(), prefix); }} }); diff --git a/src/libexpr/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 03fa226aa..03fa226aa 100644 --- a/src/libexpr/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 955bbe6fb..4250c321e 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,4 +1,6 @@ +#include "globals.hh" #include "installables.hh" +#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" @@ -99,6 +101,14 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), parseFlakeRef(flakeRef, absPath("."), true)); + }}, + .completer = {[&](size_t n, std::string_view prefix) { + if (n == 0) { + if (auto flakeRef = getFlakeRefForCompletion()) + completeFlakeInputPath(getEvalState(), *flakeRef, prefix); + } else if (n == 1) { + completeFlakeRef(getEvalState()->store, prefix); + } }} }); @@ -129,7 +139,7 @@ MixFlakeOptions::MixFlakeOptions() }); } -SourceExprCommand::SourceExprCommand() +SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) { addFlag({ .longName = "file", @@ -157,6 +167,17 @@ SourceExprCommand::SourceExprCommand() .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); + + if (supportReadOnlyMode) { + addFlag({ + .longName = "read-only", + .description = + "Do not instantiate each evaluated derivation. " + "This improves performance, but can cause errors when accessing " + "store paths of derivations during evaluation.", + .handler = {&readOnlyMode, true}, + }); + } } Strings SourceExprCommand::getDefaultFlakeAttrPaths() @@ -182,6 +203,8 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() void SourceExprCommand::completeInstallable(std::string_view prefix) { if (file) { + completionType = ctAttrs; + evalSettings.pureEval = false; auto state = getEvalState(); Expr *e = state->parseExprFromFile( @@ -210,13 +233,14 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) Value v2; state->autoCallFunction(*autoArgs, v1, v2); - completionType = ctAttrs; - if (v2.type() == nAttrs) { for (auto & i : *v2.attrs) { std::string name = i.name; if (name.find(searchWord) == 0) { - completions->add(i.name); + if (prefix_ == "") + completions->add(name); + else + completions->add(prefix_ + "." + name); } } } @@ -244,10 +268,11 @@ void completeFlakeRefWithFragment( if (hash == std::string::npos) { completeFlakeRef(evalState->store, prefix); } else { + completionType = ctAttrs; + auto fragment = prefix.substr(hash + 1); auto flakeRefS = std::string(prefix.substr(0, hash)); - // FIXME: do tilde expansion. - auto flakeRef = parseFlakeRef(flakeRefS, absPath(".")); + auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags))); @@ -259,8 +284,6 @@ void completeFlakeRefWithFragment( flake. */ attrPathPrefixes.push_back(""); - completionType = ctAttrs; - for (auto & attrPathPrefixS : attrPathPrefixes) { auto attrPathPrefix = parseAttrPath(*evalState, attrPathPrefixS); auto attrPathS = attrPathPrefixS + std::string(fragment); @@ -334,16 +357,16 @@ DerivedPath Installable::toDerivedPath() return std::move(buildables[0]); } -std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> +std::vector<ref<eval_cache::AttrCursor>> Installable::getCursors(EvalState & state) { auto evalCache = std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state, [&]() { return toValue(state).first; }); - return {{evalCache->getRoot(), ""}}; + return {evalCache->getRoot()}; } -std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string> +ref<eval_cache::AttrCursor> Installable::getCursor(EvalState & state) { auto cursors = getCursors(state); @@ -566,43 +589,21 @@ InstallableFlake::InstallableFlake( std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation() { - auto lockedFlake = getLockedFlake(); + auto attr = getCursor(*state); - auto cache = openEvalCache(*state, lockedFlake); - auto root = cache->getRoot(); + auto attrPath = attr->getAttrPathStr(); - Suggestions suggestions; + if (!attr->isDerivation()) + throw Error("flake output attribute '%s' is not a derivation", attrPath); - for (auto & attrPath : getActualAttrPaths()) { - debug("trying flake output attribute '%s'", attrPath); + auto drvPath = attr->forceDerivation(); - auto attrOrSuggestions = root->findAlongAttrPath( - parseAttrPath(*state, attrPath), - true - ); - - if (!attrOrSuggestions) { - suggestions += attrOrSuggestions.getSuggestions(); - continue; - } - - auto attr = *attrOrSuggestions; - - if (!attr->isDerivation()) - throw Error("flake output attribute '%s' is not a derivation", attrPath); - - auto drvPath = attr->forceDerivation(); - - auto drvInfo = DerivationInfo { - std::move(drvPath), - attr->getAttr(state->sOutputName)->getString() - }; - - return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)}; - } + auto drvInfo = DerivationInfo { + std::move(drvPath), + attr->getAttr(state->sOutputName)->getString() + }; - throw Error(suggestions, "flake '%s' does not provide attribute %s", - flakeRef, showAttrPaths(getActualAttrPaths())); + return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; } std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() @@ -614,33 +615,10 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state) { - auto lockedFlake = getLockedFlake(); - - auto vOutputs = getFlakeOutputs(state, *lockedFlake); - - auto emptyArgs = state.allocBindings(0); - - Suggestions suggestions; - - for (auto & attrPath : getActualAttrPaths()) { - try { - auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); - state.forceValue(*v, pos); - return {v, pos}; - } catch (AttrPathNotFound & e) { - suggestions += e.info().suggestions; - } - } - - throw Error( - suggestions, - "flake '%s' does not provide attribute %s", - flakeRef, - showAttrPaths(getActualAttrPaths()) - ); + return {&getCursor(state)->forceValue(), noPos}; } -std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> +std::vector<ref<eval_cache::AttrCursor>> InstallableFlake::getCursors(EvalState & state) { auto evalCache = openEvalCache(state, @@ -648,21 +626,55 @@ InstallableFlake::getCursors(EvalState & state) auto root = evalCache->getRoot(); - std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res; + std::vector<ref<eval_cache::AttrCursor>> res; for (auto & attrPath : getActualAttrPaths()) { auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back({*attr, attrPath}); + if (attr) res.push_back(ref(*attr)); } return res; } +ref<eval_cache::AttrCursor> InstallableFlake::getCursor(EvalState & state) +{ + auto lockedFlake = getLockedFlake(); + + auto cache = openEvalCache(state, lockedFlake); + auto root = cache->getRoot(); + + Suggestions suggestions; + + auto attrPaths = getActualAttrPaths(); + + for (auto & attrPath : attrPaths) { + debug("trying flake output attribute '%s'", attrPath); + + auto attrOrSuggestions = root->findAlongAttrPath( + parseAttrPath(state, attrPath), + true + ); + + if (!attrOrSuggestions) { + suggestions += attrOrSuggestions.getSuggestions(); + continue; + } + + return *attrOrSuggestions; + } + + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(attrPaths)); +} + std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const { - flake::LockFlags lockFlagsApplyConfig = lockFlags; - lockFlagsApplyConfig.applyNixConfig = true; if (!_lockedFlake) { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + lockFlagsApplyConfig.applyNixConfig = true; _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; @@ -687,6 +699,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( { std::vector<std::shared_ptr<Installable>> result; + if (readOnlyMode) { + settings.readOnlyMode = true; + } + if (file || expr) { if (file && expr) throw UsageError("'--file' and '--expr' are exclusive"); @@ -815,8 +831,8 @@ std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::bui auto realisation = store->queryRealisation(outputId); if (!realisation) throw Error( - "cannot operate on an output of unbuilt " - "content-addressed derivation '%s'", + "cannot operate on an output of the " + "unbuilt derivation '%s'", outputId.to_string()); outputs.insert_or_assign(output, realisation->outPath); } else { @@ -954,7 +970,7 @@ InstallablesCommand::InstallablesCommand() void InstallablesCommand::prepare() { if (_installables.empty() && useDefaultInstallables()) - // FIXME: commands like "nix install" should not have a + // FIXME: commands like "nix profile install" should not have a // default, probably. _installables.push_back("."); installables = parseInstallables(getStore(), _installables); @@ -964,13 +980,14 @@ std::optional<FlakeRef> InstallablesCommand::getFlakeRefForCompletion() { if (_installables.empty()) { if (useDefaultInstallables()) - return parseFlakeRef(".", absPath(".")); + return parseFlakeRefWithFragment(".", absPath(".")).first; return {}; } - return parseFlakeRef(_installables.front(), absPath(".")); + return parseFlakeRefWithFragment(_installables.front(), absPath(".")).first; } -InstallableCommand::InstallableCommand() +InstallableCommand::InstallableCommand(bool supportReadOnlyMode) + : SourceExprCommand(supportReadOnlyMode) { expectArgs({ .label = "installable", diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index f4bf0d406..b847f8939 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -80,10 +80,10 @@ struct Installable return {}; } - virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> + virtual std::vector<ref<eval_cache::AttrCursor>> getCursors(EvalState & state); - std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string> + virtual ref<eval_cache::AttrCursor> getCursor(EvalState & state); virtual FlakeRef nixpkgsFlakeRef() const @@ -180,9 +180,15 @@ struct InstallableFlake : InstallableValue std::pair<Value *, Pos> toValue(EvalState & state) override; - std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> + /* Get a cursor to every attrpath in getActualAttrPaths() that + exists. */ + std::vector<ref<eval_cache::AttrCursor>> getCursors(EvalState & state) override; + /* Get a cursor to the first attrpath in getActualAttrPaths() that + exists, or throw an exception with suggestions if none exists. */ + ref<eval_cache::AttrCursor> getCursor(EvalState & state) override; + std::shared_ptr<flake::LockedFlake> getLockedFlake() const; FlakeRef nixpkgsFlakeRef() const override; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 54fa9b741..7d3fd01a4 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -306,9 +306,9 @@ Value * EvalCache::getRootValue() return *value; } -std::shared_ptr<AttrCursor> EvalCache::getRoot() +ref<AttrCursor> EvalCache::getRoot() { - return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt); + return make_ref<AttrCursor>(ref(shared_from_this()), std::nullopt); } AttrCursor::AttrCursor( diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index c9a9bf471..b0709ebc2 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -33,7 +33,7 @@ public: EvalState & state, RootLoader rootLoader); - std::shared_ptr<AttrCursor> getRoot(); + ref<AttrCursor> getRoot(); }; enum AttrType { @@ -104,6 +104,8 @@ public: ref<AttrCursor> getAttr(std::string_view name); + /* Get an attribute along a chain of attrsets. Note that this does + not auto-call functors or functions. */ OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); std::string getString(); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 91dd332a2..34b1342a0 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -187,8 +187,16 @@ struct GitInputScheme : InputScheme if (submodules) cacheType += "-submodules"; if (allRefs) cacheType += "-all-refs"; + auto checkHashType = [&](const std::optional<Hash> & hash) + { + if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); + }; + auto getLockedAttrs = [&]() { + checkHashType(input.getRev()); + return Attrs({ {"type", cacheType}, {"name", name}, diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 51cf35bf4..5c5671681 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -36,7 +36,7 @@ static std::string runHg(const Strings & args, const std::optional<std::string> auto res = runProgram(std::move(opts)); if (!statusOk(res.first)) - throw ExecError(res.first, fmt("hg %1%", statusToString(res.first))); + throw ExecError(res.first, "hg %1%", statusToString(res.first)); return res.second; } @@ -203,8 +203,17 @@ struct MercurialInputScheme : InputScheme if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); + auto checkHashType = [&](const std::optional<Hash> & hash) + { + if (hash.has_value() && hash->type != htSHA1) + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); + }; + + auto getLockedAttrs = [&]() { + checkHashType(input.getRev()); + return Attrs({ {"type", "hg"}, {"name", name}, @@ -264,7 +273,7 @@ struct MercurialInputScheme : InputScheme runHg({ "recover", "-R", cacheDir }); runHg({ "pull", "-R", cacheDir, "--", actualUrl }); } else { - throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + throw ExecError(e.status, "'hg pull' %s", statusToString(e.status)); } } } else { diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 562d1b414..31454e49d 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -60,37 +60,37 @@ void printMissing(ref<Store> store, const StorePathSet & willBuild, { if (!willBuild.empty()) { if (willBuild.size() == 1) - printMsg(lvl, fmt("this derivation will be built:")); + printMsg(lvl, "this derivation will be built:"); else - printMsg(lvl, fmt("these %d derivations will be built:", willBuild.size())); + printMsg(lvl, "these %d derivations will be built:", willBuild.size()); auto sorted = store->topoSortPaths(willBuild); reverse(sorted.begin(), sorted.end()); for (auto & i : sorted) - printMsg(lvl, fmt(" %s", store->printStorePath(i))); + printMsg(lvl, " %s", store->printStorePath(i)); } if (!willSubstitute.empty()) { const float downloadSizeMiB = downloadSize / (1024.f * 1024.f); const float narSizeMiB = narSize / (1024.f * 1024.f); if (willSubstitute.size() == 1) { - printMsg(lvl, fmt("this path will be fetched (%.2f MiB download, %.2f MiB unpacked):", + printMsg(lvl, "this path will be fetched (%.2f MiB download, %.2f MiB unpacked):", downloadSizeMiB, - narSizeMiB)); + narSizeMiB); } else { - printMsg(lvl, fmt("these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", + printMsg(lvl, "these %d paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", willSubstitute.size(), downloadSizeMiB, - narSizeMiB)); + narSizeMiB); } for (auto & i : willSubstitute) - printMsg(lvl, fmt(" %s", store->printStorePath(i))); + printMsg(lvl, " %s", store->printStorePath(i)); } if (!unknown.empty()) { - printMsg(lvl, fmt("don't know how to build these paths%s:", - (settings.readOnlyMode ? " (may be caused by read-only store access)" : ""))); + printMsg(lvl, "don't know how to build these paths%s:", + (settings.readOnlyMode ? " (may be caused by read-only store access)" : "")); for (auto & i : unknown) - printMsg(lvl, fmt(" %s", store->printStorePath(i))); + printMsg(lvl, " %s", store->printStorePath(i)); } } diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 64cc97fde..4ca91f585 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -13,12 +13,27 @@ create table if not exists Realisations ( create index if not exists IndexRealisations on Realisations(drvPath, outputName); +-- We can end-up in a weird edge-case where a path depends on itself because +-- it’s an output of a CA derivation, that happens to be the same as one of its +-- dependencies. +-- In that case we have a dependency loop (path -> realisation1 -> realisation2 +-- -> path) that we need to break by removing the dependencies between the +-- realisations +create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths + begin + delete from RealisationsRefs where realisationReference in ( + select id from Realisations where outputPath = old.id + ); + end; + create table if not exists RealisationsRefs ( referrer integer not null, realisationReference integer, foreign key (referrer) references Realisations(id) on delete cascade, foreign key (realisationReference) references Realisations(id) on delete restrict ); +-- used by deletion trigger +create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); -- used by QueryRealisationReferences create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index c46262299..529a41891 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -443,14 +443,13 @@ struct curlFileTransfer : public FileTransfer : httpStatus != 0 ? FileTransferError(err, std::move(response), - fmt("unable to %s '%s': HTTP error %d ('%s')", - request.verb(), request.uri, httpStatus, statusMsg) - + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) - ) + "unable to %s '%s': HTTP error %d%s", + request.verb(), request.uri, httpStatus, + code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError(err, std::move(response), - fmt("unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code)); + "unable to %s '%s': %s (%d)", + request.verb(), request.uri, curl_easy_strerror(code), code); /* If this is a transient error, then maybe retry the download after a while. If we're writing to a @@ -704,7 +703,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri); + throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri); res.data = std::move(*s3Res.data); callback(std::move(res)); #else diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 1ad96bc10..40e7cf52c 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -31,7 +31,7 @@ struct FileTransferSettings : Config R"( The timeout (in seconds) for establishing connections in the binary cache substituter. It corresponds to `curl`’s - `--connect-timeout` option. + `--connect-timeout` option. A value of 0 means no limit. )"}; Setting<unsigned long> stalledDownloadTimeout{ diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d77fff963..5cc5c91cc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -81,7 +81,7 @@ int getSchema(Path schemaPath) void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) { - const int nixCASchemaVersion = 3; + const int nixCASchemaVersion = 4; int curCASchema = getSchema(schemaPath); if (curCASchema != nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) { @@ -143,6 +143,21 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) )"); txn.commit(); } + if (curCASchema < 4) { + SQLiteTxn txn(db); + db.exec(R"( + create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths + begin + delete from RealisationsRefs where realisationReference in ( + select id from Realisations where outputPath = old.id + ); + end; + -- used by deletion trigger + create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); + )"); + txn.commit(); + } + writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); lockFile(lockFd.get(), ltRead, true); } @@ -482,18 +497,18 @@ void LocalStore::openDB(State & state, bool create) SQLiteStmt stmt; stmt.create(db, "pragma main.journal_mode;"); if (sqlite3_step(stmt) != SQLITE_ROW) - throwSQLiteError(db, "querying journal mode"); + SQLiteError::throw_(db, "querying journal mode"); prevMode = std::string((const char *) sqlite3_column_text(stmt, 0)); } if (prevMode != mode && sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting journal mode"); + SQLiteError::throw_(db, "setting journal mode"); /* Increase the auto-checkpoint interval to 40000 pages. This seems enough to ensure that instantiating the NixOS system derivation is done in a single fsync(). */ if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "setting autocheckpoint interval"); + SQLiteError::throw_(db, "setting autocheckpoint interval"); /* Initialise the database schema, if necessary. */ if (create) { diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 1d82b4ab1..2090beabd 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -8,22 +8,32 @@ namespace nix { -[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) +SQLiteError::SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf) + : Error(""), path(path), errNo(errNo), extendedErrNo(extendedErrNo) +{ + err.msg = hintfmt("%s: %s (in '%s')", + normaltxt(hf.str()), + sqlite3_errstr(extendedErrNo), + path ? path : "(in-memory)"); +} + +[[noreturn]] void SQLiteError::throw_(sqlite3 * db, hintformat && hf) { int err = sqlite3_errcode(db); int exterr = sqlite3_extended_errcode(db); auto path = sqlite3_db_filename(db, nullptr); - if (!path) path = "(in-memory)"; if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { - throw SQLiteBusy( + auto exp = SQLiteBusy(path, err, exterr, std::move(hf)); + exp.err.msg = hintfmt( err == SQLITE_PROTOCOL - ? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path) - : fmt("SQLite database '%s' is busy", path)); - } - else - throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); + ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" + : "SQLite database '%s' is busy", + path ? path : "(in-memory)"); + throw exp; + } else + throw SQLiteError(path, err, exterr, std::move(hf)); } SQLite::SQLite(const Path & path, bool create) @@ -37,7 +47,7 @@ SQLite::SQLite(const Path & path, bool create) throw Error("cannot open SQLite database '%s'", path); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throwSQLiteError(db, "setting timeout"); + SQLiteError::throw_(db, "setting timeout"); exec("pragma foreign_keys = 1"); } @@ -46,7 +56,7 @@ SQLite::~SQLite() { try { if (db && sqlite3_close(db) != SQLITE_OK) - throwSQLiteError(db, "closing database"); + SQLiteError::throw_(db, "closing database"); } catch (...) { ignoreException(); } @@ -62,7 +72,7 @@ void SQLite::exec(const std::string & stmt) { retrySQLite<void>([&]() { if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt); + SQLiteError::throw_(db, "executing SQLite statement '%s'", stmt); }); } @@ -76,7 +86,7 @@ void SQLiteStmt::create(sqlite3 * db, const std::string & sql) checkInterrupt(); assert(!stmt); if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK) - throwSQLiteError(db, fmt("creating statement '%s'", sql)); + SQLiteError::throw_(db, "creating statement '%s'", sql); this->db = db; this->sql = sql; } @@ -85,7 +95,7 @@ SQLiteStmt::~SQLiteStmt() { try { if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throwSQLiteError(db, fmt("finalizing statement '%s'", sql)); + SQLiteError::throw_(db, "finalizing statement '%s'", sql); } catch (...) { ignoreException(); } @@ -109,7 +119,7 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (std::string_view value, bool not { if (notNull) { if (sqlite3_bind_text(stmt, curArg++, value.data(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); return *this; @@ -119,7 +129,7 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (const unsigned char * data, size { if (notNull) { if (sqlite3_bind_blob(stmt, curArg++, data, len, SQLITE_TRANSIENT) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); return *this; @@ -129,7 +139,7 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) { if (notNull) { if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); } else bind(); return *this; @@ -138,7 +148,7 @@ SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) SQLiteStmt::Use & SQLiteStmt::Use::bind() { if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throwSQLiteError(stmt.db, "binding argument"); + SQLiteError::throw_(stmt.db, "binding argument"); return *this; } @@ -152,14 +162,14 @@ void SQLiteStmt::Use::exec() int r = step(); assert(r != SQLITE_ROW); if (r != SQLITE_DONE) - throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt))); + SQLiteError::throw_(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt))); } bool SQLiteStmt::Use::next() { int r = step(); if (r != SQLITE_DONE && r != SQLITE_ROW) - throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt))); + SQLiteError::throw_(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt))); return r == SQLITE_ROW; } @@ -185,14 +195,14 @@ SQLiteTxn::SQLiteTxn(sqlite3 * db) { this->db = db; if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "starting transaction"); + SQLiteError::throw_(db, "starting transaction"); active = true; } void SQLiteTxn::commit() { if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "committing transaction"); + SQLiteError::throw_(db, "committing transaction"); active = false; } @@ -200,7 +210,7 @@ SQLiteTxn::~SQLiteTxn() { try { if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throwSQLiteError(db, "aborting transaction"); + SQLiteError::throw_(db, "aborting transaction"); } catch (...) { ignoreException(); } diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 99f0d56ce..1d1c553ea 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -96,10 +96,30 @@ struct SQLiteTxn }; -MakeError(SQLiteError, Error); -MakeError(SQLiteBusy, SQLiteError); +struct SQLiteError : Error +{ + const char *path; + int errNo, extendedErrNo; + + template<typename... Args> + [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args) { + throw_(db, hintfmt(fs, args...)); + } + + SQLiteError(const char *path, int errNo, int extendedErrNo, hintformat && hf); + +protected: -[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs); + template<typename... Args> + SQLiteError(const char *path, int errNo, int extendedErrNo, const std::string & fs, const Args & ... args) + : SQLiteError(path, errNo, extendedErrNo, hintfmt(fs, args...)) + { } + + [[noreturn]] static void throw_(sqlite3 * db, hintformat && hf); + +}; + +MakeError(SQLiteBusy, SQLiteError); void handleSQLiteBusy(const SQLiteBusy & e); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 69aa0d094..4b8c55686 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -127,11 +127,11 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) if (flag.handler.arity == ArityAny) break; throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); } - if (flag.completer) - if (auto prefix = needsCompletion(*pos)) { - anyCompleted = true; + if (auto prefix = needsCompletion(*pos)) { + anyCompleted = true; + if (flag.completer) flag.completer(n, *prefix); - } + } args.push_back(*pos++); } if (!anyCompleted) @@ -146,6 +146,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) && hasPrefix(name, std::string(*prefix, 2))) completions->add("--" + name, flag->description); } + return false; } auto i = longFlags.find(std::string(*pos, 2)); if (i == longFlags.end()) return false; @@ -187,10 +188,12 @@ bool Args::processArgs(const Strings & args, bool finish) { std::vector<std::string> ss; for (const auto &[n, s] : enumerate(args)) { - ss.push_back(s); - if (exp.completer) - if (auto prefix = needsCompletion(s)) + if (auto prefix = needsCompletion(s)) { + ss.push_back(*prefix); + if (exp.completer) exp.completer(n, *prefix); + } else + ss.push_back(s); } exp.handler.fun(ss); expectedArgs.pop_front(); @@ -279,21 +282,22 @@ static void _completePath(std::string_view prefix, bool onlyDirs) { completionType = ctFilenames; glob_t globbuf; - int flags = GLOB_NOESCAPE | GLOB_TILDE; + int flags = GLOB_NOESCAPE; #ifdef GLOB_ONLYDIR if (onlyDirs) flags |= GLOB_ONLYDIR; #endif - if (glob((std::string(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) { + // using expandTilde here instead of GLOB_TILDE(_CHECK) so that ~<Tab> expands to /home/user/ + if (glob((expandTilde(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) { for (size_t i = 0; i < globbuf.gl_pathc; ++i) { if (onlyDirs) { - auto st = lstat(globbuf.gl_pathv[i]); + auto st = stat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } completions->add(globbuf.gl_pathv[i]); } - globfree(&globbuf); } + globfree(&globbuf); } void completePath(size_t, std::string_view prefix) @@ -322,11 +326,6 @@ MultiCommand::MultiCommand(const Commands & commands_) .optional = true, .handler = {[=](std::string s) { assert(!command); - if (auto prefix = needsCompletion(s)) { - for (auto & [name, command] : commands) - if (hasPrefix(name, *prefix)) - completions->add(name); - } auto i = commands.find(s); if (i == commands.end()) { std::set<std::string> commandNames; @@ -337,6 +336,11 @@ MultiCommand::MultiCommand(const Commands & commands_) } command = {s, i->second()}; command->second->parent = this; + }}, + .completer = {[&](size_t, std::string_view prefix) { + for (auto & [name, command] : commands) + if (hasPrefix(name, prefix)) + completions->add(name); }} }); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e033a4116..df37edf57 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -58,4 +58,18 @@ std::ostream & operator <<(std::ostream & str, const ExperimentalFeature & featu return str << showExperimentalFeature(feature); } +void to_json(nlohmann::json& j, const ExperimentalFeature& feature) { + j = showExperimentalFeature(feature); +} + +void from_json(const nlohmann::json& j, ExperimentalFeature& feature) { + const std::string input = j; + const auto parsed = parseExperimentalFeature(input); + + if (parsed.has_value()) + feature = *parsed; + else + throw Error("Unknown experimental feature '%s' in JSON input", input); +} + } diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 266e41a22..a6d080094 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -51,4 +51,11 @@ public: MissingExperimentalFeature(ExperimentalFeature); }; +/** + * Semi-magic conversion to and from json. + * See the nlohmann/json readme for more details. + */ +void to_json(nlohmann::json&, const ExperimentalFeature&); +void from_json(const nlohmann::json&, ExperimentalFeature&); + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index c075a14b4..b49c1e466 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -198,6 +198,17 @@ std::string_view baseNameOf(std::string_view path) } +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + + bool isInDir(std::string_view path, std::string_view dir) { return path.substr(0, 1) == "/" @@ -213,6 +224,15 @@ bool isDirOrInDir(std::string_view path, std::string_view dir) } +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + struct stat lstat(const Path & path) { struct stat st; @@ -1062,7 +1082,7 @@ std::string runProgram(Path program, bool searchPath, const Strings & args, auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input}); if (!statusOk(res.first)) - throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); return res.second; } @@ -1190,7 +1210,7 @@ void runProgram2(const RunOptions & options) if (source) promise.get_future().get(); if (status) - throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status))); + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 20591952d..a1d0e0e6b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -68,6 +68,9 @@ Path dirOf(const PathView path); following the final `/' (trailing slashes are removed). */ std::string_view baseNameOf(std::string_view path); +/* Perform tilde expansion on a path. */ +std::string expandTilde(std::string_view path); + /* Check whether 'path' is a descendant of 'dir'. Both paths must be canonicalized. */ bool isInDir(std::string_view path, std::string_view dir); @@ -77,6 +80,7 @@ bool isInDir(std::string_view path, std::string_view dir); bool isDirOrInDir(std::string_view path, std::string_view dir); /* Get status of `path'. */ +struct stat stat(const Path & path); struct stat lstat(const Path & path); /* Return true iff the given path exists. */ diff --git a/src/nix/app.cc b/src/nix/app.cc index 803d028f0..df7303e15 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -35,7 +35,7 @@ struct InstallableDerivedPath : Installable /** * Return the rewrites that are needed to resolve a string whose context is - * included in `dependencies` + * included in `dependencies`. */ StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) { @@ -51,7 +51,7 @@ StringPairs resolveRewrites(Store & store, const BuiltPaths dependencies) } /** - * Resolve the given string assuming the given context + * Resolve the given string assuming the given context. */ std::string resolveString(Store & store, const std::string & toResolve, const BuiltPaths dependencies) { @@ -61,14 +61,18 @@ std::string resolveString(Store & store, const std::string & toResolve, const Bu UnresolvedApp Installable::toApp(EvalState & state) { - auto [cursor, attrPath] = getCursor(state); + auto cursor = getCursor(state); + auto attrPath = cursor->getAttrPath(); auto type = cursor->getAttr("type")->getString(); + std::string expected = !attrPath.empty() && attrPath[0] == "apps" ? "app" : "derivation"; + if (type != expected) + throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected); + if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - std::vector<StorePathWithOutputs> context2; for (auto & [path, name] : context) context2.push_back({path, {name}}); @@ -101,7 +105,7 @@ UnresolvedApp Installable::toApp(EvalState & state) } else - throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); + throw Error("attribute '%s' has unsupported type '%s'", cursor->getAttrPathStr(), type); } // FIXME: move to libcmd diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 8cd04d5fe..733b93661 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -16,7 +16,7 @@ struct CmdEval : MixJSON, InstallableCommand std::optional<std::string> apply; std::optional<Path> writeTo; - CmdEval() + CmdEval() : InstallableCommand(true /* supportReadOnlyMode */) { addFlag({ .longName = "raw", diff --git a/src/nix/flake.cc b/src/nix/flake.cc index a876bb3af..04b23ed0f 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -528,6 +528,16 @@ struct CmdFlakeCheck : FlakeCommand } } + else if (name == "formatter") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkApp( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } + else if (name == "packages" || name == "devShells") { state->forceAttrs(vOutput, pos); for (auto & attr : *vOutput.attrs) { @@ -705,7 +715,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand defaultTemplateAttrPathsPrefixes, lockFlags); - auto [cursor, attrPath] = installable.getCursor(*evalState); + auto cursor = installable.getCursor(*evalState); auto templateDirAttr = cursor->getAttr("path"); auto templateDir = templateDirAttr->getString(); @@ -1011,6 +1021,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 1 && ( attrPath[0] == "defaultPackage" || attrPath[0] == "devShell" + || attrPath[0] == "formatter" || attrPath[0] == "nixosConfigurations" || attrPath[0] == "nixosModules" || attrPath[0] == "defaultApp" @@ -1027,7 +1038,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON } else if ( - (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell")) + (attrPath.size() == 2 && (attrPath[0] == "defaultPackage" || attrPath[0] == "devShell" || attrPath[0] == "formatter")) || (attrPath.size() == 3 && (attrPath[0] == "checks" || attrPath[0] == "packages" || attrPath[0] == "devShells")) ) { diff --git a/src/nix/flake.md b/src/nix/flake.md index d59915eeb..7d179a6c4 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -177,8 +177,8 @@ Currently the `type` attribute can be one of the following: attribute `url`. In URL form, the schema must be `http://`, `https://` or `file://` - URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz`, - `.tar.bz2` or `.tar.zst`. + URLs and the extension must be `.zip`, `.tar`, `.tgz`, `.tar.gz`, + `.tar.xz`, `.tar.bz2` or `.tar.zst`. * `github`: A more efficient way to fetch repositories from GitHub. The following attributes are required: diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc new file mode 100644 index 000000000..e5d44bd38 --- /dev/null +++ b/src/nix/fmt.cc @@ -0,0 +1,53 @@ +#include "command.hh" +#include "run.hh" + +using namespace nix; + +struct CmdFmt : SourceExprCommand { + std::vector<std::string> args; + + CmdFmt() { expectArgs({.label = "args", .handler = {&args}}); } + + std::string description() override { + return "reformat your code in the standard style"; + } + + std::string doc() override { + return + #include "fmt.md" + ; + } + + Category category() override { return catSecondary; } + + Strings getDefaultFlakeAttrPaths() override { + return Strings{"formatter." + settings.thisSystem.get()}; + } + + Strings getDefaultFlakeAttrPathPrefixes() override { return Strings{}; } + + void run(ref<Store> store) { + auto evalState = getEvalState(); + auto evalStore = getEvalStore(); + + auto installable = parseInstallable(store, "."); + auto app = installable->toApp(*evalState).resolve(evalStore, store); + + Strings programArgs{app.program}; + + // Propagate arguments from the CLI + if (args.empty()) { + // Format the current flake out of the box + programArgs.push_back("."); + } else { + // User wants more power, let them decide which paths to include/exclude + for (auto &i : args) { + programArgs.push_back(i); + } + } + + runProgramInStore(store, app.program, programArgs); + }; +}; + +static auto r2 = registerCommand<CmdFmt>("fmt"); diff --git a/src/nix/fmt.md b/src/nix/fmt.md new file mode 100644 index 000000000..1c78bb36f --- /dev/null +++ b/src/nix/fmt.md @@ -0,0 +1,53 @@ +R""( + +# Examples + +With [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt): + +```nix +# flake.nix +{ + outputs = { nixpkgs, self }: { + formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixpkgs-fmt; + }; +} +``` + +- Format the current flake: `$ nix fmt` + +- Format a specific folder or file: `$ nix fmt ./folder ./file.nix` + +With [nixfmt](https://github.com/serokell/nixfmt): + +```nix +# flake.nix +{ + outputs = { nixpkgs, self }: { + formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.nixfmt; + }; +} +``` + +- Format specific files: `$ nix fmt ./file1.nix ./file2.nix` + +With [Alejandra](https://github.com/kamadorueda/alejandra): + +```nix +# flake.nix +{ + outputs = { nixpkgs, self }: { + formatter.x86_64-linux = nixpkgs.legacyPackages.x86_64-linux.alejandra; + }; +} +``` + +- Format the current flake: `$ nix fmt` + +- Format a specific folder or file: `$ nix fmt ./folder ./file.nix` + +# Description + +`nix fmt` will rewrite all Nix files (\*.nix) to a canonical format +using the formatter specified in your flake. + +)"" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 1f9d4fb4e..0eb037858 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -33,6 +33,7 @@ extern "C" { #include "command.hh" #include "finally.hh" #include "markdown.hh" +#include "local-fs-store.hh" #if HAVE_BOEHMGC #define GC_INCLUDE_NEW @@ -119,7 +120,7 @@ std::string runNix(Path program, const Strings & args, }); if (!statusOk(res.first)) - throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); return res.second; } @@ -419,7 +420,8 @@ bool NixRepl::processLine(std::string line) << " <expr> Evaluate and print expression\n" << " <x> = <expr> Bind expression to variable\n" << " :a <expr> Add attributes from resulting set to scope\n" - << " :b <expr> Build derivation\n" + << " :b <expr> Build a derivation\n" + << " :bl <expr> Build a derivation, creating GC roots in the working directory\n" << " :e <expr> Open package or function in $EDITOR\n" << " :i <expr> Build derivation, then install result into current profile\n" << " :l <path> Load Nix expression and add it to scope\n" @@ -502,18 +504,26 @@ bool NixRepl::processLine(std::string line) runNix("nix-shell", {state->store->printStorePath(drvPath)}); } - else if (command == ":b" || command == ":i" || command == ":s" || command == ":log") { + else if (command == ":b" || command == ":bl" || command == ":i" || command == ":s" || command == ":log") { Value v; evalString(arg, v); StorePath drvPath = getDerivationPath(v); Path drvPathRaw = state->store->printStorePath(drvPath); - if (command == ":b") { + if (command == ":b" || command == ":bl") { state->store->buildPaths({DerivedPath::Built{drvPath}}); auto drv = state->store->readDerivation(drvPath); logger->cout("\nThis derivation produced the following outputs:"); - for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) - logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath)); + for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) { + auto localStore = state->store.dynamic_pointer_cast<LocalFSStore>(); + if (localStore && command == ":bl") { + std::string symlink = "repl-result-" + outputName; + localStore->addPermRoot(outputPath, absPath(symlink)); + logger->cout(" ./%s -> %s", symlink, state->store->printStorePath(outputPath)); + } else { + logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath)); + } + } } else if (command == ":i") { runNix("nix-env", {"-i", drvPathRaw}); } else if (command == ":log") { diff --git a/src/nix/search.cc b/src/nix/search.cc index e9307342c..e96a85ea2 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -165,8 +165,8 @@ struct CmdSearch : InstallableCommand, MixJSON } }; - for (auto & [cursor, prefix] : installable->getCursors(*state)) - visit(*cursor, parseAttrPath(*state, prefix), true); + for (auto & cursor : installable->getCursors(*state)) + visit(*cursor, cursor->getAttrPath(), true); if (!json && !results) throw Error("no results for the given search term(s)!"); diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/build-remote-content-addressed-floating.sh index 1f474dde0..e83b42b41 100644 --- a/tests/build-remote-content-addressed-floating.sh +++ b/tests/build-remote-content-addressed-floating.sh @@ -2,7 +2,7 @@ source common.sh file=build-hook-ca-floating.nix -enableFeatures "ca-derivations ca-references" +enableFeatures "ca-derivations" CONTENT_ADDRESSED=true diff --git a/tests/ca/common.sh b/tests/ca/common.sh index b9d415863..b104b5a78 100644 --- a/tests/ca/common.sh +++ b/tests/ca/common.sh @@ -1,5 +1,5 @@ source ../common.sh -enableFeatures "ca-derivations ca-references" +enableFeatures "ca-derivations" restartDaemon diff --git a/tests/ca/selfref-gc.sh b/tests/ca/selfref-gc.sh new file mode 100755 index 000000000..248778894 --- /dev/null +++ b/tests/ca/selfref-gc.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +requireDaemonNewerThan "2.4pre20210626" + +enableFeatures "ca-derivations nix-command flakes" + +export NIX_TESTS_CA_BY_DEFAULT=1 +cd .. +source ./selfref-gc.sh diff --git a/tests/flakes-run.sh b/tests/flakes-run.sh new file mode 100644 index 000000000..88fc3e628 --- /dev/null +++ b/tests/flakes-run.sh @@ -0,0 +1,29 @@ +source common.sh + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local +cp ./shell-hello.nix ./config.nix $TEST_HOME +cd $TEST_HOME + +cat <<EOF > flake.nix +{ + outputs = {self}: { + packages.$system.pkgAsPkg = (import ./shell-hello.nix).hello; + packages.$system.appAsApp = self.packages.$system.appAsApp; + + apps.$system.pkgAsApp = self.packages.$system.pkgAsPkg; + apps.$system.appAsApp = { + type = "app"; + program = "\${(import ./shell-hello.nix).hello}/bin/hello"; + }; + }; +} +EOF +nix run --no-write-lock-file .#appAsApp +nix run --no-write-lock-file .#pkgAsPkg + +! nix run --no-write-lock-file .#pkgAsApp || fail "'nix run' shouldn’t accept an 'app' defined under 'packages'" +! nix run --no-write-lock-file .#appAsPkg || fail "elements of 'apps' should be of type 'app'" + +clearStore + diff --git a/tests/fmt.sh b/tests/fmt.sh new file mode 100644 index 000000000..bc05118ff --- /dev/null +++ b/tests/fmt.sh @@ -0,0 +1,30 @@ +source common.sh + +set -o pipefail + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local + +cp ./simple.nix ./simple.builder.sh ./fmt.simple.sh ./config.nix $TEST_HOME + +cd $TEST_HOME + +nix fmt --help | grep "Format" + +cat << EOF > flake.nix +{ + outputs = _: { + formatter.$system = + with import ./config.nix; + mkDerivation { + name = "formatter"; + buildCommand = "mkdir -p \$out/bin; cp \${./fmt.simple.sh} \$out/bin/formatter"; + }; + }; +} +EOF +nix fmt ./file ./folder | grep 'Formatting: ./file ./folder' +nix flake check +nix flake show | grep -P "package 'formatter'" + +clearStore diff --git a/tests/fmt.simple.sh b/tests/fmt.simple.sh new file mode 100755 index 000000000..4c8c67ebb --- /dev/null +++ b/tests/fmt.simple.sh @@ -0,0 +1 @@ +echo Formatting: "${@}" diff --git a/tests/local.mk b/tests/local.mk index 668b34500..e3c4ff4eb 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,5 +1,6 @@ nix_tests = \ flakes.sh \ + flakes-run.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ @@ -79,6 +80,7 @@ nix_tests = \ post-hook.sh \ function-trace.sh \ flake-local-settings.sh \ + fmt.sh \ eval-store.sh \ why-depends.sh \ import-derivation.sh \ @@ -90,6 +92,7 @@ nix_tests = \ plugins.sh \ build.sh \ ca/nix-run.sh \ + selfref-gc.sh ca/selfref-gc.sh \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index a7a4d4fa2..fad62b993 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -3,7 +3,7 @@ source common.sh clearStore clearProfiles -enableFeatures "ca-derivations ca-references" +enableFeatures "ca-derivations" restartDaemon # Make a flake. diff --git a/tests/repl.sh b/tests/repl.sh index 6505f1741..b6937b9e9 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -1,29 +1,37 @@ source common.sh +testDir="$PWD" +cd "$TEST_ROOT" + replCmds=" simple = 1 -simple = import ./simple.nix -:b simple +simple = import $testDir/simple.nix +:bl simple :log simple " replFailingCmds=" -failing = import ./simple-failing.nix +failing = import $testDir/simple-failing.nix :b failing :log failing " replUndefinedVariable=" -import ./undefined-variable.nix +import $testDir/undefined-variable.nix " testRepl () { local nixArgs=("$@") + rm -rf repl-result-out || true # cleanup from other runs backed by a foreign nix store local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replCmds")" echo "$replOutput" local outPath=$(echo "$replOutput" |& grep -o -E "$NIX_STORE_DIR/\w*-simple") nix path-info "${nixArgs[@]}" "$outPath" + [ "$(realpath ./repl-result-out)" == "$outPath" ] || fail "nix repl :bl doesn't make a symlink" + # run it again without checking the output to ensure the previously created symlink gets overwritten + nix repl "${nixArgs[@]}" <<< "$replCmds" || fail "nix repl does not work twice with the same inputs" + # simple.nix prints a PATH during build echo "$replOutput" | grep -qs 'PATH=' || fail "nix repl :log doesn't output logs" local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replFailingCmds" 2>&1)" diff --git a/tests/selfref-gc.sh b/tests/selfref-gc.sh new file mode 100644 index 000000000..3f1f50eea --- /dev/null +++ b/tests/selfref-gc.sh @@ -0,0 +1,30 @@ +source common.sh + +requireDaemonNewerThan "2.6.0pre20211215" + +clearStore + +nix-build --no-out-link -E ' + with import ./config.nix; + + let d1 = mkDerivation { + name = "selfref-gc"; + outputs = [ "out" ]; + buildCommand = " + echo SELF_REF: $out > $out + "; + }; in + + # the only change from d1 is d1 as an (unused) build input + # to get identical store path in CA. + mkDerivation { + name = "selfref-gc"; + outputs = [ "out" ]; + buildCommand = " + echo UNUSED: ${d1} + echo SELF_REF: $out > $out + "; + } +' + +nix-collect-garbage |