diff options
38 files changed, 474 insertions, 221 deletions
diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 942d7a971..be3f6af97 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -11,6 +11,10 @@ assignees: '' <!-- describe your problem --> +## Proposal + +<!-- propose a solution --> + ## Checklist <!-- make sure this issue is not redundant or obsolete --> @@ -22,10 +26,6 @@ assignees: '' [source]: https://github.com/NixOS/nix/tree/master/doc/manual/src [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation -## Proposal - -<!-- propose a solution --> - ## Priorities Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index a210583ae..025f758e7 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -4,7 +4,7 @@ # Synopsis -`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--rollback` [*generation*] } +`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--list-generations` | `--rollback` [*generation*] } # Description @@ -39,6 +39,15 @@ This command has the following operations: for `nix-env` operations (by symlinking them from the directory `~/.nix-defexpr`). + - `--list-generations`\ + Prints a list of all the current existing generations for the + channel profile. + + Works the same way as + ``` + nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations + ``` + - `--rollback` \[*generation*\]\ Reverts the previous call to `nix-channel --update`. Optionally, you can specify a specific channel generation diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 51db5fc67..a679ceaf7 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -1,6 +1,6 @@ # Name -`nix-collect-garbage` - delete unreachable store paths +`nix-collect-garbage` - delete unreachable [store objects] # Synopsis @@ -8,17 +8,57 @@ # Description -The command `nix-collect-garbage` is mostly an alias of [`nix-store ---gc`](@docroot@/command-ref/nix-store/gc.md), that is, it deletes all -unreachable paths in the Nix store to clean up your system. However, -it provides two additional options: `-d` (`--delete-old`), which -deletes all old generations of all profiles in `/nix/var/nix/profiles` -by invoking `nix-env --delete-generations old` on all profiles (of -course, this makes rollbacks to previous configurations impossible); -and `--delete-older-than` *period*, where period is a value such as -`30d`, which deletes all generations older than the specified number -of days in all profiles in `/nix/var/nix/profiles` (except for the -generations that were active at that point in time). +The command `nix-collect-garbage` is mostly an alias of [`nix-store --gc`](@docroot@/command-ref/nix-store/gc.md). +That is, it deletes all unreachable [store objects] in the Nix store to clean up your system. + +However, it provides two additional options, +[`--delete-old`](#opt-delete-old) and [`--delete-older-than`](#opt-delete-older-than), +which also delete old [profiles], allowing potentially more [store objects] to be deleted because profiles are also garbage collection roots. +These options are the equivalent of running +[`nix-env --delete-generations`](@docroot@/command-ref/nix-env/delete-generations.md) +with various augments on multiple profiles, +prior to running `nix-collect-garbage` (or just `nix-store --gc`) without any flags. + +> **Note** +> +> Deleting previous configurations makes rollbacks to them impossible. + +These flags should be used with care, because they potentially delete generations of profiles used by other users on the system. + +## Locations searched for profiles + +`nix-collect-garbage` cannot know about all profiles; that information doesn't exist. +Instead, it looks in a few locations, and acts on all profiles it finds there: + +1. The default profile locations as specified in the [profiles] section of the manual. + +2. > **NOTE** + > + > Not stable; subject to change + > + > Do not rely on this functionality; it just exists for migration purposes and is may change in the future. + > These deprecated paths remain a private implementation detail of Nix. + + `$NIX_STATE_DIR/profiles` and `$NIX_STATE_DIR/profiles/per-user`. + + With the exception of `$NIX_STATE_DIR/profiles/per-user/root` and `$NIX_STATE_DIR/profiles/default`, these directories are no longer used by other commands. + `nix-collect-garbage` looks there anyways in order to clean up profiles from older versions of Nix. + +# Options + +These options are for deleting old [profiles] prior to deleting unreachable [store objects]. + +- <span id="opt-delete-old">[`--delete-old`](#opt-delete-old)</span> / `-d`\ + Delete all old generations of profiles. + + This is the equivalent of invoking `nix-env --delete-generations old` on each found profile. + +- <span id="opt-delete-older-than">[`--delete-older-than`](#opt-delete-older-than)</span> *period*\ + Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). + *period* is a value such as `30d`, which would mean 30 days. + + This is the equivalent of invoking [`nix-env --delete-generations <period>`](@docroot@/command-ref/nix-env/delete-generations.md#generations-days) on each found profile. + See the documentation of that command for additional information about the *period* argument. {{#include ./opt-common.md}} @@ -32,3 +72,6 @@ generations of each profile, do ```console $ nix-collect-garbage -d ``` + +[profiles]: @docroot@/command-ref/files/profiles.md +[store objects]: @docroot@/glossary.md#gloss-store-object diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md index 92cb7f0d9..d828a5b9e 100644 --- a/doc/manual/src/command-ref/nix-env/delete-generations.md +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -9,14 +9,39 @@ # Description This operation deletes the specified generations of the current profile. -The generations can be a list of generation numbers, the special value -`old` to delete all non-current generations, a value such as `30d` to -delete all generations older than the specified number of days (except -for the generation that was active at that point in time), or a value -such as `+5` to keep the last `5` generations ignoring any newer than -current, e.g., if `30` is the current generation `+5` will delete -generation `25` and all older generations. Periodically deleting old -generations is important to make garbage collection effective. + +*generations* can be a one of the following: + +- <span id="generations-list">`<number>...`</span>:\ + A list of generation numbers, each one a separate command-line argument. + + Delete exactly the profile generations given by their generation number. + Deleting the current generation is not allowed. + +- The special value <span id="generations-old">`old`</span> + + Delete all generations older than the current one. + +- <span id="generations-days">`<days>d`</span>:\ + The last *days* days + + *Example*: `30d` + + Delete all generations older than *days* days. + The generation that was active at that point in time is excluded, and will not be deleted. + +- <span id="generations-count">`+<count>`</span>:\ + The last *count* generations up to the present + + *Example*: `+5` + + Keep the last *count* generations, along with any newer than current. + +Periodically deleting old generations is important to make garbage collection +effective. +The is because profiles are also garbage collection roots — any [store object] reachable from a profile is "alive" and ineligible for deletion. + +[store object]: @docroot@/glossary.md#gloss-store-object {{#include ./opt-common.md}} @@ -28,19 +53,35 @@ generations is important to make garbage collection effective. # Examples +## Delete explicit generation numbers + ```console $ nix-env --delete-generations 3 4 8 ``` +Delete the generations numbered 3, 4, and 8, so long as the current active generation is not any of those. + +## Keep most-recent by count count + ```console $ nix-env --delete-generations +5 ``` +Suppose `30` is the current generation, and we currently have generations numbered `20` through `32`. + +Then this command will delete generations `20` through `25` (`<= 30 - 5`), +and keep generations `26` through `31` (`> 30 - 5`). + +## Keep most-recent in days + ```console $ nix-env --delete-generations 30d ``` +This command will delete all generations older than 30 days, except for the generation that was active 30 days ago (if it currently exists). + +## Delete all older + ```console $ nix-env --profile other_profile --delete-generations old ``` - diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index e142bd415..47a484826 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -112,9 +112,10 @@ from some server. - [substituter]{#gloss-substituter}\ - A *substituter* is an additional store from which Nix will - copy store objects it doesn't have. For details, see the - [`substituters` option](./command-ref/conf-file.md#conf-substituters). + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. + + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. [substituter]: #gloss-substituter diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 5a708f11b..d4babf1ea 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -10,7 +10,7 @@ - Bash Shell. The `./configure` script relies on bashisms, so Bash is required. - - A version of GCC or Clang that supports C++17. + - A version of GCC or Clang that supports C++20. - `pkg-config` to locate dependencies. If your distribution does not provide it, you can get it from diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 78ae99f4b..bde9057c6 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,2 +1,3 @@ # Release X.Y (202?-??-??) +- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 5111a5dde..0326d3415 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -100,7 +100,7 @@ poly_extra_try_me_commands() { poly_configure_nix_daemon_service() { task "Setting up the nix-daemon LaunchDaemon" _sudo "to set up the nix-daemon as a LaunchDaemon" \ - /bin/cp -f "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" + /usr/bin/install -m -rw-r--r-- "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" _sudo "to load the LaunchDaemon plist for nix-daemon" \ launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index a2b882355..10b077fb5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -701,7 +701,7 @@ RawInstallablesCommand::RawInstallablesCommand() { addFlag({ .longName = "stdin", - .description = "Read installables from the standard input.", + .description = "Read installables from the standard input. No default installable applied.", .handler = {&readFromStdIn, true} }); @@ -730,9 +730,9 @@ void RawInstallablesCommand::run(ref<Store> store) while (std::cin >> word) { rawInstallables.emplace_back(std::move(word)); } + } else { + applyDefaultInstallables(rawInstallables); } - - applyDefaultInstallables(rawInstallables); run(store, std::move(rawInstallables)); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index d6f4560a5..ea1f5975b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -741,7 +741,7 @@ struct EvalSettings : Config If set to `true`, the Nix evaluator will not allow access to any files outside of the Nix search path (as set via the `NIX_PATH` environment variable or the `-I` option), or to URIs outside of - `allowed-uri`. The default is `false`. + `allowed-uris`. The default is `false`. )"}; Setting<bool> pureEval{this, false, "pure-eval", diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 42efca4e7..5b2f7e8b7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -6,7 +6,7 @@ #include "globals.hh" #include "json-to-value.hh" #include "names.hh" -#include "references.hh" +#include "path-references.hh" #include "store-api.hh" #include "util.hh" #include "value-to-json.hh" @@ -4058,18 +4058,6 @@ static RegisterPrimOp primop_splitVersion({ RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; -RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) -{ - if (!primOps) primOps = new PrimOps; - primOps->push_back({ - .name = name, - .args = {}, - .arity = arity, - .fun = fun, - }); -} - - RegisterPrimOp::RegisterPrimOp(Info && info) { if (!primOps) primOps = new PrimOps; diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 4ae73fe1f..73b7b866c 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -28,11 +28,6 @@ struct RegisterPrimOp * will get called during EvalState initialization, so there * may be primops not yet added and builtins is not yet sorted. */ - RegisterPrimOp( - std::string name, - size_t arity, - PrimOpFun fun); - RegisterPrimOp(Info && info); }; diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 07bf400cf..8b3468009 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -12,7 +12,11 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, v.mkString(*s); } -static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); +static RegisterPrimOp primop_unsafeDiscardStringContext({ + .name = "__unsafeDiscardStringContext", + .arity = 1, + .fun = prim_unsafeDiscardStringContext +}); static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -22,7 +26,16 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, v.mkBool(!context.empty()); } -static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); +static RegisterPrimOp primop_hasContext({ + .name = "__hasContext", + .args = {"s"}, + .doc = R"( + Return `true` if string *s* has a non-empty context. The + context can be obtained with + [`getContext`](#builtins-getContext). + )", + .fun = prim_hasContext +}); /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a @@ -51,7 +64,11 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p v.mkString(*s, context2); } -static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); +static RegisterPrimOp primop_unsafeDiscardOutputDependency({ + .name = "__unsafeDiscardOutputDependency", + .arity = 1, + .fun = prim_unsafeDiscardOutputDependency +}); /* Extract the context of a string as a structured Nix value. @@ -119,7 +136,30 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, v.mkAttrs(attrs); } -static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); +static RegisterPrimOp primop_getContext({ + .name = "__getContext", + .args = {"s"}, + .doc = R"( + Return the string context of *s*. + + The string context tracks references to derivations within a string. + It is represented as an attribute set of [store derivation](@docroot@/glossary.md#gloss-store-derivation) paths mapping to output names. + + Using [string interpolation](@docroot@/language/string-interpolation.md) on a derivation will add that derivation to the string context. + For example, + + ```nix + builtins.getContext "${derivation { name = "a"; builder = "b"; system = "c"; }}" + ``` + + evaluates to + + ``` + { "/nix/store/arhvjaf6zmlyn8vh8fgn55rpwnxq0n7l-a.drv" = { outputs = [ "out" ]; }; } + ``` + )", + .fun = prim_getContext +}); /* Append the given context to a given string. @@ -192,6 +232,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar v.mkString(orig, context); } -static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); +static RegisterPrimOp primop_appendContext({ + .name = "__appendContext", + .arity = 2, + .fun = prim_appendContext +}); } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 2c0d98e74..322692b52 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -88,6 +88,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a state.allowPath(tree.storePath); } -static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); +static RegisterPrimOp r_fetchMercurial({ + .name = "fetchMercurial", + .arity = 1, + .fun = prim_fetchMercurial +}); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index fe880aaa8..be8159cc8 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -194,7 +194,11 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, } // FIXME: document -static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); +static RegisterPrimOp primop_fetchTree({ + .name = "fetchTree", + .arity = 1, + .fun = prim_fetchTree +}); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, const std::string & who, bool unpack, std::string name) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index e2a8b3c3a..2f4d4022e 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -90,6 +90,24 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V } } -static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML); +static RegisterPrimOp primop_fromTOML({ + .name = "fromTOML", + .args = {"e"}, + .doc = R"( + Convert a TOML string to a Nix value. For example, + + ```nix + builtins.fromTOML '' + x=1 + s="a" + [table] + y=2 + '' + ``` + + returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`. + )", + .fun = prim_fromTOML +}); } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 0b0bd3328..7f87cdf55 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -4,7 +4,7 @@ #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "references.hh" +#include "path-references.hh" #include "finally.hh" #include "util.hh" #include "archive.hh" @@ -2394,18 +2394,21 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() continue; auto references = *referencesOpt; - auto rewriteOutput = [&]() { + auto rewriteOutput = [&](const StringMap & rewrites) { /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { + if (!rewrites.empty()) { debug("rewriting hashes in '%1%'; cross fingers", actualPath); - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); + /* FIXME: Is this actually streaming? */ + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink(rewrites, nextSink); + dumpPath(actualPath, rsink); + rsink.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); deletePath(actualPath); - sink.s = rewriteStrings(sink.s, outputRewrites); - StringSource source(sink.s); - restorePath(actualPath, source); + movePath(tmpPath, actualPath); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ @@ -2454,7 +2457,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", actualPath); } - rewriteOutput(); + rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; HashModuloSink caSink { outputHash.hashType, oldHashPart }; @@ -2492,16 +2495,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() Hash::dummy, }; if (*scratchPath != newInfo0.path) { - // Also rewrite the output path - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - dumpPath(actualPath, rsink2); - rsink2.flush(); - }); - Path tmpPath = actualPath + ".tmp"; - restorePath(tmpPath, *source); - deletePath(actualPath); - movePath(tmpPath, actualPath); + // If the path has some self-references, we need to rewrite + // them. + // (note that this doesn't invalidate the ca hash we calculated + // above because it's computed *modulo the self-references*, so + // it already takes this rewrite into account). + rewriteOutput( + StringMap{{oldHashPart, + std::string(newInfo0.path.hashPart())}}); } HashResult narHashAndSize = hashPath(htSHA256, actualPath); @@ -2523,7 +2524,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() outputRewrites.insert_or_assign( std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); + rewriteOutput(outputRewrites); auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 940cb48f5..820898350 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -691,20 +691,19 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - to be used as substituters, separated by whitespace. - Substituters are tried based on their Priority value, which each substituter can set - independently. Lower value means higher priority. - The default is `https://cache.nixos.org`, with a Priority of 40. + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace. + A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. - At least one of the following conditions must be met for Nix to use - a substituter: + Substituters are tried based on their priority value, which each substituter can set independently. + Lower value means higher priority. + The default is `https://cache.nixos.org`, which has a priority of 40. + + At least one of the following conditions must be met for Nix to use a substituter: - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list - In addition, each store path should be trusted as described - in [`trusted-public-keys`](#conf-trusted-public-keys) + In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc new file mode 100644 index 000000000..33cf66ce3 --- /dev/null +++ b/src/libstore/path-references.cc @@ -0,0 +1,73 @@ +#include "path-references.hh" +#include "hash.hh" +#include "util.hh" +#include "archive.hh" + +#include <map> +#include <cstdlib> +#include <mutex> +#include <algorithm> + + +namespace nix { + + +PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap) + : RefScanSink(std::move(hashes)) + , backMap(std::move(backMap)) +{ } + +PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) +{ + StringSet hashes; + std::map<std::string, StorePath> backMap; + + for (auto & i : refs) { + std::string hashPart(i.hashPart()); + auto inserted = backMap.emplace(hashPart, i).second; + assert(inserted); + hashes.insert(hashPart); + } + + return PathRefScanSink(std::move(hashes), std::move(backMap)); +} + +StorePathSet PathRefScanSink::getResultPaths() +{ + /* Map the hashes found back to their store paths. */ + StorePathSet found; + for (auto & i : getResult()) { + auto j = backMap.find(i); + assert(j != backMap.end()); + found.insert(j->second); + } + + return found; +} + + +std::pair<StorePathSet, HashResult> scanForReferences( + const std::string & path, + const StorePathSet & refs) +{ + HashSink hashSink { htSHA256 }; + auto found = scanForReferences(hashSink, path, refs); + auto hash = hashSink.finish(); + return std::pair<StorePathSet, HashResult>(found, hash); +} + +StorePathSet scanForReferences( + Sink & toTee, + const Path & path, + const StorePathSet & refs) +{ + PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); + TeeSink sink { refsSink, toTee }; + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + return refsSink.getResultPaths(); +} + +} diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh new file mode 100644 index 000000000..7b44e3261 --- /dev/null +++ b/src/libstore/path-references.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "references.hh" +#include "path.hh" + +namespace nix { + +std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs); + +StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); + +class PathRefScanSink : public RefScanSink +{ + std::map<std::string, StorePath> backMap; + + PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap); + +public: + + static PathRefScanSink fromPaths(const StorePathSet & refs); + + StorePathSet getResultPaths(); +}; + +} diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 56be76ecc..11cc0c0e7 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -63,30 +63,19 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) return {std::move(fd), tmpl}; } -void createSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime) +void createSymlink(const Path & target, const Path & link) { if (symlink(target.c_str(), link.c_str())) throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } } -void replaceSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime) +void replaceSymlink(const Path & target, const Path & link) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp, mtime); + createSymlink(target, tmp); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; diff --git a/src/libstore/references.cc b/src/libutil/references.cc index 345f4528b..74003584a 100644 --- a/src/libstore/references.cc +++ b/src/libutil/references.cc @@ -6,6 +6,7 @@ #include <map> #include <cstdlib> #include <mutex> +#include <algorithm> namespace nix { @@ -66,69 +67,20 @@ void RefScanSink::operator () (std::string_view data) } -PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap) - : RefScanSink(std::move(hashes)) - , backMap(std::move(backMap)) -{ } - -PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) +RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) + : RewritingSink({{from, to}}, nextSink) { - StringSet hashes; - std::map<std::string, StorePath> backMap; - - for (auto & i : refs) { - std::string hashPart(i.hashPart()); - auto inserted = backMap.emplace(hashPart, i).second; - assert(inserted); - hashes.insert(hashPart); - } - - return PathRefScanSink(std::move(hashes), std::move(backMap)); } -StorePathSet PathRefScanSink::getResultPaths() +RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink) + : rewrites(rewrites), nextSink(nextSink) { - /* Map the hashes found back to their store paths. */ - StorePathSet found; - for (auto & i : getResult()) { - auto j = backMap.find(i); - assert(j != backMap.end()); - found.insert(j->second); + long unsigned int maxRewriteSize = 0; + for (auto & [from, to] : rewrites) { + assert(from.size() == to.size()); + maxRewriteSize = std::max(maxRewriteSize, from.size()); } - - return found; -} - - -std::pair<StorePathSet, HashResult> scanForReferences( - const std::string & path, - const StorePathSet & refs) -{ - HashSink hashSink { htSHA256 }; - auto found = scanForReferences(hashSink, path, refs); - auto hash = hashSink.finish(); - return std::pair<StorePathSet, HashResult>(found, hash); -} - -StorePathSet scanForReferences( - Sink & toTee, - const Path & path, - const StorePathSet & refs) -{ - PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); - TeeSink sink { refsSink, toTee }; - - /* Look for the hashes in the NAR dump of the path. */ - dumpPath(path, sink); - - return refsSink.getResultPaths(); -} - - -RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) - : from(from), to(to), nextSink(nextSink) -{ - assert(from.size() == to.size()); + this->maxRewriteSize = maxRewriteSize; } void RewritingSink::operator () (std::string_view data) @@ -136,13 +88,13 @@ void RewritingSink::operator () (std::string_view data) std::string s(prev); s.append(data); - size_t j = 0; - while ((j = s.find(from, j)) != std::string::npos) { - matches.push_back(pos + j); - s.replace(j, from.size(), to); - } + s = rewriteStrings(s, rewrites); - prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1); + prev = s.size() < maxRewriteSize + ? s + : maxRewriteSize == 0 + ? "" + : std::string(s, s.size() - maxRewriteSize + 1, maxRewriteSize - 1); auto consumed = s.size() - prev.size(); diff --git a/src/libstore/references.hh b/src/libutil/references.hh index 52d71b333..ffd730e7b 100644 --- a/src/libstore/references.hh +++ b/src/libutil/references.hh @@ -2,14 +2,9 @@ ///@file #include "hash.hh" -#include "path.hh" namespace nix { -std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs); - -StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); - class RefScanSink : public Sink { StringSet hashes; @@ -28,28 +23,18 @@ public: void operator () (std::string_view data) override; }; -class PathRefScanSink : public RefScanSink -{ - std::map<std::string, StorePath> backMap; - - PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap); - -public: - - static PathRefScanSink fromPaths(const StorePathSet & refs); - - StorePathSet getResultPaths(); -}; - struct RewritingSink : Sink { - std::string from, to, prev; + const StringMap rewrites; + long unsigned int maxRewriteSize; + std::string prev; Sink & nextSink; uint64_t pos = 0; std::vector<uint64_t> matches; RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + RewritingSink(const StringMap & rewrites, Sink & nextSink); void operator () (std::string_view data) override; diff --git a/src/libutil/tests/references.cc b/src/libutil/tests/references.cc new file mode 100644 index 000000000..a517d9aa1 --- /dev/null +++ b/src/libutil/tests/references.cc @@ -0,0 +1,46 @@ +#include "references.hh" +#include <gtest/gtest.h> + +namespace nix { + +using std::string; + +struct RewriteParams { + string originalString, finalString; + StringMap rewrites; + + friend std::ostream& operator<<(std::ostream& os, const RewriteParams& bar) { + StringSet strRewrites; + for (auto & [from, to] : bar.rewrites) + strRewrites.insert(from + "->" + to); + return os << + "OriginalString: " << bar.originalString << std::endl << + "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl << + "Expected result: " << bar.finalString; + } +}; + +class RewriteTest : public ::testing::TestWithParam<RewriteParams> { +}; + +TEST_P(RewriteTest, IdentityRewriteIsIdentity) { + RewriteParams param = GetParam(); + StringSink rewritten; + auto rewriter = RewritingSink(param.rewrites, rewritten); + rewriter(param.originalString); + rewriter.flush(); + ASSERT_EQ(rewritten.s, param.finalString); +} + +INSTANTIATE_TEST_CASE_P( + references, + RewriteTest, + ::testing::Values( + RewriteParams{ "foooo", "baroo", {{"foo", "bar"}, {"bar", "baz"}}}, + RewriteParams{ "foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}}, + RewriteParams{ "foooo", "foooo", {}} + ) +); + +} + diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 00fcb9b79..b302d6f45 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -256,14 +256,12 @@ inline Paths createDirs(PathView path) /** * Create a symlink. */ -void createSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime = {}); +void createSymlink(const Path & target, const Path & link); /** * Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime = {}); +void replaceSymlink(const Path & target, const Path & link); void renameFile(const Path & src, const Path & dst); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 740737ffe..c1c8edd1d 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -177,6 +177,7 @@ static int main_nix_channel(int argc, char ** argv) cRemove, cList, cUpdate, + cListGenerations, cRollback } cmd = cNone; std::vector<std::string> args; @@ -193,6 +194,8 @@ static int main_nix_channel(int argc, char ** argv) cmd = cList; } else if (*arg == "--update") { cmd = cUpdate; + } else if (*arg == "--list-generations") { + cmd = cListGenerations; } else if (*arg == "--rollback") { cmd = cRollback; } else { @@ -237,6 +240,11 @@ static int main_nix_channel(int argc, char ** argv) case cUpdate: update(StringSet(args.begin(), args.end())); break; + case cListGenerations: + if (!args.empty()) + throw UsageError("'--list-generations' expects no arguments"); + std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush; + break; case cRollback: if (args.size() > 1) throw UsageError("'--rollback' has at most one argument"); diff --git a/tests/build.sh b/tests/build.sh index 697aff0f9..8ae20f0df 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -129,3 +129,7 @@ nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status (.drvPath | match(".*multiple-outputs-e.drv")) and (.outputs | keys == ["a_a", "b"])) ' + +# Make sure that `--stdin` works and does not apply any defaults +printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []' +printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")' diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index a207f647c..21650eb06 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -5,6 +5,12 @@ enableFeatures "fetch-closure" clearStore clearCacheCache +# Old daemons don't properly zero out the self-references when +# calculating the CA hashes, so this breaks `nix store +# make-content-addressed` which expects the client and the daemon to +# compute the same hash +requireDaemonNewerThan "2.16.0pre20230524" + # Initialize binary cache. nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out) caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') diff --git a/tests/gc.sh b/tests/gc.sh index 95669e25c..ad09a8b39 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -50,31 +50,3 @@ if test -e $outPath/foobar; then false; fi # Check that the store is empty. rmdir $NIX_STORE_DIR/.links rmdir $NIX_STORE_DIR - -## Test `nix-collect-garbage -d` -testCollectGarbageD () { - clearProfiles - # Run two `nix-env` commands, should create two generations of - # the profile - nix-env -f ./user-envs.nix -i foo-1.0 - nix-env -f ./user-envs.nix -i foo-2.0pre1 - [[ $(nix-env --list-generations | wc -l) -eq 2 ]] - - # Clear the profile history. There should be only one generation - # left - nix-collect-garbage -d - [[ $(nix-env --list-generations | wc -l) -eq 1 ]] -} -# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're -# using them -if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then - testCollectGarbageD - - # Run the same test, but forcing the profiles at their legacy location under - # /nix/var/nix. - # - # Regression test for #8294 - rm ~/.nix-profile - ln -s $NIX_STATE_DIR/profiles/per-user/me ~/.nix-profile - testCollectGarbageD -fi diff --git a/tests/lang/eval-fail-bad-antiquote-1.nix b/tests/lang/eval-fail-bad-string-interpolation-1.nix index ffe9c983c..ffe9c983c 100644 --- a/tests/lang/eval-fail-bad-antiquote-1.nix +++ b/tests/lang/eval-fail-bad-string-interpolation-1.nix diff --git a/tests/lang/eval-fail-bad-antiquote-2.nix b/tests/lang/eval-fail-bad-string-interpolation-2.nix index 3745235ce..3745235ce 100644 --- a/tests/lang/eval-fail-bad-antiquote-2.nix +++ b/tests/lang/eval-fail-bad-string-interpolation-2.nix diff --git a/tests/lang/eval-fail-bad-antiquote-3.nix b/tests/lang/eval-fail-bad-string-interpolation-3.nix index 65b9d4f50..65b9d4f50 100644 --- a/tests/lang/eval-fail-bad-antiquote-3.nix +++ b/tests/lang/eval-fail-bad-string-interpolation-3.nix diff --git a/tests/lang/eval-okay-path-antiquotation.exp b/tests/lang/eval-okay-path-string-interpolation.exp index 5b8ea0243..5b8ea0243 100644 --- a/tests/lang/eval-okay-path-antiquotation.exp +++ b/tests/lang/eval-okay-path-string-interpolation.exp diff --git a/tests/lang/eval-okay-path-antiquotation.nix b/tests/lang/eval-okay-path-string-interpolation.nix index 497d7c1c7..497d7c1c7 100644 --- a/tests/lang/eval-okay-path-antiquotation.nix +++ b/tests/lang/eval-okay-path-string-interpolation.nix diff --git a/tests/local.mk b/tests/local.mk index 9e340e2e2..2da47d243 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -16,6 +16,7 @@ nix_tests = \ flakes/flake-in-submodule.sh \ ca/gc.sh \ gc.sh \ + nix-collect-garbage-d.sh \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index dbb3114f1..b5d935004 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -8,6 +8,7 @@ rm -f $TEST_HOME/.nix-channels $TEST_HOME/.nix-profile nix-channel --add http://foo/bar xyzzy nix-channel --list | grepQuiet http://foo/bar nix-channel --remove xyzzy +[[ $(nix-channel --list-generations | wc -l) == 1 ]] [ -e $TEST_HOME/.nix-channels ] [ "$(cat $TEST_HOME/.nix-channels)" = '' ] @@ -38,6 +39,7 @@ ln -s dependencies.nix $TEST_ROOT/nixexprs/default.nix # Test the update action. nix-channel --add file://$TEST_ROOT/foo nix-channel --update +[[ $(nix-channel --list-generations | wc -l) == 2 ]] # Do a query. nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml diff --git a/tests/nix-collect-garbage-d.sh b/tests/nix-collect-garbage-d.sh new file mode 100644 index 000000000..bf30f8938 --- /dev/null +++ b/tests/nix-collect-garbage-d.sh @@ -0,0 +1,40 @@ +source common.sh + +clearStore + +## Test `nix-collect-garbage -d` + +# TODO make `nix-env` doesn't work with CA derivations, and make +# `ca/nix-collect-garbage-d.sh` wrapper. + +testCollectGarbageD () { + clearProfiles + # Run two `nix-env` commands, should create two generations of + # the profile + nix-env -f ./user-envs.nix -i foo-1.0 "$@" + nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@" + [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]] + + # Clear the profile history. There should be only one generation + # left + nix-collect-garbage -d + [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]] +} + +testCollectGarbageD + +# Run the same test, but forcing the profiles an arbitrary location. +rm ~/.nix-profile +ln -s $TEST_ROOT/blah ~/.nix-profile +testCollectGarbageD + +# Run the same test, but forcing the profiles at their legacy location under +# /nix/var/nix. +# +# Note that we *don't* use the default profile; `nix-collect-garbage` will +# need to check the legacy conditional unconditionally not just follow +# `~/.nix-profile` to pass this test. +# +# Regression test for #8294 +rm ~/.nix-profile +testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me" diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc index 04b791021..e02fd68d5 100644 --- a/tests/plugins/plugintest.cc +++ b/tests/plugins/plugintest.cc @@ -21,4 +21,8 @@ static void prim_anotherNull (EvalState & state, const PosIdx pos, Value ** args v.mkBool(false); } -static RegisterPrimOp rp("anotherNull", 0, prim_anotherNull); +static RegisterPrimOp rp({ + .name = "anotherNull", + .arity = 0, + .fun = prim_anotherNull, +}); |