diff options
155 files changed, 3022 insertions, 1650 deletions
diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md new file mode 100644 index 000000000..537aa0909 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -0,0 +1,7 @@ +**Release Notes** +Please include relevant [release notes](https://github.com/NixOS/nix/blob/master/doc/manual/src/release-notes/rl-next.md) as needed. + + +**Testing** + +If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master). diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml new file mode 100644 index 000000000..ec7ab4516 --- /dev/null +++ b/.github/workflows/backport.yml @@ -0,0 +1,26 @@ +name: Backport +on: + pull_request_target: + types: [closed, labeled] +jobs: + backport: + name: Backport Pull Request + if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + # required to find all branches + fetch-depth: 0 + - name: Create backport PRs + # should be kept in sync with `version` + uses: zeebe-io/backport-action@v0.0.7 + with: + # Config README: https://github.com/zeebe-io/backport-action#backport-action + github_token: ${{ secrets.GITHUB_TOKEN }} + github_workspace: ${{ github.workspace }} + pull_description: |- + Bot-based backport to `${target_branch}`, triggered by a label in #${pull_number}. + # should be kept in sync with `uses` + version: v0.0.5 diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml new file mode 100644 index 000000000..b97076bd7 --- /dev/null +++ b/.github/workflows/hydra_status.yml @@ -0,0 +1,16 @@ +name: Hydra status +on: + schedule: + - cron: "12,42 * * * *" + workflow_dispatch: +jobs: + check_hydra_status: + name: Check Hydra status + if: github.repository_owner == 'NixOS' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.4.0 + with: + fetch-depth: 0 + - run: bash scripts/check-hydra-status.sh + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9105b5396..1b655e27d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,20 +1,23 @@ name: "Test" + on: pull_request: push: + jobs: + tests: needs: [check_cachix] strategy: matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} - + timeout-minutes: 60 steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2.4.0 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v14 + - uses: cachix/install-nix-action@v16 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v10 if: needs.check_cachix.outputs.secret == 'true' @@ -23,6 +26,7 @@ jobs: signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix-build -A checks.$(nix-instantiate --eval -E '(builtins.currentSystem)') + check_cachix: name: Cachix secret present for installer tests runs-on: ubuntu-latest @@ -34,6 +38,7 @@ jobs: env: _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} run: echo "::set-output name=secret::${{ env._CACHIX_SECRETS != '' }}" + installer: needs: [tests, check_cachix] if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' @@ -41,11 +46,11 @@ jobs: outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2.4.0 with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v14 + - uses: cachix/install-nix-action@v16 - uses: cachix/cachix-action@v10 with: name: '${{ env.CACHIX_NAME }}' @@ -53,6 +58,7 @@ jobs: authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - id: prepare-installer run: scripts/prepare-installer-for-github-actions + installer_test: needs: [installer, check_cachix] if: github.event_name == 'push' && needs.check_cachix.outputs.secret == 'true' @@ -61,9 +67,9 @@ jobs: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2.4.0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v14 + - uses: cachix/install-nix-action@v16 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" diff --git a/.gitignore b/.gitignore index b6a5f6adc..2889a56eb 100644 --- a/.gitignore +++ b/.gitignore @@ -26,8 +26,6 @@ perl/Makefile.config # /scripts/ /scripts/nix-profile.sh -/scripts/nix-reduce-build -/scripts/nix-http-export.cgi /scripts/nix-profile-daemon.sh # /src/libexpr/ @@ -1 +1 @@ -2.5
\ No newline at end of file +2.5.0
\ No newline at end of file diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index fa8dd0325..e659bf470 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -1,8 +1,8 @@ diff --git a/pthread_stop_world.c b/pthread_stop_world.c -index 1cee6a0b..46c3acd9 100644 +index 4b2c429..1fb4c52 100644 --- a/pthread_stop_world.c +++ b/pthread_stop_world.c -@@ -674,6 +674,8 @@ GC_INNER void GC_push_all_stacks(void) +@@ -673,6 +673,8 @@ GC_INNER void GC_push_all_stacks(void) struct GC_traced_stack_sect_s *traced_stack_sect; pthread_t self = pthread_self(); word total_size = 0; @@ -11,7 +11,7 @@ index 1cee6a0b..46c3acd9 100644 if (!EXPECT(GC_thr_initialized, TRUE)) GC_thr_init(); -@@ -723,6 +725,28 @@ GC_INNER void GC_push_all_stacks(void) +@@ -722,6 +724,31 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ @@ -22,6 +22,9 @@ index 1cee6a0b..46c3acd9 100644 + if (pthread_attr_getstacksize(&pattr, &stack_limit)) { + ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!"); + } ++ if (pthread_attr_destroy(&pattr)) { ++ ABORT("GC_push_all_stacks: pthread_attr_destroy failed!"); ++ } + // When a thread goes into a coroutine, we lose its original sp until + // control flow returns to the thread. + // While in the coroutine, the sp points outside the thread stack, diff --git a/configure.ac b/configure.ac index 65478ecc5..c35065704 100644 --- a/configure.ac +++ b/configure.ac @@ -188,17 +188,24 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) ]) -# Look for libsodium, an optional dependency. +# Look for libsodium. PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Look for libbrotli{enc,dec}. PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) # Look for libcpuid. +have_libcpuid= if test "$machine_name" = "x86_64"; then - PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"]) - have_libcpuid=1 - AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid]) + AC_ARG_ENABLE([cpuid], + AS_HELP_STRING([--disable-cpuid], [Do not determine microarchitecture levels with libcpuid (relevant to x86_64 only)])) + if test "x$enable_cpuid" != "xno"; then + PKG_CHECK_MODULES([LIBCPUID], [libcpuid], + [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS" + have_libcpuid=1 + AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])] + ) + fi fi AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid]) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index e43d9f2fb..6b232a736 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -12,11 +12,13 @@ man-pages := $(foreach n, \ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. +# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine dummy-env = env -i \ HOME=/dummy \ NIX_CONF_DIR=/dummy \ NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \ - NIX_STATE_DIR=/dummy + NIX_STATE_DIR=/dummy \ + NIX_CONFIG='cores = 0' nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index df9209c7d..2876be52a 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -9,6 +9,7 @@ - [Prerequisites](installation/prerequisites-source.md) - [Obtaining a Source Distribution](installation/obtaining-source.md) - [Building Nix from Source](installation/building-source.md) + - [Using Nix within Docker](installation/installing-docker.md) - [Security](installation/nix-security.md) - [Single-User Mode](installation/single-user.md) - [Multi-User Mode](installation/multi-user.md) @@ -70,7 +71,9 @@ - [Hacking](contributing/hacking.md) - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - - [Release 2.4 (2021-XX-XX)](release-notes/rl-2.4.md) + - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md) + - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md) - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md) - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md) - [Release 2.1 (2018-09-02)](release-notes/rl-2.1.md) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 580b36736..c4c60db15 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -53,8 +53,8 @@ example, the following command allows you to build a derivation for $ uname Linux -$ nix build \ - '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ +$ nix build --impure \ + --expr '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ --builders 'ssh://mac x86_64-darwin' [1/0/1 built, 0.0 MiB DL] building foo on ssh://mac diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 3140170ab..44b7ba86d 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -16,8 +16,9 @@ By default Nix reads settings from the following places: will be loaded in reverse order. Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` - and `XDG_CONFIG_HOME`. If these are unset, it will look in - `$HOME/.config/nix.conf`. + and `XDG_CONFIG_HOME`. If unset, `XDG_CONFIG_DIRS` defaults to + `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` + as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). - If `NIX_CONFIG` is set, its contents is treated as the contents of a configuration file. diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index 9138fa05a..8d6abaf52 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -238,7 +238,16 @@ a number of possible ways: ## Examples -To install a specific version of `gcc` from the active Nix expression: +To install a package using a specific attribute path from the active Nix expression: + +```console +$ nix-env -iA gcc40mips +installing `gcc-4.0.2' +$ nix-env -iA xorg.xorgserver +installing `xorg-server-1.2.0' +``` + +To install a specific version of `gcc` using the derivation name: ```console $ nix-env --install gcc-3.3.2 @@ -246,6 +255,9 @@ installing `gcc-3.3.2' uninstalling `gcc-3.1' ``` +Using attribute path for selecting a package is preferred, +as it is much faster and there will not be multiple matches. + Note the previously installed version is removed, since `--preserve-installed` was not specified. @@ -256,13 +268,6 @@ $ nix-env --install gcc installing `gcc-3.3.2' ``` -To install using a specific attribute: - -```console -$ nix-env -i -A gcc40mips -$ nix-env -i -A xorg.xorgserver -``` - To install all derivations in the Nix expression `foo.nix`: ```console @@ -374,22 +379,29 @@ For the other flags, see `--install`. ## Examples ```console -$ nix-env --upgrade gcc +$ nix-env --upgrade -A nixpkgs.gcc upgrading `gcc-3.3.1' to `gcc-3.4' ``` +When there are no updates available, nothing will happen: + ```console -$ nix-env -u gcc-3.3.2 --always (switch to a specific version) -upgrading `gcc-3.4' to `gcc-3.3.2' +$ nix-env --upgrade -A nixpkgs.pan ``` +Using `-A` is preferred when possible, as it is faster and unambiguous but +it is also possible to upgrade to a specific version by matching the derivation name: + ```console -$ nix-env --upgrade pan -(no upgrades available, so nothing happens) +$ nix-env -u gcc-3.3.2 --always +upgrading `gcc-3.4' to `gcc-3.3.2' ``` +To try to upgrade everything +(matching packages based on the part of the derivation name without version): + ```console -$ nix-env -u (try to upgrade everything) +$ nix-env -u upgrading `hello-2.1.2' to `hello-2.1.3' upgrading `mozilla-1.2' to `mozilla-1.4' ``` @@ -401,7 +413,7 @@ of a derivation `x` by looking at their respective `name` attributes. The names (e.g., `gcc-3.3.1` are split into two parts: the package name (`gcc`), and the version (`3.3.1`). The version part starts after the first dash not followed by a letter. `x` is considered an upgrade of `y` -if their package names match, and the version of `y` is higher that that +if their package names match, and the version of `y` is higher than that of `x`. The versions are compared by splitting them into contiguous components diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 0bd54b955..873311649 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -11,8 +11,8 @@ [`--command` *cmd*] [`--run` *cmd*] [`--exclude` *regexp*] - [--pure] - [--keep *name*] + [`--pure`] + [`--keep` *name*] {{`--packages` | `-p`} {*packages* | *expressions*} … | [*path*]} # Description diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/src/command-ref/nix-store.md index 7a131dc02..26292f1bb 100644 --- a/doc/manual/src/command-ref/nix-store.md +++ b/doc/manual/src/command-ref/nix-store.md @@ -125,7 +125,7 @@ Special exit codes: - `104`\ Not deterministic, the build succeeded in check mode but the - resulting output is not binary reproducable. + resulting output is not binary reproducible. With the `--keep-going` flag it's possible for multiple failures to occur, in this case the 1xx status codes are or combined using binary diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 47862bc09..7ee1a26bc 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -162,11 +162,11 @@ Most Nix commands accept the following command-line options: }: ... ``` - So if you call this Nix expression (e.g., when you do `nix-env -i + So if you call this Nix expression (e.g., when you do `nix-env -iA pkgname`), the function will be called automatically using the value [`builtins.currentSystem`](../expressions/builtins.md) for the `system` argument. You can override this using `--arg`, e.g., - `nix-env -i pkgname --arg system \"i686-freebsd\"`. (Note that + `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the quotes.) diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index 0132867c8..01a1b1e73 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -3,7 +3,7 @@ ## Goals Purpose of this document is to provide a clear direction to **help design -delightful command line** experience. This document contain guidelines to +delightful command line** experience. This document contains guidelines to follow to ensure a consistent and approachable user experience. ## Overview @@ -103,7 +103,7 @@ impacted the most by bad user experience. # Help is essential Help should be built into your command line so that new users can gradually -discover new features when they need them. +discover new features when they need them. ## Looking for help @@ -115,7 +115,7 @@ The rules are: - Help is shown by using `--help` or `help` command (eg `nix` `--``help` or `nix help`). -- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show +- For non-COMMANDs (eg. `nix` `--``help` and `nix store` `--``help`) we **show a summary** of most common use cases. Summary is presented on the STDOUT without any use of PAGER. - For COMMANDs (eg. `nix init` `--``help` or `nix help init`) we display the @@ -176,7 +176,7 @@ $ nix init --template=template#pyton ------------------------------------------------------------------------ Initializing Nix project at `/path/to/here`. Select a template for you new project: - |> template#pyton + |> template#python template#python-pip template#python-poetry ``` @@ -230,17 +230,17 @@ Now **Learn** part of the output is where you educate users. You should only show it when you know that a build will take some time and not annoy users of the builds that take only few seconds. -Every feature like this should go though a intensive review and testing to -collect as much a feedback as possible and to fine tune every little detail. If +Every feature like this should go through an intensive review and testing to +collect as much feedback as possible and to fine tune every little detail. If done right this can be an awesome features beginners and advance users will love, but if not done perfectly it will annoy users and leave bad impression. # Input -Input to a command is provided via `ARGUMENTS` and `OPTIONS`. +Input to a command is provided via `ARGUMENTS` and `OPTIONS`. `ARGUMENTS` represent a required input for a function. When choosing to use -`ARGUMENT` over function please be aware of the downsides that come with it: +`ARGUMENTS` over `OPTIONS` please be aware of the downsides that come with it: - User will need to remember the order of `ARGUMENTS`. This is not a problem if there is only one `ARGUMENT`. @@ -253,7 +253,7 @@ developer consider the downsides and choose wisely. ## Naming the `OPTIONS` -Then only naming convention - apart from the ones mentioned in Naming the +The only naming convention - apart from the ones mentioned in Naming the `COMMANDS` section is how flags are named. Flags are a type of `OPTION` that represent an option that can be turned ON of @@ -271,12 +271,12 @@ to improve the discoverability of possible input. A new user will most likely not know which `ARGUMENTS` and `OPTIONS` are required or which values are possible for those options. -In cases, the user might not provide the input or they provide wrong input, -rather then show the error, prompt a user with an option to find and select +In case the user does not provide the input or they provide wrong input, +rather than show the error, prompt a user with an option to find and select correct input (see examples). Prompting is of course not required when TTY is not attached to STDIN. This -would mean that scripts wont need to handle prompt, but rather handle errors. +would mean that scripts won't need to handle prompt, but rather handle errors. A place to use prompt and provide user with interactive select @@ -300,9 +300,9 @@ going to happen. ```shell $ nix build --option substitutors https://cache.example.org ------------------------------------------------------------------------ - Warning! A security related question need to be answered. + Warning! A security related question needs to be answered. ------------------------------------------------------------------------ - The following substitutors will be used to in `my-project`: + The following substitutors will be used to in `my-project`: - https://cache.example.org Do you allow `my-project` to use above mentioned substitutors? @@ -311,14 +311,14 @@ $ nix build --option substitutors https://cache.example.org # Output -Terminal output can be quite limiting in many ways. Which should forces us to +Terminal output can be quite limiting in many ways. Which should force us to think about the experience even more. As with every design the output is a compromise between being terse and being verbose, between showing help to beginners and annoying advance users. For this it is important that we know what are the priorities. Nix command line should be first and foremost written with beginners in mind. -But users wont stay beginners for long and what was once useful might quickly +But users won't stay beginners for long and what was once useful might quickly become annoying. There is no golden rule that we can give in this guideline that would make it easier how to draw a line and find best compromise. @@ -342,7 +342,7 @@ also allowing them to redirect content to a file. For example: ```shell $ nix build > build.txt ------------------------------------------------------------------------ - Error! Atrribute `bin` missing at (1:94) from string. + Error! Attribute `bin` missing at (1:94) from string. ------------------------------------------------------------------------ 1| with import <nixpkgs> { }; (pkgs.runCommandCC or pkgs.runCommand) "shell" { buildInputs = [ (surge.bin) ]; } "" @@ -408,7 +408,7 @@ Above command clearly states that command successfully completed. And in case of `nix build`, which is a command that might take some time to complete, it is equally important to also show that a command started. -## Text alignment +## Text alignment Text alignment is the number one design element that will present all of the Nix commands as a family and not as separate tools glued together. @@ -419,7 +419,7 @@ The format we should follow is: $ nix COMMAND VERB_1 NOUN and other words VERB__1 NOUN and other words - |> Some details + |> Some details ``` Few rules that we can extract from above example: @@ -444,13 +444,13 @@ is not even notable, therefore relying on it wouldn’t make much sense. **The bright text is much better supported** across terminals and color schemes. Most of the time the difference is perceived as if the bright text -would be bold. +would be bold. ## Colors Humans are already conditioned by society to attach certain meaning to certain colors. While the meaning is not universal, a simple collection of colors is -used to represent basic emotions. +used to represent basic emotions. Colors that can be used in output @@ -508,7 +508,7 @@ can, with a few key strokes, be changed into and advance introspection tool. ### Progress -For longer running commands we should provide and overview of the progress. +For longer running commands we should provide and overview the progress. This is shown best in `nix build` example: ```shell @@ -553,9 +553,9 @@ going to happen. ```shell $ nix build --option substitutors https://cache.example.org ------------------------------------------------------------------------ - Warning! A security related question need to be answered. + Warning! A security related question needs to be answered. ------------------------------------------------------------------------ - The following substitutors will be used to in `my-project`: + The following substitutors will be used to in `my-project`: - https://cache.example.org Do you allow `my-project` to use above mentioned substitutors? @@ -566,7 +566,7 @@ $ nix build --option substitutors https://cache.example.org There are many ways that you can control verbosity. -Verbosity levels are: +Verbosity levels are: - `ERROR` (level 0) - `WARN` (level 1) @@ -586,4 +586,4 @@ There are also two shortcuts, `--debug` to run in `DEBUG` verbosity level and # Appendix 1: Commands naming exceptions -`nix init` and `nix repl` are well established +`nix init` and `nix repl` are well established diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 2a1e55e5b..90a8f1f94 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -35,6 +35,25 @@ variables are set up so that those dependencies can be found: $ nix-shell ``` +or if you have a flake-enabled nix: + +```console +$ nix develop +``` + +To get a shell with a different compilation environment (e.g. stdenv, +gccStdenv, clangStdenv, clang11Stdenv): + +```console +$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages +``` + +or if you have a flake-enabled nix: + +```console +$ nix develop .#clang11StdenvPackages +``` + To build Nix itself in this shell: ```console diff --git a/doc/manual/src/expressions/advanced-attributes.md b/doc/manual/src/expressions/advanced-attributes.md index 5b208df67..000595815 100644 --- a/doc/manual/src/expressions/advanced-attributes.md +++ b/doc/manual/src/expressions/advanced-attributes.md @@ -237,7 +237,7 @@ Derivations can declare some infrequently used optional attributes. - `preferLocalBuild`\ If this attribute is set to `true` and [distributed building is enabled](../advanced-topics/distributed-builds.md), then, if - possible, the derivaton will be built locally instead of forwarded + possible, the derivation will be built locally instead of forwarded to a remote machine. This is appropriate for trivial builders where the cost of doing a download or remote build would exceed the cost of building locally. diff --git a/doc/manual/src/expressions/builtins-prefix.md b/doc/manual/src/expressions/builtins-prefix.md index 87127de2a..c631a8453 100644 --- a/doc/manual/src/expressions/builtins-prefix.md +++ b/doc/manual/src/expressions/builtins-prefix.md @@ -12,5 +12,5 @@ For instance, `derivation` is also available as `builtins.derivation`. <dl> <dt><code>derivation <var>attrs</var></code>; <code>builtins.derivation <var>attrs</var></code></dt> - <dd><p><var>derivation</var> in described in + <dd><p><var>derivation</var> is described in <a href="derivations.md">its own section</a>.</p></dd> diff --git a/doc/manual/src/expressions/expression-syntax.md b/doc/manual/src/expressions/expression-syntax.md index 2a1306e32..6b93e692c 100644 --- a/doc/manual/src/expressions/expression-syntax.md +++ b/doc/manual/src/expressions/expression-syntax.md @@ -26,7 +26,7 @@ elements (referenced from the figure by number): called with three arguments: `stdenv`, `fetchurl`, and `perl`. They are needed to build Hello, but we don't know how to build them here; that's why they are function arguments. `stdenv` is a package that - is used by almost all Nix Packages packages; it provides a + is used by almost all Nix Packages; it provides a “standard” environment consisting of the things you would expect in a basic Unix environment: a C/C++ compiler (GCC, to be precise), the Bash shell, fundamental Unix tools such as `cp`, `grep`, `tar`, diff --git a/doc/manual/src/expressions/language-operators.md b/doc/manual/src/expressions/language-operators.md index b7fd6f4c6..268b44f4c 100644 --- a/doc/manual/src/expressions/language-operators.md +++ b/doc/manual/src/expressions/language-operators.md @@ -17,12 +17,12 @@ order of precedence (from strongest to weakest binding). | String Concatenation | *string1* `+` *string2* | left | String concatenation. | 7 | | Not | `!` *e* | none | Boolean negation. | 8 | | Update | *e1* `//` *e2* | right | Return a set consisting of the attributes in *e1* and *e2* (with the latter taking precedence over the former in case of equally named attributes). | 9 | -| Less Than | *e1* `<` *e2*, | none | Arithmetic comparison. | 10 | -| Less Than or Equal To | *e1* `<=` *e2* | none | Arithmetic comparison. | 10 | -| Greater Than | *e1* `>` *e2* | none | Arithmetic comparison. | 10 | -| Greater Than or Equal To | *e1* `>=` *e2* | none | Arithmetic comparison. | 10 | +| Less Than | *e1* `<` *e2*, | none | Arithmetic/lexicographic comparison. | 10 | +| Less Than or Equal To | *e1* `<=` *e2* | none | Arithmetic/lexicographic comparison. | 10 | +| Greater Than | *e1* `>` *e2* | none | Arithmetic/lexicographic comparison. | 10 | +| Greater Than or Equal To | *e1* `>=` *e2* | none | Arithmetic/lexicographic comparison. | 10 | | Equality | *e1* `==` *e2* | none | Equality. | 11 | | Inequality | *e1* `!=` *e2* | none | Inequality. | 11 | | Logical AND | *e1* `&&` *e2* | left | Logical AND. | 12 | -| Logical OR | *e1* `\|\|` *e2* | left | Logical OR. | 13 | -| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to `!e1 \|\| e2`). | 14 | +| Logical OR | *e1* <code>||</code> *e2* | left | Logical OR. | 13 | +| Logical Implication | *e1* `->` *e2* | none | Logical implication (equivalent to <code>!e1 || e2</code>). | 14 | diff --git a/doc/manual/src/expressions/language-values.md b/doc/manual/src/expressions/language-values.md index 28fa23b58..75ae9f2eb 100644 --- a/doc/manual/src/expressions/language-values.md +++ b/doc/manual/src/expressions/language-values.md @@ -64,7 +64,7 @@ Nix has the following basic data types: the start of each line. To be precise, it strips from each line a number of spaces equal to the minimal indentation of the string as a whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two space, while the third + the first and second line are indented two spaces, while the third line is indented four spaces. Thus, two spaces are stripped from each line, so the resulting string is diff --git a/doc/manual/src/expressions/simple-building-testing.md b/doc/manual/src/expressions/simple-building-testing.md index 6f730a936..7f0d8f841 100644 --- a/doc/manual/src/expressions/simple-building-testing.md +++ b/doc/manual/src/expressions/simple-building-testing.md @@ -1,6 +1,6 @@ # Building and Testing -You can now try to build Hello. Of course, you could do `nix-env -i +You can now try to build Hello. Of course, you could do `nix-env -f . -iA hello`, but you may not want to install a possibly broken package just yet. The best way to test the package is by using the command `nix-build`, which builds a Nix expression and creates a symlink named diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index bb350d9de..71ff13275 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -47,7 +47,7 @@ the store object at `P` contains the path `Q` somewhere. The *references* of a store path are the set of store paths to which it has a reference. - + A derivation can reference other derivations and sources (but not output paths), whereas an output path only references other output paths. @@ -66,7 +66,7 @@ is necessary to deploy whole closures, since otherwise at runtime files could be missing. The command `nix-store -qR` prints out closures of store paths. - + As an example, if the store object at path `P` contains a reference to path `Q`, then `Q` is in the closure of `P`. Further, if `Q` references `R` then `R` is also in the closure of `P`. @@ -98,3 +98,7 @@ store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + - `∅` \ + The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + - `ε` \ + The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md index d21a51a82..ed1efffd8 100644 --- a/doc/manual/src/installation/building-source.md +++ b/doc/manual/src/installation/building-source.md @@ -1,9 +1,9 @@ # Building Nix from Source -After unpacking or checking out the Nix sources, issue the following -commands: +After cloning Nix's Git repository, issue the following commands: ```console +$ ./bootstrap.sh $ ./configure options... $ make $ make install @@ -11,13 +11,6 @@ $ make install Nix requires GNU Make so you may need to invoke `gmake` instead. -When building from the Git repository, these should be preceded by the -command: - -```console -$ ./bootstrap.sh -``` - The installation path can be specified by passing the `--prefix=prefix` to `configure`. The default installation directory is `/usr/local`. You can change this to any location you like. You must have write permission diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 96fa34635..4367654a2 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -119,6 +119,30 @@ this to run the installer, but it may help if you run into trouble: - update `/etc/synthetic.conf` to direct macOS to create a "synthetic" empty root directory to mount your volume - specify mount options for the volume in `/etc/fstab` + - `rw`: read-write + - `noauto`: prevent the system from auto-mounting the volume (so the + LaunchDaemon mentioned below can control mounting it, and to avoid + masking problems with that mounting service). + - `nobrowse`: prevent the Nix Store volume from showing up on your + desktop; also keeps Spotlight from spending resources to index + this volume + <!-- TODO: + - `suid`: honor setuid? surely not? ... + - `owners`: honor file ownership on the volume + + For now I'll avoid pretending to understand suid/owners more + than I do. There've been some vague reports of file-ownership + and permission issues, particularly in cloud/VM/headless setups. + My pet theory is that this has something to do with these setups + not having a token that gets delegated to initial/admin accounts + on macOS. See scripts/create-darwin-volume.sh for a little more. + + In any case, by Dec 4 2021, it _seems_ like some combination of + suid, owners, and calling diskutil enableOwnership have stopped + new reports from coming in. But I hesitate to celebrate because we + haven't really named and catalogued the behavior, understood what + we're fixing, and validated that all 3 components are essential. + --> - if you have FileVault enabled - generate an encryption password - put it in your system Keychain diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md new file mode 100644 index 000000000..ee7614bdb --- /dev/null +++ b/doc/manual/src/installation/installing-docker.md @@ -0,0 +1,59 @@ +# Using Nix within Docker + +To run the latest stable release of Nix with Docker run the following command: + +```console +$ docker run -ti nixos/nix +Unable to find image 'nixos/nix:latest' locally +latest: Pulling from nixos/nix +5843afab3874: Pull complete +b52bf13f109c: Pull complete +1e2415612aa3: Pull complete +Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff +Status: Downloaded newer image for nixos/nix:latest +35ca4ada6e96:/# nix --version +nix (Nix) 2.3.12 +35ca4ada6e96:/# exit +``` + +# What is included in Nix' Docker image? + +The official Docker image is created using `pkgs.dockerTools.buildLayeredImage` +(and not with `Dockerfile` as it is usual with Docker images). You can still +base your custom Docker image on it as you would do with any other Docker +image. + +The Docker image is also not based on any other image and includes minimal set +of runtime dependencies that are required to use Nix: + + - pkgs.nix + - pkgs.bashInteractive + - pkgs.coreutils-full + - pkgs.gnutar + - pkgs.gzip + - pkgs.gnugrep + - pkgs.which + - pkgs.curl + - pkgs.less + - pkgs.wget + - pkgs.man + - pkgs.cacert.out + - pkgs.findutils + +# Docker image with the latest development version of Nix + +To get the latest image that was built by [Hydra](https://hydra.nixos.org) run +the following command: + +```console +$ curl -L https://hydra.nixos.org/job/nix/master/dockerImage.x86_64-linux/latest/download/1 | docker load +$ docker run -ti nix:2.5pre20211105 +``` + +You can also build a Docker image from source yourself: + +```console +$ nix build ./\#hydraJobs.dockerImage.x86_64-linux +$ docker load -i ./result +$ docker run -ti nix:2.5pre20211105 +``` diff --git a/doc/manual/src/installation/installing-source.md b/doc/manual/src/installation/installing-source.md index e52d38a03..09b4e4887 100644 --- a/doc/manual/src/installation/installing-source.md +++ b/doc/manual/src/installation/installing-source.md @@ -1,4 +1,4 @@ # Installing Nix from Source -If no binary package is available, you can download and compile a source -distribution. +If no binary package is available or if you want to hack on Nix, you +can build Nix from its Git repository. diff --git a/doc/manual/src/installation/obtaining-source.md b/doc/manual/src/installation/obtaining-source.md index 0a906e390..da05d243d 100644 --- a/doc/manual/src/installation/obtaining-source.md +++ b/doc/manual/src/installation/obtaining-source.md @@ -1,14 +1,9 @@ -# Obtaining a Source Distribution +# Obtaining the Source -The source tarball of the most recent stable release can be downloaded -from the [Nix homepage](http://nixos.org/nix/download.html). You can -also grab the [most recent development -release](http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents). - -Alternatively, the most recent sources of Nix can be obtained from its -[Git repository](https://github.com/NixOS/nix). For example, the -following command will check out the latest revision into a directory -called `nix`: +The most recent sources of Nix can be obtained from its [Git +repository](https://github.com/NixOS/nix). For example, the following +command will check out the latest revision into a directory called +`nix`: ```console $ git clone https://github.com/NixOS/nix diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 40cb79627..6f4eb3008 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -2,9 +2,8 @@ - GNU Autoconf (<https://www.gnu.org/software/autoconf/>) and the autoconf-archive macro collection - (<https://www.gnu.org/software/autoconf-archive/>). These are only - needed to run the bootstrap script, and are not necessary if your - source distribution came with a pre-built `./configure` script. + (<https://www.gnu.org/software/autoconf-archive/>). These are + needed to run the bootstrap script. - GNU Make. @@ -45,6 +44,11 @@ obtained from the its repository <https://github.com/troglobit/editline>. + - The `libsodium` library for verifying cryptographic signatures + of contents fetched from binary caches. + It can be obtained from the official web site + <https://libsodium.org>. + - Recent versions of Bison and Flex to build the parser. (This is because Nix needs GLR support in Bison and reentrancy support in Flex.) For Bison, you need version 2.6, which can be obtained from @@ -52,11 +56,18 @@ you need version 2.5.35, which is available on [SourceForge](http://lex.sourceforge.net/). Slightly older versions may also work, but ancient versions like the ubiquitous 2.5.4a - won't. Note that these are only required if you modify the parser or - when you are building from the Git repository. + won't. - The `libseccomp` is used to provide syscall filtering on Linux. This is an optional dependency and can be disabled passing a `--disable-seccomp-sandboxing` option to the `configure` script (Not recommended unless your system doesn't support `libseccomp`). To get the library, visit <https://github.com/seccomp/libseccomp>. + + - On 64-bit x86 machines only, `libcpuid` library + is used to determine which microarchitecture levels are supported + (e.g., as whether to have `x86_64-v2-linux` among additional system types). + The library is available from its homepage + <http://libcpuid.sourceforge.net>. + This is an optional dependency and can be disabled + by providing a `--disable-cpuid` to the `configure` script. diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md index d68445c95..d87487a07 100644 --- a/doc/manual/src/introduction.md +++ b/doc/manual/src/introduction.md @@ -76,7 +76,7 @@ there after an upgrade. This means that you can _roll back_ to the old version: ```console -$ nix-env --upgrade some-packages +$ nix-env --upgrade -A nixpkgs.some-package $ nix-env --rollback ``` @@ -122,12 +122,12 @@ Nix expressions generally describe how to build a package from source, so an installation action like ```console -$ nix-env --install firefox +$ nix-env --install -A nixpkgs.firefox ``` _could_ cause quite a bit of build activity, as not only Firefox but also all its dependencies (all the way up to the C library and the -compiler) would have to built, at least if they are not already in the +compiler) would have to be built, at least if they are not already in the Nix store. This is a _source deployment model_. For most users, building from source is not very pleasant as it takes far too long. However, Nix can automatically skip building from source and instead diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md index 9702a29eb..5f1d7a89c 100644 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ b/doc/manual/src/package-management/basic-package-mgmt.md @@ -24,7 +24,7 @@ collection; you could write your own Nix expressions based on Nixpkgs, or completely new ones.) You can manually download the latest version of Nixpkgs from -<http://nixos.org/nixpkgs/download.html>. However, it’s much more +<https://github.com/NixOS/nixpkgs>. However, it’s much more convenient to use the Nixpkgs [*channel*](channels.md), since it makes it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is automatically added to your list of “subscribed” channels when you @@ -40,48 +40,52 @@ $ nix-channel --update > > On NixOS, you’re automatically subscribed to a NixOS channel > corresponding to your NixOS major release (e.g. -> <http://nixos.org/channels/nixos-14.12>). A NixOS channel is identical +> <http://nixos.org/channels/nixos-21.11>). A NixOS channel is identical > to the Nixpkgs channel, except that it contains only Linux binaries > and is updated only if a set of regression tests succeed. You can view the set of available packages in Nixpkgs: ```console -$ nix-env -qa -aterm-2.2 -bash-3.0 -binutils-2.15 -bison-1.875d -blackdown-1.4.2 -bzip2-1.0.2 +$ nix-env -qaP +nixpkgs.aterm aterm-2.2 +nixpkgs.bash bash-3.0 +nixpkgs.binutils binutils-2.15 +nixpkgs.bison bison-1.875d +nixpkgs.blackdown blackdown-1.4.2 +nixpkgs.bzip2 bzip2-1.0.2 … ``` -The flag `-q` specifies a query operation, and `-a` means that you want +The flag `-q` specifies a query operation, `-a` means that you want to show the “available” (i.e., installable) packages, as opposed to the -installed packages. If you downloaded Nixpkgs yourself, or if you -checked it out from GitHub, then you need to pass the path to your -Nixpkgs tree using the `-f` flag: +installed packages, and `-P` prints the attribute paths that can be used +to unambiguously select a package for installation (listed in the first column). +If you downloaded Nixpkgs yourself, or if you checked it out from GitHub, +then you need to pass the path to your Nixpkgs tree using the `-f` flag: ```console -$ nix-env -qaf /path/to/nixpkgs +$ nix-env -qaPf /path/to/nixpkgs +aterm aterm-2.2 +bash bash-3.0 +… ``` where */path/to/nixpkgs* is where you’ve unpacked or checked out Nixpkgs. -You can select specific packages by name: +You can filter the packages by name: ```console -$ nix-env -qa firefox -firefox-34.0.5 -firefox-with-plugins-34.0.5 +$ nix-env -qaP firefox +nixpkgs.firefox-esr firefox-91.3.0esr +nixpkgs.firefox firefox-94.0.1 ``` and using regular expressions: ```console -$ nix-env -qa 'firefox.*' +$ nix-env -qaP 'firefox.*' ``` It is also possible to see the *status* of available packages, i.e., @@ -89,11 +93,11 @@ whether they are installed into the user environment and/or present in the system: ```console -$ nix-env -qas +$ nix-env -qaPs … --PS bash-3.0 ---S binutils-2.15 -IPS bison-1.875d +-PS nixpkgs.bash bash-3.0 +--S nixpkgs.binutils binutils-2.15 +IPS nixpkgs.bison bison-1.875d … ``` @@ -106,13 +110,13 @@ which is Nix’s mechanism for doing binary deployment. It just means that Nix knows that it can fetch a pre-built package from somewhere (typically a network server) instead of building it locally. -You can install a package using `nix-env -i`. For instance, +You can install a package using `nix-env -iA`. For instance, ```console -$ nix-env -i subversion +$ nix-env -iA nixpkgs.subversion ``` -will install the package called `subversion` (which is, of course, the +will install the package called `subversion` from `nixpkgs` channel (which is, of course, the [Subversion version management system](http://subversion.tigris.org/)). > **Note** @@ -122,7 +126,7 @@ will install the package called `subversion` (which is, of course, the > binary cache <https://cache.nixos.org>; it contains binaries for most > packages in Nixpkgs. Only if no binary is available in the binary > cache, Nix will build the package from source. So if `nix-env -> -i subversion` results in Nix building stuff from source, then either +> -iA nixpkgs.subversion` results in Nix building stuff from source, then either > the package is not built for your platform by the Nixpkgs build > servers, or your version of Nixpkgs is too old or too new. For > instance, if you have a very recent checkout of Nixpkgs, then the @@ -133,7 +137,10 @@ will install the package called `subversion` (which is, of course, the > using a Git checkout of the Nixpkgs tree), you will get binaries for > most packages. -Naturally, packages can also be uninstalled: +Naturally, packages can also be uninstalled. Unlike when installing, you will +need to use the derivation name (though the version part can be omitted), +instead of the attribute path, as `nix-env` does not record which attribute +was used for installing: ```console $ nix-env -e subversion @@ -143,7 +150,7 @@ Upgrading to a new version is just as easy. If you have a new release of Nix Packages, you can do: ```console -$ nix-env -u subversion +$ nix-env -uA nixpkgs.subversion ``` This will *only* upgrade Subversion if there is a “newer” version in the diff --git a/doc/manual/src/package-management/binary-cache-substituter.md b/doc/manual/src/package-management/binary-cache-substituter.md index bdc5038fc..ef738794b 100644 --- a/doc/manual/src/package-management/binary-cache-substituter.md +++ b/doc/manual/src/package-management/binary-cache-substituter.md @@ -9,7 +9,7 @@ The daemon that handles binary cache requests via HTTP, `nix-serve`, is not part of the Nix distribution, but you can install it from Nixpkgs: ```console -$ nix-env -i nix-serve +$ nix-env -iA nixpkgs.nix-serve ``` You can then start the server, listening for HTTP connections on @@ -35,7 +35,7 @@ On the client side, you can tell Nix to use your binary cache using `--option extra-binary-caches`, e.g.: ```console -$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/ +$ nix-env -iA nixpkgs.firefox --option extra-binary-caches http://avalon:8080/ ``` The option `extra-binary-caches` tells Nix to use this binary cache in diff --git a/doc/manual/src/package-management/garbage-collection.md b/doc/manual/src/package-management/garbage-collection.md index fecb30fd6..29a3b3101 100644 --- a/doc/manual/src/package-management/garbage-collection.md +++ b/doc/manual/src/package-management/garbage-collection.md @@ -44,7 +44,7 @@ collector as follows: $ nix-store --gc ``` -The behaviour of the gargage collector is affected by the +The behaviour of the garbage collector is affected by the `keep-derivations` (default: true) and `keep-outputs` (default: false) options in the Nix configuration file. The defaults will ensure that all derivations that are build-time dependencies of garbage collector roots diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md index fbbfb7320..d1a2580d4 100644 --- a/doc/manual/src/package-management/profiles.md +++ b/doc/manual/src/package-management/profiles.md @@ -39,7 +39,7 @@ just Subversion 1.1.2 (arrows in the figure indicate symlinks). This would be what we would obtain if we had done ```console -$ nix-env -i subversion +$ nix-env -iA nixpkgs.subversion ``` on a set of Nix expressions that contained Subversion 1.1.2. @@ -54,7 +54,7 @@ environment is generated based on the current one. For instance, generation 43 was created from generation 42 when we did ```console -$ nix-env -i subversion firefox +$ nix-env -iA nixpkgs.subversion nixpkgs.firefox ``` on a set of Nix expressions that contained Firefox and a new version of @@ -127,7 +127,7 @@ All `nix-env` operations work on the profile pointed to by (abbreviation `-p`): ```console -$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion +$ nix-env -p /nix/var/nix/profiles/other-profile -iA nixpkgs.subversion ``` This will *not* change the `~/.nix-profile` symlink. diff --git a/doc/manual/src/package-management/ssh-substituter.md b/doc/manual/src/package-management/ssh-substituter.md index 6e5e258bc..c59933f61 100644 --- a/doc/manual/src/package-management/ssh-substituter.md +++ b/doc/manual/src/package-management/ssh-substituter.md @@ -6,7 +6,7 @@ automatically fetching any store paths in Firefox’s closure if they are available on the server `avalon`: ```console -$ nix-env -i firefox --substituters ssh://alice@avalon +$ nix-env -iA nixpkgs.firefox --substituters ssh://alice@avalon ``` This works similar to the binary cache substituter that Nix usually diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 71205923b..b54e73500 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -19,19 +19,19 @@ to subsequent chapters. channel: ```console - $ nix-env -qa - docbook-xml-4.3 - docbook-xml-4.5 - firefox-33.0.2 - hello-2.9 - libxslt-1.1.28 + $ nix-env -qaP + nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 + nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 + nixpkgs.firefox firefox-33.0.2 + nixpkgs.hello hello-2.9 + nixpkgs.libxslt libxslt-1.1.28 … ``` 1. Install some packages from the channel: ```console - $ nix-env -i hello + $ nix-env -iA nixpkgs.hello ``` This should download pre-built packages; it should not build them diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 3c94e6740..0f632f100 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -281,7 +281,9 @@ more than 2800 commits from 195 contributors since release 2.3. * The `nix` command is now marked as an experimental feature. This means that you need to add - > experimental-features = nix-command + ``` + experimental-features = nix-command + ``` to your `nix.conf` if you want to use it, or pass `--extra-experimental-features nix-command` on the command line. @@ -289,30 +291,42 @@ more than 2800 commits from 195 contributors since release 2.3. * The `nix` command no longer has a syntax for referring to packages in a channel. This means that the following no longer works: - > nix build nixpkgs.hello # Nix 2.3 + ```console + nix build nixpkgs.hello # Nix 2.3 + ``` Instead, you can either use the `#` syntax to select a package from a flake, e.g. - > nix build nixpkgs#hello + ```console + nix build nixpkgs#hello + ``` Or, if you want to use the `nixpkgs` channel in the `NIX_PATH` environment variable: - > nix build -f '<nixpkgs>' hello + ```console + nix build -f '<nixpkgs>' hello + ``` * The old `nix run` has been renamed to `nix shell`, while there is a new `nix run` that runs a default command. So instead of - > nix run nixpkgs.hello -c hello # Nix 2.3 + ```console + nix run nixpkgs.hello -c hello # Nix 2.3 + ``` you should use - > nix shell nixpkgs#hello -c hello + ```console + nix shell nixpkgs#hello -c hello + ``` or just - > nix run nixpkgs#hello + ```console + nix run nixpkgs#hello + ``` if the command you want to run has the same name as the package. @@ -381,6 +395,7 @@ dramforever, Dustin DeWeese, edef, Eelco Dolstra, +Ellie Hermaszewska, Emilio Karakey, Emily, Eric Culp, @@ -391,7 +406,7 @@ Federico Pellegrin, Finn Behrens, Florian Franzen, Félix Baylac-Jacqué, -Gabriel Gonzalez, +Gabriella Gonzalez, Geoff Reedy, Georges Dubus, Graham Christensen, @@ -414,7 +429,6 @@ Jaroslavas Pocepko, Jarrett Keifer, Jeremy Schlatter, Joachim Breitner, -Joe Hermaszewski, Joe Pea, John Ericson, Jonathan Ringer, diff --git a/doc/manual/src/release-notes/rl-2.5.md b/doc/manual/src/release-notes/rl-2.5.md index 9a29938c2..61c189d22 100644 --- a/doc/manual/src/release-notes/rl-2.5.md +++ b/doc/manual/src/release-notes/rl-2.5.md @@ -1,5 +1,16 @@ # Release 2.5 (2021-XX-XX) +* The garbage collector no longer blocks new builds, so the message + `waiting for the big garbage collector lock...` is a thing of the + past. + * Binary cache stores now have a setting `compression-level`. * `nix develop` now has a flag `--unpack` to run `unpackPhase`. + +* Lists can now be compared lexicographically using the `<` operator. + +* New built-in function: `builtins.groupBy`, with the same functionality as + Nixpkgs' `lib.groupBy`, but faster. + +* `nix repl` now has a `:log` command. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md new file mode 100644 index 000000000..c869b5e2f --- /dev/null +++ b/doc/manual/src/release-notes/rl-next.md @@ -0,0 +1 @@ +# Release X.Y (202?-??-??) diff --git a/docker.nix b/docker.nix new file mode 100644 index 000000000..2a13c23fb --- /dev/null +++ b/docker.nix @@ -0,0 +1,251 @@ +{ pkgs ? import <nixpkgs> { } +, lib ? pkgs.lib +, name ? "nix" +, tag ? "latest" +, channelName ? "nixpkgs" +, channelURL ? "https://nixos.org/channels/nixpkgs-unstable" +}: +let + defaultPkgs = with pkgs; [ + nix + bashInteractive + coreutils-full + gnutar + gzip + gnugrep + which + curl + less + wget + man + cacert.out + findutils + ]; + + users = { + + root = { + uid = 0; + shell = "/bin/bash"; + home = "/root"; + gid = 0; + }; + + } // lib.listToAttrs ( + map + ( + n: { + name = "nixbld${toString n}"; + value = { + uid = 30000 + n; + gid = 30000; + groups = [ "nixbld" ]; + description = "Nix build user ${toString n}"; + }; + } + ) + (lib.lists.range 1 32) + ); + + groups = { + root.gid = 0; + nixbld.gid = 30000; + }; + + userToPasswd = ( + k: + { uid + , gid ? 65534 + , home ? "/var/empty" + , description ? "" + , shell ? "/bin/false" + , groups ? [ ] + }: "${k}:x:${toString uid}:${toString gid}:${description}:${home}:${shell}" + ); + passwdContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs userToPasswd users)) + ); + + userToShadow = k: { ... }: "${k}:!:1::::::"; + shadowContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs userToShadow users)) + ); + + # Map groups to members + # { + # group = [ "user1" "user2" ]; + # } + groupMemberMap = ( + let + # Create a flat list of user/group mappings + mappings = ( + builtins.foldl' + ( + acc: user: + let + groups = users.${user}.groups or [ ]; + in + acc ++ map + (group: { + inherit user group; + }) + groups + ) + [ ] + (lib.attrNames users) + ); + in + ( + builtins.foldl' + ( + acc: v: acc // { + ${v.group} = acc.${v.group} or [ ] ++ [ v.user ]; + } + ) + { } + mappings) + ); + + groupToGroup = k: { gid }: + let + members = groupMemberMap.${k} or [ ]; + in + "${k}:x:${toString gid}:${lib.concatStringsSep "," members}"; + groupContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs groupToGroup groups)) + ); + + nixConf = { + sandbox = "false"; + build-users-group = "nixbld"; + trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + }; + nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; + + baseSystem = + let + nixpkgs = pkgs.path; + channel = pkgs.runCommand "channel-nixos" { } '' + mkdir $out + ln -s ${nixpkgs} $out/nixpkgs + echo "[]" > $out/manifest.nix + ''; + rootEnv = pkgs.buildPackages.buildEnv { + name = "root-profile-env"; + paths = defaultPkgs; + }; + profile = pkgs.buildPackages.runCommand "user-environment" { } '' + mkdir $out + cp -a ${rootEnv}/* $out/ + + cat > $out/manifest.nix <<EOF + [ + ${lib.concatStringsSep "\n" (builtins.map (drv: let + outputs = drv.outputsToInstall or [ "out" ]; + in '' + { + ${lib.concatStringsSep "\n" (builtins.map (output: '' + ${output} = { outPath = "${lib.getOutput output drv}"; }; + '') outputs)} + outputs = [ ${lib.concatStringsSep " " (builtins.map (x: "\"${x}\"") outputs)} ]; + name = "${drv.name}"; + outPath = "${drv}"; + system = "${drv.system}"; + type = "derivation"; + meta = { }; + } + '') defaultPkgs)} + ] + EOF + ''; + in + pkgs.runCommand "base-system" + { + inherit passwdContents groupContents shadowContents nixConfContents; + passAsFile = [ + "passwdContents" + "groupContents" + "shadowContents" + "nixConfContents" + ]; + allowSubstitutes = false; + preferLocalBuild = true; + } '' + env + set -x + mkdir -p $out/etc + + cat $passwdContentsPath > $out/etc/passwd + echo "" >> $out/etc/passwd + + cat $groupContentsPath > $out/etc/group + echo "" >> $out/etc/group + + cat $shadowContentsPath > $out/etc/shadow + echo "" >> $out/etc/shadow + + mkdir -p $out/usr + ln -s /nix/var/nix/profiles/share $out/usr/ + + mkdir -p $out/nix/var/nix/gcroots + + mkdir $out/tmp + + mkdir -p $out/etc/nix + cat $nixConfContentsPath > $out/etc/nix/nix.conf + + mkdir -p $out/root + mkdir -p $out/nix/var/nix/profiles/per-user/root + + ln -s ${profile} $out/nix/var/nix/profiles/default-1-link + ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default + ln -s /nix/var/nix/profiles/default $out/root/.nix-profile + + ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link + ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels + + mkdir -p $out/root/.nix-defexpr + ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels + echo "${channelURL} ${channelName}" > $out/root/.nix-channels + + mkdir -p $out/bin $out/usr/bin + ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env + ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh + ''; + +in +pkgs.dockerTools.buildLayeredImageWithNixDb { + + inherit name tag; + + contents = [ baseSystem ]; + + extraCommands = '' + rm -rf nix-support + ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles + ''; + + config = { + Cmd = [ "/root/.nix-profile/bin/bash" ]; + Env = [ + "USER=root" + "PATH=${lib.concatStringsSep ":" [ + "/root/.nix-profile/bin" + "/nix/var/nix/profiles/default/bin" + "/nix/var/nix/profiles/default/sbin" + ]}" + "MANPATH=${lib.concatStringsSep ":" [ + "/root/.nix-profile/share/man" + "/nix/var/nix/profiles/default/share/man" + ]}" + "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" + ]; + }; + +} @@ -22,15 +22,36 @@ crossSystems = [ "armv6l-linux" "armv7l-linux" ]; + stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" ]; + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); + forAllSystemsAndStdenvs = f: forAllSystems (system: + nixpkgs.lib.listToAttrs + (map + (n: + nixpkgs.lib.nameValuePair "${n}Packages" ( + f system n + )) stdenvs + ) + ); + + forAllStdenvs = stdenvs: f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv); # Memoize nixpkgs for different platforms for efficiency. - nixpkgsFor = forAllSystems (system: - import nixpkgs { - inherit system; - overlays = [ self.overlay ]; - } - ); + nixpkgsFor = + let stdenvsPackages = forAllSystemsAndStdenvs + (system: stdenv: + import nixpkgs { + inherit system; + overlays = [ + (overlayFor (p: p.${stdenv})) + ]; + } + ); + in + # Add the `stdenvPackages` at toplevel, both because these are the ones + # we want most of the time and for backwards compatibility + forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages); commonDeps = pkgs: with pkgs; rec { # Use "busybox-sandbox-shell" if present, @@ -91,7 +112,7 @@ libarchive boost lowdown-nix - gmock + gtest ] ++ lib.optionals stdenv.isLinux [libseccomp] ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium @@ -255,18 +276,15 @@ $(cat ${installerClosureInfo}/store-paths) ''; - in { - - # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. - overlay = final: prev: { - + overlayFor = getStdenv: final: prev: + let currentStdenv = getStdenv final; in + { nixStable = prev.nix; # Forward from the previous stage as we don’t want it to pick the lowdown override nixUnstable = prev.nixUnstable; - nix = with final; with commonDeps pkgs; stdenv.mkDerivation { + nix = with final; with commonDeps pkgs; currentStdenv.mkDerivation { name = "nix-${version}"; inherit version; @@ -288,9 +306,9 @@ mkdir -p $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a - ${lib.optionalString stdenv.isLinux '' + ${lib.optionalString currentStdenv.isLinux '' chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* ''} ''; @@ -317,7 +335,7 @@ strictDeps = true; - passthru.perl-bindings = with final; stdenv.mkDerivation { + passthru.perl-bindings = with final; currentStdenv.mkDerivation { name = "nix-perl-${version}"; src = self; @@ -336,8 +354,8 @@ pkgs.perl boost ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium + ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; configureFlags = '' --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} @@ -351,7 +369,7 @@ }; - lowdown-nix = with final; stdenv.mkDerivation rec { + lowdown-nix = with final; currentStdenv.mkDerivation rec { name = "lowdown-0.9.0"; src = lowdown-src; @@ -361,15 +379,20 @@ nativeBuildInputs = [ buildPackages.which ]; configurePhase = '' - ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} + ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} ./configure \ PREFIX=${placeholder "dev"} \ BINDIR=${placeholder "bin"}/bin - ''; + ''; }; - }; + in { + + # A Nixpkgs overlay that overrides the 'nix' and + # 'nix.perl-bindings' packages. + overlay = overlayFor (p: p.stdenv); + hydraJobs = { # Binary package for various platforms. @@ -405,6 +428,21 @@ installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; + # docker image with Nix inside + dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system: + let + pkgs = nixpkgsFor.${system}; + image = import ./docker.nix { inherit pkgs; tag = version; }; + in pkgs.runCommand "docker-image-tarball-${version}" + { meta.description = "Docker image with Nix for ${system}"; + } + '' + mkdir -p $out/nix-support + image=$out/image.tar.gz + ln -s ${image} $image + echo "file binary-dist $image" >> $out/nix-support/hydra-build-products + ''); + # Line coverage analysis. coverage = with nixpkgsFor.x86_64-linux; @@ -489,12 +527,7 @@ ''; */ - }; - - checks = forAllSystems (system: { - binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; - installTests = + installTests = forAllSystems (system: let pkgs = nixpkgsFor.${system}; in pkgs.runCommand "install-tests" { againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; @@ -506,8 +539,17 @@ # Disabled because the latest stable version doesn't handle # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"; - }); + } "touch $out"); + + }; + + checks = forAllSystems (system: { + binaryTarball = self.hydraJobs.binaryTarball.${system}; + perlBindings = self.hydraJobs.perlBindings.${system}; + installTests = self.hydraJobs.installTests.${system}; + } // (if system == "x86_64-linux" then { + dockerImage = self.hydraJobs.dockerImage.${system}; + } else {})); packages = forAllSystems (system: { inherit (nixpkgsFor.${system}) nix; @@ -591,15 +633,22 @@ doInstallCheck = true; installCheckFlags = "sysconfdir=$(out)/etc"; }; - }) crossSystems))); + }) crossSystems)) // (builtins.listToAttrs (map (stdenvName: + nixpkgsFor.${system}.lib.nameValuePair + "nix-${stdenvName}" + nixpkgsFor.${system}."${stdenvName}Packages".nix + ) stdenvs)) + ); defaultPackage = forAllSystems (system: self.packages.${system}.nix); - devShell = forAllSystems (system: + devShell = forAllSystems (system: self.devShells.${system}.stdenvPackages); + + devShells = forAllSystemsAndStdenvs (system: stdenv: with nixpkgsFor.${system}; with commonDeps pkgs; - stdenv.mkDerivation { + nixpkgsFor.${system}.${stdenv}.mkDerivation { name = "nix"; outputs = [ "out" "dev" "doc" ]; diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index f2d6896db..18ab33424 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -12,7 +12,6 @@ use LWP::UserAgent; use Net::Amazon::S3; my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; -my $nixRev = $ARGV[1] or die; # FIXME my $releasesBucketName = "nix-releases"; my $channelsBucketName = "nix-channels"; @@ -36,10 +35,11 @@ sub fetch { } my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; -#my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); +my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); - -#my $nixRev = $evalInfo->{jobsetevalinputs}->{nix}->{revision} or die; +my $flakeUrl = $evalInfo->{flake} or die; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); +my $nixRev = $flakeInfo->{revision} or die; my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); #print Dumper($buildInfo); @@ -48,7 +48,7 @@ my $releaseName = $buildInfo->{nixname}; $releaseName =~ /nix-(.*)$/ or die; my $version = $1; -print STDERR "Nix revision is $nixRev, version is $version\n"; +print STDERR "Flake URL is $flakeUrl, Nix revision is $nixRev, version is $version\n"; my $releaseDir = "nix/$releaseName"; @@ -143,12 +143,7 @@ if ($isLatest) { sub getStorePath { my ($jobName) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - for my $product (values %{$buildInfo->{buildproducts}}) { - next unless $product->{type} eq "nix-build"; - next if $product->{path} =~ /[a-z]+$/; - return $product->{path}; - } - die; + return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'"; } write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index f1b439840..da1970f69 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -25,5 +25,10 @@ <string>/var/log/nix-daemon.log</string> <key>StandardOutPath</key> <string>/dev/null</string> + <key>SoftResourceLimits</key> + <dict> + <key>NumberOfFiles</key> + <integer>4096</integer> + </dict> </dict> </plist> diff --git a/perl/configure.ac b/perl/configure.ac index eb65ac17b..a02cb06c9 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -41,7 +41,7 @@ perlarchname=$($perl -e 'use Config; print $Config{archname};') AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) AC_MSG_RESULT($perllibdir) -# Look for libsodium, an optional dependency. +# Look for libsodium. PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Check for the required Perl dependencies (DBI and DBD::SQLite). diff --git a/perl/lib/Nix/Config.pm.in b/perl/lib/Nix/Config.pm.in index f7c6f2484..508a15e15 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/perl/lib/Nix/Config.pm.in @@ -1,6 +1,7 @@ package Nix::Config; use MIME::Base64; +use Nix::Store; $version = "@PACKAGE_VERSION@"; diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh new file mode 100644 index 000000000..c1d2d7c40 --- /dev/null +++ b/scripts/check-hydra-status.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -euo pipefail +# set -x + + +# mapfile BUILDS_FOR_LATEST_EVAL < <( +# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ +# jq -r '.evals[0].builds[] | @sh') +BUILDS_FOR_LATEST_EVAL=$( +curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ + jq -r '.evals[0].builds[]') + +someBuildFailed=0 + +for buildId in $BUILDS_FOR_LATEST_EVAL; do + buildInfo=$(curl -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") + + buildStatus=$(echo "$buildInfo" | \ + jq -r '.buildstatus') + + if [[ "$buildStatus" -ne 0 ]]; then + someBuildFailed=1 + echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra" + fi +done + +exit "$someBuildFailed" diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index b52232dd3..bd8a7ee3a 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -440,7 +440,22 @@ add_nix_vol_fstab_line() { # shellcheck disable=SC1003,SC2026 local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}" shift - EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" <<EOF + + # wrap `ex` to work around a problem with vim plugins breaking exit codes; + # (see https://github.com/NixOS/nix/issues/5468) + # we'd prefer EDITOR="/usr/bin/ex --noplugin" but vifs doesn't word-split + # the EDITOR env. + # + # TODO: at some point we should switch to `--clean`, but it wasn't added + # until https://github.com/vim/vim/releases/tag/v8.0.1554 while the macOS + # minver 10.12.6 seems to have released with vim 7.4 + cat > "$SCRATCH/ex_cleanroom_wrapper" <<EOF +#!/bin/sh +/usr/bin/ex --noplugin "\$@" +EOF + chmod 755 "$SCRATCH/ex_cleanroom_wrapper" + + EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" <<EOF :a UUID=$uuid $escaped_mountpoint apfs rw,noauto,nobrowse,suid,owners . @@ -631,7 +646,7 @@ EOF # technically /etc/synthetic.d/nix is supported in Big Sur+ # but handling both takes even more code... _sudo "to add Nix to /etc/synthetic.conf" \ - /usr/bin/ex /etc/synthetic.conf <<EOF + /usr/bin/ex --noplugin /etc/synthetic.conf <<EOF :a ${NIX_ROOT:1} . @@ -742,6 +757,9 @@ setup_volume() { use_special="${NIX_VOLUME_USE_SPECIAL:-$(create_volume)}" + _sudo "to ensure the Nix volume is not mounted" \ + /usr/sbin/diskutil unmount force "$use_special" || true # might not be mounted + use_uuid=${NIX_VOLUME_USE_UUID:-$(volume_uuid_from_special "$use_special")} setup_fstab "$use_uuid" @@ -791,7 +809,7 @@ setup_volume_daemon() { local volume_uuid="$2" if ! test_voldaemon; then task "Configuring LaunchDaemon to mount '$NIX_VOLUME_LABEL'" >&2 - _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" <<EOF + _sudo "to install the Nix volume mounter" /usr/bin/ex --noplugin "$NIX_VOLUME_MOUNTD_DEST" <<EOF :a $(generate_mount_daemon "$cmd_type" "$volume_uuid") . diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 96eba8310..afaa6783b 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -218,7 +218,7 @@ EOF setup_darwin_volume fi - if [ "$(diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then - failure "This script needs a /nix volume with global permissions! This may require running sudo diskutil enableOwnership /nix." + if [ "$(/usr/sbin/diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "<false/>" ]; then + failure "This script needs a /nix volume with global permissions! This may require running sudo /usr/sbin/diskutil enableOwnership /nix." fi } diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 513127a62..3a24296ee 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -377,6 +377,11 @@ cure_artifacts() { } validate_starting_assumptions() { + task "Checking for artifacts of previous installs" + cat <<EOF +Before I try to install, I'll check for signs Nix already is or has +been installed on this system. +EOF if type nix-env 2> /dev/null >&2; then warning <<EOF Nix already appears to be installed. This installer may run into issues. @@ -386,20 +391,34 @@ $(uninstall_directions) EOF fi + # TODO: I think it would be good for this step to accumulate more + # knowledge of older obsolete artifacts, if there are any. + # We could issue a "reminder" here that the user might want + # to clean them up? + for profile_target in "${PROFILE_TARGETS[@]}"; do + # TODO: I think it would be good to accumulate a list of all + # of the copies so that people don't hit this 2 or 3x in + # a row for different files. if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then + # this backup process first released in Nix 2.1 failure <<EOF -When this script runs, it backs up the current $profile_target to -$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though. +I back up shell profile/rc scripts before I add Nix to them. +I need to back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX, +but the latter already exists. + +Here's how to clean up the old backup file: -Please follow these instructions to clean up the old backup file: +1. Back up (copy) $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX + to another location, just in case. -1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just -in case. +2. Ensure $profile_target$PROFILE_BACKUP_SUFFIX does not have anything + Nix-related in it. If it does, something is probably quite + wrong. Please open an issue or get in touch immediately. -2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like -it has anything nix-related in it. If it does, something is probably -quite wrong. Please open an issue or get in touch immediately. +3. Once you confirm $profile_target is backed up and + $profile_target$PROFILE_BACKUP_SUFFIX doesn't mention Nix, run: + mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target EOF fi done @@ -599,7 +618,7 @@ manager. This will happen in a few stages: 1. Make sure your computer doesn't already have Nix. If it does, I will show you instructions on how to clean up your old install. -2. Show you what we are going to install and where. Then we will ask +2. Show you what I am going to install and where. Then I will ask if you are ready to continue. 3. Create the system users and groups that the Nix daemon uses to run @@ -614,14 +633,14 @@ manager. This will happen in a few stages: EOF - if ui_confirm "Would you like to see a more detailed list of what we will do?"; then + if ui_confirm "Would you like to see a more detailed list of what I will do?"; then cat <<EOF -We will: +I will: - make sure your computer doesn't already have Nix files (if it does, I will tell you how to clean them up.) - - create local users (see the list above for the users we'll make) + - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - install Nix in to $NIX_ROOT - create a configuration file in /etc/nix @@ -656,7 +675,7 @@ run in a headless fashion, like this: $ curl -L https://nixos.org/nix/install | sh -or maybe in a CI pipeline. Because of that, we're going to skip the +or maybe in a CI pipeline. Because of that, I'm going to skip the verbose output in the interest of brevity. If you would like to @@ -670,7 +689,7 @@ EOF fi cat <<EOF -This script is going to call sudo a lot. Every time we do, it'll +This script is going to call sudo a lot. Every time I do, it'll output exactly what it'll do, and why. Just like this: @@ -682,15 +701,15 @@ EOF cat <<EOF This might look scary, but everything can be undone by running just a -few commands. We used to ask you to confirm each time sudo ran, but it +few commands. I used to ask you to confirm each time sudo ran, but it was too many times. Instead, I'll just ask you this one time: EOF - if ui_confirm "Can we use sudo?"; then + if ui_confirm "Can I use sudo?"; then ok "Yay! Thanks! Let's get going!" else failure <<EOF -That is okay, but we can't install. +That is okay, but I can't install. EOF fi } @@ -809,10 +828,10 @@ main() { # can fail faster in this case. Sourcing install-darwin... now runs # `touch /` to detect Read-only root, but it could update times on # pre-Catalina macOS if run as root user. - if [ $EUID -eq 0 ]; then + if [ "$EUID" -eq 0 ]; then failure <<EOF -Please do not run this script with root privileges. We will call sudo -when we need to. +Please do not run this script with root privileges. I will call sudo +when I need to. EOF fi diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index 28d884c79..d543b4463 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -38,7 +38,7 @@ fi # Determine if we could use the multi-user installer or not if [ "$(uname -s)" = "Linux" ]; then - echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 + echo "Note: a multi-user installation is possible. See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" >&2 fi case "$(uname -s)" in @@ -98,7 +98,7 @@ while [ $# -gt 0 ]; do echo " providing multi-user support and better isolation for local builds." echo " Both for security and reproducibility, this method is recommended if" echo " supported on your platform." - echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" + echo " See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" echo "" echo " --no-daemon: Simple, single-user installation that does not require root and is" echo " trivial to uninstall." @@ -134,7 +134,7 @@ fi echo "performing a single-user installation of Nix..." >&2 -if ! [ -e $dest ]; then +if ! [ -e "$dest" ]; then cmd="mkdir -m 0755 $dest && chown $USER $dest" echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 if ! sudo sh -c "$cmd"; then @@ -143,12 +143,12 @@ if ! [ -e $dest ]; then fi fi -if ! [ -w $dest ]; then - echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 +if ! [ -w "$dest" ]; then + echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/manual/nix/stable/installation/multi-user.html. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 exit 1 fi -mkdir -p $dest/store +mkdir -p "$dest/store" printf "copying Nix to %s..." "${dest}/store" >&2 # Insert a newline if no progress is shown. @@ -189,17 +189,17 @@ fi # Install an SSL certificate bundle. if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then - $nix/bin/nix-env -i "$cacert" + "$nix/bin/nix-env" -i "$cacert" export NIX_SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt" fi # Subscribe the user to the Nixpkgs channel and fetch it. if [ -z "$NIX_INSTALLER_NO_CHANNEL_ADD" ]; then - if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then - $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable + if ! "$nix/bin/nix-channel" --list | grep -q "^nixpkgs "; then + "$nix/bin/nix-channel" --add https://nixos.org/channels/nixpkgs-unstable fi if [ -z "$_NIX_INSTALLER_TEST" ]; then - if ! $nix/bin/nix-channel --update nixpkgs; then + if ! "$nix/bin/nix-channel" --update nixpkgs; then echo "Fetching the nixpkgs channel failed. (Are you offline?)" echo "To try again later, run \"nix-channel --update nixpkgs\"." fi @@ -215,7 +215,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then if [ -w "$fn" ]; then if ! grep -q "$p" "$fn"; then echo "modifying $fn..." >&2 - echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn" fi added=1 break @@ -226,7 +226,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then if [ -w "$fn" ]; then if ! grep -q "$p" "$fn"; then echo "modifying $fn..." >&2 - echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + printf '\nif [ -e %s ]; then . %s; fi # added by Nix installer\n' "$p" "$p" >> "$fn" fi added=1 break diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index 81c61b2a0..f4a2dfc5d 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -15,7 +15,7 @@ readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf create_systemd_override() { header "Configuring proxy for the nix-daemon service" - _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)" + _sudo "create directory for systemd unit override" mkdir -p "$(dirname "$SERVICE_OVERRIDE")" cat <<EOF | _sudo "create systemd unit override" tee "$SERVICE_OVERRIDE" [Service] $1 diff --git a/scripts/install.in b/scripts/install.in index 5be4f9dda..38d1fb36f 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -81,10 +81,10 @@ if [ "$(uname -s)" != "Darwin" ]; then require_util xz "unpack the binary tarball" fi -if command -v wget > /dev/null 2>&1; then - fetch() { wget "$1" -O "$2"; } -elif command -v curl > /dev/null 2>&1; then +if command -v curl > /dev/null 2>&1; then fetch() { curl -L "$1" -o "$2"; } +elif command -v wget > /dev/null 2>&1; then + fetch() { wget "$1" -O "$2"; } else oops "you don't have wget or curl installed, which I need to download the binary tarball" fi diff --git a/scripts/local.mk b/scripts/local.mk index 2a0055852..b8477178e 100644 --- a/scripts/local.mk +++ b/scripts/local.mk @@ -1,7 +1,5 @@ nix_noinst_scripts := \ - $(d)/nix-http-export.cgi \ - $(d)/nix-profile.sh \ - $(d)/nix-reduce-build + $(d)/nix-profile.sh noinst-scripts += $(nix_noinst_scripts) diff --git a/scripts/nix-http-export.cgi.in b/scripts/nix-http-export.cgi.in deleted file mode 100755 index 19a505af1..000000000 --- a/scripts/nix-http-export.cgi.in +++ /dev/null @@ -1,51 +0,0 @@ -#! /bin/sh - -export HOME=/tmp -export NIX_REMOTE=daemon - -TMP_DIR="${TMP_DIR:-/tmp/nix-export}" - -@coreutils@/mkdir -p "$TMP_DIR" || true -@coreutils@/chmod a+r "$TMP_DIR" - -needed_path="?$QUERY_STRING" -needed_path="${needed_path#*[?&]needed_path=}" -needed_path="${needed_path%%&*}" -#needed_path="$(echo $needed_path | ./unhttp)" -needed_path="${needed_path//%2B/+}" -needed_path="${needed_path//%3D/=}" - -echo needed_path: "$needed_path" >&2 - -NIX_STORE="${NIX_STORE_DIR:-/nix/store}" - -echo NIX_STORE: "${NIX_STORE}" >&2 - -full_path="${NIX_STORE}"/"$needed_path" - -if [ "$needed_path" != "${needed_path%.drv}" ]; then - echo "Status: 403 You should create the derivation file yourself" - echo "Content-Type: text/plain" - echo - echo "Refusing to disclose derivation contents" - exit -fi - -if @bindir@/nix-store --check-validity "$full_path"; then - if ! [ -e nix-export/"$needed_path".nar.gz ]; then - @bindir@/nix-store --export "$full_path" | @gzip@ > "$TMP_DIR"/"$needed_path".nar.gz - @coreutils@/ln -fs "$TMP_DIR"/"$needed_path".nar.gz nix-export/"$needed_path".nar.gz - fi; - echo "Status: 301 Moved" - echo "Location: nix-export/"$needed_path".nar.gz" - echo -else - echo "Status: 404 No such path found" - echo "Content-Type: text/plain" - echo - echo "Path not found:" - echo "$needed_path" - echo "checked:" - echo "$full_path" -fi - diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 500a98992..0a47571ac 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -5,7 +5,7 @@ __ETC_PROFILE_NIX_SOURCED=1 export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. -if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then +if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then : # Allow users to override the NIX_SSL_CERT_FILE elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt @@ -18,14 +18,14 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS else # Fall back to what is in the nix profiles, favouring whatever is defined last. check_nix_profiles() { - if [ "$ZSH_VERSION" ]; then + if [ -n "$ZSH_VERSION" ]; then # Zsh by default doesn't split words in unquoted parameter expansion. # Set local_options for these options to be reverted at the end of the function # and shwordsplit to force splitting words in $NIX_PROFILES below. setopt local_options shwordsplit fi for i in $NIX_PROFILES; do - if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then + if [ -e "$i/etc/ssl/certs/ca-bundle.crt" ]; then export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt fi done diff --git a/scripts/nix-reduce-build.in b/scripts/nix-reduce-build.in deleted file mode 100755 index 50beb9d10..000000000 --- a/scripts/nix-reduce-build.in +++ /dev/null @@ -1,171 +0,0 @@ -#! @bash@ - -WORKING_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}"/nix-reduce-build-XXXXXX); -cd "$WORKING_DIRECTORY"; - -if test -z "$1" || test "a--help" = "a$1" ; then - echo 'nix-reduce-build (paths or Nix expressions) -- (package sources)' >&2 - echo As in: >&2 - echo nix-reduce-build /etc/nixos/nixos -- ssh://user@somewhere.nowhere.example.org >&2 - echo nix-reduce-build /etc/nixos/nixos -- \\ - echo " " \''http://somewhere.nowhere.example.org/nix/nix-http-export.cgi?needed_path='\' >&2 - echo " store path name will be added into the end of the URL" >&2 - echo nix-reduce-build /etc/nixos/nixos -- file://home/user/nar/ >&2 - echo " that should be a directory where gzipped 'nix-store --export' ">&2 - echo " files are located (they should have .nar.gz extension)" >&2 - echo " Or all together: " >&2 - echo -e nix-reduce-build /expr.nix /e2.nix -- \\\\\\\n\ - " ssh://a@b.example.com http://n.example.com/get-nar?q= file://nar/" >&2 - echo " Also supports best-effort local builds of failing expression set:" >&2 - echo "nix-reduce-build /e.nix -- nix-daemon:// nix-self://" >&2 - echo " nix-daemon:// builds using daemon" - echo " nix-self:// builds directly using nix-store from current installation" >&2 - echo " nix-daemon-fixed:// and nix-self-fixed:// do the same, but only for" >&2; - echo "derivations with specified output hash (sha256, sha1 or md5)." >&2 - echo " nix-daemon-substitute:// and nix-self-substitute:// try to substitute" >&2; - echo "maximum amount of paths" >&2; - echo " nix-daemon-build:// and nix-self-build:// try to build (not substitute)" >&2; - echo "maximum amount of paths" >&2; - echo " If no package sources are specified, required paths are listed." >&2; - exit; -fi; - -while ! test "$1" = "--" || test "$1" = "" ; do - echo "$1" >> initial; >&2 - shift; -done -shift; -echo Will work on $(cat initial | wc -l) targets. >&2 - -while read ; do - case "$REPLY" in - ${NIX_STORE_DIR:-/nix/store}/*) - echo "$REPLY" >> paths; >&2 - ;; - *) - ( - IFS=: ; - nix-instantiate $REPLY >> paths; - ); - ;; - esac; -done < initial; -echo Proceeding $(cat paths | wc -l) paths. >&2 - -while read; do - case "$REPLY" in - *.drv) - echo "$REPLY" >> derivers; >&2 - ;; - *) - nix-store --query --deriver "$REPLY" >>derivers; - ;; - esac; -done < paths; -echo Found $(cat derivers | wc -l) derivers. >&2 - -cat derivers | xargs nix-store --query -R > derivers-closure; -echo Proceeding at most $(cat derivers-closure | wc -l) derivers. >&2 - -cat derivers-closure | egrep '[.]drv$' | xargs nix-store --query --outputs > wanted-paths; -cat derivers-closure | egrep -v '[.]drv$' >> wanted-paths; -echo Prepared $(cat wanted-paths | wc -l) paths to get. >&2 - -cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths; -echo We need $(cat needed-paths | wc -l) paths. >&2 - -egrep '[.]drv$' derivers-closure > critical-derivers; - -if test -z "$1" ; then - cat needed-paths; -fi; - -refresh_critical_derivers() { - echo "Finding needed derivers..." >&2; - cat critical-derivers | while read; do - if ! (nix-store --query --outputs "$REPLY" | xargs nix-store --check-validity &> /dev/null;); then - echo "$REPLY"; - fi; - done > new-critical-derivers; - mv new-critical-derivers critical-derivers; - echo The needed paths are realized by $(cat critical-derivers | wc -l) derivers. >&2 -} - -build_here() { - cat critical-derivers | while read; do - echo "Realising $REPLY using nix-daemon" >&2 - @bindir@/nix-store -r "${REPLY}" - done; -} - -try_to_substitute(){ - cat needed-paths | while read ; do - echo "Building $REPLY using nix-daemon" >&2 - @bindir@/nix-store -r "${NIX_STORE_DIR:-/nix/store}/${REPLY##*/}" - done; -} - -for i in "$@"; do - sshHost="${i#ssh://}"; - httpHost="${i#http://}"; - httpsHost="${i#https://}"; - filePath="${i#file:/}"; - if [ "$i" != "$sshHost" ]; then - cat needed-paths | while read; do - echo "Getting $REPLY and its closure over ssh" >&2 - nix-copy-closure --from "$sshHost" --gzip "$REPLY" </dev/null || true; - done; - elif [ "$i" != "$httpHost" ] || [ "$i" != "$httpsHost" ]; then - cat needed-paths | while read; do - echo "Getting $REPLY over http/https" >&2 - curl ${BAD_CERTIFICATE:+-k} -L "$i${REPLY##*/}" | gunzip | nix-store --import; - done; - elif [ "$i" != "$filePath" ] ; then - cat needed-paths | while read; do - echo "Installing $REPLY from file" >&2 - gunzip < "$filePath/${REPLY##*/}".nar.gz | nix-store --import; - done; - elif [ "$i" = "nix-daemon://" ] ; then - NIX_REMOTE=daemon try_to_substitute; - refresh_critical_derivers; - NIX_REMOTE=daemon build_here; - elif [ "$i" = "nix-self://" ] ; then - NIX_REMOTE= try_to_substitute; - refresh_critical_derivers; - NIX_REMOTE= build_here; - elif [ "$i" = "nix-daemon-fixed://" ] ; then - refresh_critical_derivers; - - cat critical-derivers | while read; do - if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then - echo "Realising $REPLY using nix-daemon" >&2 - NIX_REMOTE=daemon @bindir@/nix-store -r "${REPLY}" - fi; - done; - elif [ "$i" = "nix-self-fixed://" ] ; then - refresh_critical_derivers; - - cat critical-derivers | while read; do - if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then - echo "Realising $REPLY using direct Nix build" >&2 - NIX_REMOTE= @bindir@/nix-store -r "${REPLY}" - fi; - done; - elif [ "$i" = "nix-daemon-substitute://" ] ; then - NIX_REMOTE=daemon try_to_substitute; - elif [ "$i" = "nix-self-substitute://" ] ; then - NIX_REMOTE= try_to_substitute; - elif [ "$i" = "nix-daemon-build://" ] ; then - refresh_critical_derivers; - NIX_REMOTE=daemon build_here; - elif [ "$i" = "nix-self-build://" ] ; then - refresh_critical_derivers; - NIX_REMOTE= build_here; - fi; - mv needed-paths wanted-paths; - cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths; - echo We still need $(cat needed-paths | wc -l) paths. >&2 -done; - -cd / -rm -r "$WORKING_DIRECTORY" diff --git a/scripts/prepare-installer-for-github-actions b/scripts/prepare-installer-for-github-actions index 92d930384..4b994a753 100755 --- a/scripts/prepare-installer-for-github-actions +++ b/scripts/prepare-installer-for-github-actions @@ -3,7 +3,7 @@ set -e script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link) -installerHash=$(echo $script | cut -b12-43 -) +installerHash=$(echo "$script" | cut -b12-43 -) installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index fd3edfc46..429cd32cc 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -73,8 +73,13 @@ ref<Store> EvalCommand::getEvalStore() ref<EvalState> EvalCommand::getEvalState() { - if (!evalState) - evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore()); + if (!evalState) evalState = +#if HAVE_BOEHMGC + std::allocate_shared<EvalState>(traceable_allocator<EvalState>(), +#else + std::make_shared<EvalState>( +#endif + searchPath, getEvalStore(), getStore()); return ref<EvalState>(evalState); } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5758b52ad..ef200b1d2 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -291,6 +291,9 @@ void completeFlakeRefWithFragment( void completeFlakeRef(ref<Store> store, std::string_view prefix) { + if (!settings.isExperimentalFeatureEnabled(Xp::Flakes)) + return; + if (prefix == "") completions->add("."); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index db1e7e56d..b987e1888 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -119,8 +119,8 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu case tList2: case tListN: str << "[ "; - for (unsigned int n = 0; n < v.listSize(); ++n) { - printValue(str, active, *v.listElems()[n]); + for (auto v2 : v.listItems()) { + printValue(str, active, *v2); str << " "; } str << "]"; @@ -519,8 +519,12 @@ Path EvalState::checkSourcePath(const Path & path_) } } - if (!found) - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath); + if (!found) { + auto modeInformation = evalSettings.pureEval + ? "in pure eval mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation); + } /* Resolve symlinks. */ debug(format("checking access to '%s'") % abspath); @@ -533,7 +537,7 @@ Path EvalState::checkSourcePath(const Path & path_) } } - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); + throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path); } @@ -583,14 +587,20 @@ Value * EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v2; - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + addConstant(name, v2); return v2; } +void EvalState::addConstant(const string & name, Value * v) +{ + staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); + baseEnv.values[baseEnvDispl++] = v; + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); +} + + Value * EvalState::addPrimOp(const string & name, size_t arity, PrimOpFun primOp) { @@ -609,7 +619,7 @@ Value * EvalState::addPrimOp(const string & name, Value * v = allocValue(); v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym }); - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); return v; @@ -635,7 +645,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) Value * v = allocValue(); v->mkPrimOp(new PrimOp(std::move(primOp))); - staticBaseEnv.vars[envName] = baseEnvDispl; + staticBaseEnv.vars.emplace_back(envName, baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); return v; @@ -785,7 +795,7 @@ void mkPath(Value & v, const char * s) inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) { - for (size_t l = var.level; l; --l, env = env->up) ; + for (auto l = var.level; l; --l, env = env->up) ; if (!var.fromWith) return env->values[var.displ]; @@ -1058,7 +1068,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - size_t displ = 0; + Displacement displ = 0; for (auto & i : attrs) { Value * vAttr; if (hasOverrides && !i.second.inherited) { @@ -1134,7 +1144,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - size_t displ = 0; + Displacement displ = 0; for (auto & i : attrs->attrs) env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); @@ -1145,8 +1155,8 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) void ExprList::eval(EvalState & state, Env & env, Value & v) { state.mkList(v, elems.size()); - for (size_t n = 0; n < elems.size(); ++n) - v.listElems()[n] = elems[n]->maybeThunk(state, env); + for (auto [n, v2] : enumerate(v.listItems())) + const_cast<Value * &>(v2) = elems[n]->maybeThunk(state, env); } @@ -1251,144 +1261,184 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) } -void ExprApp::eval(EvalState & state, Env & env, Value & v) +void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos) { - /* FIXME: vFun prevents GCC from doing tail call optimisation. */ - Value vFun; - e1->eval(state, env, vFun); - state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos); -} + auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr; + forceValue(fun, pos); -void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) -{ - /* Figure out the number of arguments still needed. */ - size_t argsDone = 0; - Value * primOp = &fun; - while (primOp->isPrimOpApp()) { - argsDone++; - primOp = primOp->primOpApp.left; - } - assert(primOp->isPrimOp()); - auto arity = primOp->primOp->arity; - auto argsLeft = arity - argsDone; - - if (argsLeft == 1) { - /* We have all the arguments, so call the primop. */ - - /* Put all the arguments in an array. */ - Value * vArgs[arity]; - auto n = arity - 1; - vArgs[n--] = &arg; - for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left) - vArgs[n--] = arg->primOpApp.right; - - /* And call the primop. */ - nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, pos, vArgs, v); - } else { - Value * fun2 = allocValue(); - *fun2 = fun; - v.mkPrimOpApp(fun2, &arg); - } -} + Value vCur(fun); -void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) -{ - auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr; + auto makeAppChain = [&]() + { + vRes = vCur; + for (size_t i = 0; i < nrArgs; ++i) { + auto fun2 = allocValue(); + *fun2 = vRes; + vRes.mkPrimOpApp(fun2, args[i]); + } + }; - forceValue(fun, pos); + Attr * functor; - if (fun.isPrimOp() || fun.isPrimOpApp()) { - callPrimOp(fun, arg, v, pos); - return; - } + while (nrArgs > 0) { - if (fun.type() == nAttrs) { - auto found = fun.attrs->find(sFunctor); - if (found != fun.attrs->end()) { - /* fun may be allocated on the stack of the calling function, - * but for functors we may keep a reference, so heap-allocate - * a copy and use that instead. - */ - auto & fun2 = *allocValue(); - fun2 = fun; - /* !!! Should we use the attr pos here? */ - Value v2; - callFunction(*found->value, fun2, v2, pos); - return callFunction(v2, arg, v, pos); - } - } + if (vCur.isLambda()) { - if (!fun.isLambda()) - throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); + ExprLambda & lambda(*vCur.lambda.fun); - ExprLambda & lambda(*fun.lambda.fun); + auto size = + (lambda.arg.empty() ? 0 : 1) + + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); + Env & env2(allocEnv(size)); + env2.up = vCur.lambda.env; - auto size = - (lambda.arg.empty() ? 0 : 1) + - (lambda.hasFormals() ? lambda.formals->formals.size() : 0); - Env & env2(allocEnv(size)); - env2.up = fun.lambda.env; + Displacement displ = 0; - size_t displ = 0; + if (!lambda.hasFormals()) + env2.values[displ++] = args[0]; - if (!lambda.hasFormals()) - env2.values[displ++] = &arg; + else { + forceAttrs(*args[0], pos); - else { - forceAttrs(arg, pos); - - if (!lambda.arg.empty()) - env2.values[displ++] = &arg; - - /* For each formal argument, get the actual argument. If - there is no matching actual argument but the formal - argument has a default, use the default. */ - size_t attrsUsed = 0; - for (auto & i : lambda.formals->formals) { - Bindings::iterator j = arg.attrs->find(i.name); - if (j == arg.attrs->end()) { - if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", - lambda, i.name); - env2.values[displ++] = i.def->maybeThunk(*this, env2); + if (!lambda.arg.empty()) + env2.values[displ++] = args[0]; + + /* For each formal argument, get the actual argument. If + there is no matching actual argument but the formal + argument has a default, use the default. */ + size_t attrsUsed = 0; + for (auto & i : lambda.formals->formals) { + auto j = args[0]->attrs->get(i.name); + if (!j) { + if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", + lambda, i.name); + env2.values[displ++] = i.def->maybeThunk(*this, env2); + } else { + attrsUsed++; + env2.values[displ++] = j->value; + } + } + + /* Check that each actual argument is listed as a formal + argument (unless the attribute match specifies a `...'). */ + if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) { + /* Nope, so show the first unexpected argument to the + user. */ + for (auto & i : *args[0]->attrs) + if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) + throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); + abort(); // can't happen + } + } + + nrFunctionCalls++; + if (countCalls) incrFunctionCall(&lambda); + + /* Evaluate the body. */ + try { + lambda.body->eval(*this, env2, vCur); + } catch (Error & e) { + if (loggerSettings.showTrace.get()) { + addErrorTrace(e, lambda.pos, "while evaluating %s", + (lambda.name.set() + ? "'" + (string) lambda.name + "'" + : "anonymous lambda")); + addErrorTrace(e, pos, "from call site%s", ""); + } + throw; + } + + nrArgs--; + args += 1; + } + + else if (vCur.isPrimOp()) { + + size_t argsLeft = vCur.primOp->arity; + + if (nrArgs < argsLeft) { + /* We don't have enough arguments, so create a tPrimOpApp chain. */ + makeAppChain(); + return; } else { - attrsUsed++; - env2.values[displ++] = j->value; + /* We have all the arguments, so call the primop. */ + nrPrimOpCalls++; + if (countCalls) primOpCalls[vCur.primOp->name]++; + vCur.primOp->fun(*this, pos, args, vCur); + + nrArgs -= argsLeft; + args += argsLeft; } } - /* Check that each actual argument is listed as a formal - argument (unless the attribute match specifies a `...'). */ - if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { - /* Nope, so show the first unexpected argument to the - user. */ - for (auto & i : *arg.attrs) - if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) - throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); - abort(); // can't happen + else if (vCur.isPrimOpApp()) { + /* Figure out the number of arguments still needed. */ + size_t argsDone = 0; + Value * primOp = &vCur; + while (primOp->isPrimOpApp()) { + argsDone++; + primOp = primOp->primOpApp.left; + } + assert(primOp->isPrimOp()); + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; + + if (nrArgs < argsLeft) { + /* We still don't have enough arguments, so extend the tPrimOpApp chain. */ + makeAppChain(); + return; + } else { + /* We have all the arguments, so call the primop with + the previous and new arguments. */ + + Value * vArgs[arity]; + auto n = argsDone; + for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) + vArgs[--n] = arg->primOpApp.right; + + for (size_t i = 0; i < argsLeft; ++i) + vArgs[argsDone + i] = args[i]; + + nrPrimOpCalls++; + if (countCalls) primOpCalls[primOp->primOp->name]++; + primOp->primOp->fun(*this, pos, vArgs, vCur); + + nrArgs -= argsLeft; + args += argsLeft; + } + } + + else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) { + /* 'vCur' may be allocated on the stack of the calling + function, but for functors we may keep a reference, so + heap-allocate a copy and use that instead. */ + Value * args2[] = {allocValue(), args[0]}; + *args2[0] = vCur; + /* !!! Should we use the attr pos here? */ + callFunction(*functor->value, 2, args2, vCur, pos); + nrArgs--; + args++; } + + else + throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur); } - nrFunctionCalls++; - if (countCalls) incrFunctionCall(&lambda); + vRes = vCur; +} - /* Evaluate the body. This is conditional on showTrace, because - catching exceptions makes this function not tail-recursive. */ - if (loggerSettings.showTrace.get()) - try { - lambda.body->eval(*this, env2, v); - } catch (Error & e) { - addErrorTrace(e, lambda.pos, "while evaluating %s", - (lambda.name.set() - ? "'" + (string) lambda.name + "'" - : "anonymous lambda")); - addErrorTrace(e, pos, "from call site%s", ""); - throw; - } - else - fun.lambda.fun->body->eval(*this, env2, v); + +void ExprCall::eval(EvalState & state, Env & env, Value & v) +{ + Value vFun; + fun->eval(state, env, vFun); + + Value * vArgs[args.size()]; + for (size_t i = 0; i < args.size(); ++i) + vArgs[i] = args[i]->maybeThunk(state, env); + + state.callFunction(vFun, args.size(), vArgs, v, pos); } @@ -1686,8 +1736,8 @@ void EvalState::forceValueDeep(Value & v) } else if (v.isList()) { - for (size_t n = 0; n < v.listSize(); ++n) - recurse(*v.listElems()[n]); + for (auto v2 : v.listItems()) + recurse(*v2); } }; @@ -1871,12 +1921,12 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, if (v.isList()) { string result; - for (size_t n = 0; n < v.listSize(); ++n) { - result += coerceToString(pos, *v.listElems()[n], + for (auto [n, v2] : enumerate(v.listItems())) { + result += coerceToString(pos, *v2, context, coerceMore, copyToStore); if (n < v.listSize() - 1 /* !!! not quite correct */ - && (!v.listElems()[n]->isList() || v.listElems()[n]->listSize() != 0)) + && (!v2->isList() || v2->listSize() != 0)) result += " "; } return result; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 69119599a..1aab8e166 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -277,6 +277,8 @@ private: Value * addConstant(const string & name, Value & v); + void addConstant(const string & name, Value * v); + Value * addPrimOp(const string & name, size_t arity, PrimOpFun primOp); @@ -316,8 +318,14 @@ public: bool isFunctor(Value & fun); - void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos); - void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos); + // FIXME: use std::span + void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos); + + void callFunction(Value & fun, Value & arg, Value & vRes, const Pos & pos) + { + Value * args[] = {&arg}; + callFunction(fun, 1, args, vRes, pos); + } /* Automatically call a function for which each argument has a default value or has a binding in the `args' map. */ diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 41b6f78ed..c03f4106c 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,4 +1,5 @@ #include "flake.hh" +#include "globals.hh" #include <nlohmann/json.hpp> @@ -52,21 +53,19 @@ void ConfigFile::apply() auto trustedList = readTrustedList(); bool trusted = false; - - if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) { + if (nix::settings.acceptFlakeConfig){ + trusted = true; + } else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) { trusted = *saved; + warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS); } else { // FIXME: filter ANSI escapes, newlines, \r, etc. - if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { - if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = false; - writeTrustedList(trustedList); - } - } else { - if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = trusted = true; - writeTrustedList(trustedList); - } + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') { + trusted = true; + } + if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?", trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') { + trustedList[name][valueS] = trusted; + writeTrustedList(trustedList); } } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index c9d848495..33d253eee 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -257,8 +257,7 @@ static Flake getFlake( flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); else if (setting.value->type() == nList) { std::vector<std::string> ss; - for (unsigned int n = 0; n < setting.value->listSize(); ++n) { - auto elem = setting.value->listElems()[n]; + for (auto elem : setting.value->listItems()) { if (elem->type() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", setting.name, showType(*setting.value)); @@ -307,7 +306,7 @@ LockedFlake lockFlake( if (lockFlags.applyNixConfig) { flake.config.apply(); - // FIXME: send new config to the daemon. + state.store->setOptions(); } try { @@ -446,22 +445,18 @@ LockedFlake lockFlake( update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); - auto hasChildUpdate = + auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->size() > inputPath.size() && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); - if (hasChildUpdate) { - auto inputFlake = getFlake( - state, oldLock->lockedRef, false, flakeCache); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, parent, parentPath); - } else { + FlakeInputs fakeInputs; + + if (!mustRefetch) { /* No need to fetch this flake, we can be lazy. However there may be new overrides on the inputs of this flake, so we need to check those. */ - FlakeInputs fakeInputs; - for (auto & i : oldLock->inputs) { if (auto lockedNode = std::get_if<0>(&i.second)) { fakeInputs.emplace(i.first, FlakeInput { @@ -469,15 +464,33 @@ LockedFlake lockFlake( .isFlake = (*lockedNode)->isFlake, }); } else if (auto follows = std::get_if<1>(&i.second)) { + auto o = input.overrides.find(i.first); + // If the override disappeared, we have to refetch the flake, + // since some of the inputs may not be present in the lockfile. + if (o == input.overrides.end()) { + mustRefetch = true; + // There's no point populating the rest of the fake inputs, + // since we'll refetch the flake anyways. + break; + } fakeInputs.emplace(i.first, FlakeInput { .follows = *follows, }); } } - - computeLocks(fakeInputs, childNode, inputPath, oldLock, parent, parentPath); } + LockParent newParent { + .path = inputPath, + .absolute = false + }; + + computeLocks( + mustRefetch + ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs + : fakeInputs, + childNode, inputPath, oldLock, newParent, parentPath); + } else { /* We need to create a new lock file entry. So fetch this input. */ diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index f774e6493..ed4c47fbb 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -102,9 +102,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) state->forceList(*i->value, *i->pos); /* For each output... */ - for (unsigned int j = 0; j < i->value->listSize(); ++j) { + for (auto elem : i->value->listItems()) { /* Evaluate the corresponding set. */ - string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos); + string name = state->forceStringNoCtx(*elem, *i->pos); Bindings::iterator out = attrs->find(state->symbols.create(name)); if (out == attrs->end()) continue; // FIXME: throw error? state->forceAttrs(*out->value); @@ -128,9 +128,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) throw errMsg; Outputs result; - for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->type() != nString) throw errMsg; - auto out = outputs.find((*i)->string.s); + for (auto elem : outTI->listItems()) { + if (elem->type() != nString) throw errMsg; + auto out = outputs.find(elem->string.s); if (out == outputs.end()) throw errMsg; result.insert(*out); } @@ -174,8 +174,8 @@ bool DrvInfo::checkMeta(Value & v) { state->forceValue(v); if (v.type() == nList) { - for (unsigned int n = 0; n < v.listSize(); ++n) - if (!checkMeta(*v.listElems()[n])) return false; + for (auto elem : v.listItems()) + if (!checkMeta(*elem)) return false; return true; } else if (v.type() == nAttrs) { @@ -364,10 +364,10 @@ static void getDerivations(EvalState & state, Value & vIn, } else if (v.type() == nList) { - for (unsigned int n = 0; n < v.listSize(); ++n) { - string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); - if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) - getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + for (auto [n, elem] : enumerate(v.listItems())) { + string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n)); + if (getDerivation(state, *elem, pathPrefix2, drvs, done, ignoreAssertionFailures)) + getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 51593eccd..c18877e29 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -64,6 +64,7 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) } +// FIXME: optimize static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) { string t; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 0d0f3e469..57c2f6e44 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -143,6 +143,16 @@ void ExprLambda::show(std::ostream & str) const str << ": " << *body << ")"; } +void ExprCall::show(std::ostream & str) const +{ + str << '(' << *fun; + for (auto e : args) { + str << ' '; + str << *e; + } + str << ')'; +} + void ExprLet::show(std::ostream & str) const { str << "(let "; @@ -263,13 +273,13 @@ void ExprVar::bindVars(const StaticEnv & env) /* Check whether the variable appears in the environment. If so, set its level and displacement. */ const StaticEnv * curEnv; - unsigned int level; + Level level; int withLevel = -1; for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { if (curEnv->isWith) { if (withLevel == -1) withLevel = level; } else { - StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); + auto i = curEnv->find(name); if (i != curEnv->vars.end()) { fromWith = false; this->level = level; @@ -311,14 +321,16 @@ void ExprOpHasAttr::bindVars(const StaticEnv & env) void ExprAttrs::bindVars(const StaticEnv & env) { const StaticEnv * dynamicEnv = &env; - StaticEnv newEnv(false, &env); + StaticEnv newEnv(false, &env, recursive ? attrs.size() : 0); if (recursive) { dynamicEnv = &newEnv; - unsigned int displ = 0; + Displacement displ = 0; for (auto & i : attrs) - newEnv.vars[i.first] = i.second.displ = displ++; + newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + + // No need to sort newEnv since attrs is in sorted order. for (auto & i : attrs) i.second.e->bindVars(i.second.inherited ? env : newEnv); @@ -342,15 +354,20 @@ void ExprList::bindVars(const StaticEnv & env) void ExprLambda::bindVars(const StaticEnv & env) { - StaticEnv newEnv(false, &env); + StaticEnv newEnv( + false, &env, + (hasFormals() ? formals->formals.size() : 0) + + (arg.empty() ? 0 : 1)); - unsigned int displ = 0; + Displacement displ = 0; - if (!arg.empty()) newEnv.vars[arg] = displ++; + if (!arg.empty()) newEnv.vars.emplace_back(arg, displ++); if (hasFormals()) { for (auto & i : formals->formals) - newEnv.vars[i.name] = displ++; + newEnv.vars.emplace_back(i.name, displ++); + + newEnv.sort(); for (auto & i : formals->formals) if (i.def) i.def->bindVars(newEnv); @@ -359,13 +376,22 @@ void ExprLambda::bindVars(const StaticEnv & env) body->bindVars(newEnv); } +void ExprCall::bindVars(const StaticEnv & env) +{ + fun->bindVars(env); + for (auto e : args) + e->bindVars(env); +} + void ExprLet::bindVars(const StaticEnv & env) { - StaticEnv newEnv(false, &env); + StaticEnv newEnv(false, &env, attrs->attrs.size()); - unsigned int displ = 0; + Displacement displ = 0; for (auto & i : attrs->attrs) - newEnv.vars[i.first] = i.second.displ = displ++; + newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + + // No need to sort newEnv since attrs->attrs is in sorted order. for (auto & i : attrs->attrs) i.second.e->bindVars(i.second.inherited ? env : newEnv); @@ -379,7 +405,7 @@ void ExprWith::bindVars(const StaticEnv & env) level so that `lookupVar' can look up variables in the previous `with' if this one doesn't contain the desired attribute. */ const StaticEnv * curEnv; - unsigned int level; + Level level; prevWith = 0; for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++) if (curEnv->isWith) { @@ -452,5 +478,4 @@ size_t SymbolTable::totalSize() const return n; } - } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 851e875bd..13256272c 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -4,8 +4,6 @@ #include "symbol-table.hh" #include "error.hh" -#include <map> - namespace nix { @@ -135,6 +133,9 @@ struct ExprPath : Expr Value * maybeThunk(EvalState & state, Env & env); }; +typedef uint32_t Level; +typedef uint32_t Displacement; + struct ExprVar : Expr { Pos pos; @@ -150,8 +151,8 @@ struct ExprVar : Expr value is obtained by getting the attribute named `name' from the set stored in the environment that is `level' levels up from the current one.*/ - unsigned int level; - unsigned int displ; + Level level; + Displacement displ; ExprVar(const Symbol & name) : name(name) { }; ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; @@ -185,7 +186,7 @@ struct ExprAttrs : Expr bool inherited; Expr * e; Pos pos; - unsigned int displ; // displacement + Displacement displ; // displacement AttrDef(Expr * e, const Pos & pos, bool inherited=false) : inherited(inherited), e(e), pos(pos) { }; AttrDef() { }; @@ -250,6 +251,17 @@ struct ExprLambda : Expr COMMON_METHODS }; +struct ExprCall : Expr +{ + Expr * fun; + std::vector<Expr *> args; + Pos pos; + ExprCall(const Pos & pos, Expr * fun, std::vector<Expr *> && args) + : fun(fun), args(args), pos(pos) + { } + COMMON_METHODS +}; + struct ExprLet : Expr { ExprAttrs * attrs; @@ -308,7 +320,6 @@ struct ExprOpNot : Expr void eval(EvalState & state, Env & env, Value & v); \ }; -MakeBinOp(ExprApp, "") MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=") MakeBinOp(ExprOpAnd, "&&") @@ -342,9 +353,28 @@ struct StaticEnv { bool isWith; const StaticEnv * up; - typedef std::map<Symbol, unsigned int> Vars; + + // Note: these must be in sorted order. + typedef std::vector<std::pair<Symbol, Displacement>> Vars; Vars vars; - StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { }; + + StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { + vars.reserve(expectedSize); + }; + + void sort() + { + std::sort(vars.begin(), vars.end(), + [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; }); + } + + Vars::const_iterator find(const Symbol & name) const + { + Vars::value_type key(name, 0); + auto i = std::lower_bound(vars.begin(), vars.end(), key); + if (i != vars.end() && i->first == name) return i; + return vars.end(); + } }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 813ff2fc3..c1f4e72e0 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -33,11 +33,9 @@ namespace nix { Symbol file; FileOrigin origin; std::optional<ErrorInfo> error; - Symbol sLetBody; ParseData(EvalState & state) : state(state) , symbols(state.symbols) - , sLetBody(symbols.create("<let-body>")) { }; }; @@ -126,14 +124,14 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, auto j2 = jAttrs->attrs.find(ad.first); if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. dupAttr(ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs[ad.first] = ad.second; + jAttrs->attrs.emplace(ad.first, ad.second); } } else { dupAttr(attrPath, pos, j->second.pos); } } else { // This attr path is not defined. Let's create it. - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); + attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); e->setName(i->symbol); } } else { @@ -283,7 +281,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err } %type <e> start expr expr_function expr_if expr_op -%type <e> expr_app expr_select expr_simple +%type <e> expr_select expr_simple expr_app %type <list> expr_list %type <attrs> binds %type <formals> formals @@ -353,13 +351,13 @@ expr_if expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } - | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); } + | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); } - | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); } + | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } @@ -367,17 +365,22 @@ expr_op | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } | expr_op '+' expr_op { $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); } - | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); } - | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); } - | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); } + | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); } | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } | expr_app ; expr_app - : expr_app expr_select - { $$ = new ExprApp(CUR_POS, $1, $2); } - | expr_select { $$ = $1; } + : expr_app expr_select { + if (auto e2 = dynamic_cast<ExprCall *>($1)) { + e2->args.push_back($2); + $$ = $1; + } else + $$ = new ExprCall(CUR_POS, $1, {$2}); + } + | expr_select ; expr_select @@ -388,7 +391,7 @@ expr_select | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW - { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.create("or"))); } + { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } | expr_simple { $$ = $1; } ; @@ -412,10 +415,10 @@ expr_simple } | SPATH { string path($1 + 1, strlen($1) - 2); - $$ = new ExprApp(CUR_POS, - new ExprApp(new ExprVar(data->symbols.create("__findFile")), - new ExprVar(data->symbols.create("__nixPath"))), - new ExprString(data->symbols.create(path))); + $$ = new ExprCall(CUR_POS, + new ExprVar(data->symbols.create("__findFile")), + {new ExprVar(data->symbols.create("__nixPath")), + new ExprString(data->symbols.create(path))}); } | URI { static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); @@ -483,7 +486,7 @@ binds if ($$->attrs.find(i.symbol) != $$->attrs.end()) dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); Pos pos = makeCurPos(@3, data); - $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } } | binds INHERIT '(' expr ')' attrs ';' @@ -492,7 +495,7 @@ binds for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); - $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); } } | { $$ = new ExprAttrs(makeCurPos(@0, data)); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 6b3cafec8..66af373d7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -70,7 +70,7 @@ void EvalState::realiseContext(const PathSet & context) if (outputPaths.count(outputName) == 0) throw Error("derivation '%s' does not have an output named '%s'", store->printStorePath(drvPath), outputName); - allowedPaths->insert(store->printStorePath(outputPaths.at(outputName))); + allowPath(outputPaths.at(outputName)); } } } @@ -184,14 +184,17 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; - StaticEnv staticEnv(false, &state.staticBaseEnv); + StaticEnv staticEnv(false, &state.staticBaseEnv, vScope->attrs->size()); unsigned int displ = 0; for (auto & attr : *vScope->attrs) { - staticEnv.vars[attr.name] = displ; + staticEnv.vars.emplace_back(attr.name, displ); env->values[displ++] = attr.value; } + // No need to call staticEnv.sort(), because + // args[0]->attrs is already sorted. + printTalkative("evaluating file '%1%'", realPath); Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); @@ -332,9 +335,8 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) PathSet context; auto program = state.coerceToString(pos, *elems[0], context, false, false); Strings commandArgs; - for (unsigned int i = 1; i < args[0]->listSize(); ++i) { + for (unsigned int i = 1; i < args[0]->listSize(); ++i) commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false)); - } try { state.realiseContext(context); } catch (InvalidPathError & e) { @@ -514,7 +516,11 @@ static RegisterPrimOp primop_isPath({ struct CompareValues { - bool operator () (const Value * v1, const Value * v2) const + EvalState & state; + + CompareValues(EvalState & state) : state(state) { }; + + bool operator () (Value * v1, Value * v2) const { if (v1->type() == nFloat && v2->type() == nInt) return v1->fpoint < v2->integer; @@ -531,6 +537,17 @@ struct CompareValues return strcmp(v1->string.s, v2->string.s) < 0; case nPath: return strcmp(v1->path, v2->path) < 0; + case nList: + // Lexicographic comparison + for (size_t i = 0;; i++) { + if (i == v2->listSize()) { + return false; + } else if (i == v1->listSize()) { + return true; + } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i])) { + return (*this)(v1->listElems()[i], v2->listElems()[i]); + } + } default: throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); } @@ -598,8 +615,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar state.forceList(*startSet->value, pos); ValueList workSet; - for (unsigned int n = 0; n < startSet->value->listSize(); ++n) - workSet.push_back(startSet->value->listElems()[n]); + for (auto elem : startSet->value->listItems()) + workSet.push_back(elem); /* Get the operator. */ Bindings::iterator op = getAttr( @@ -618,7 +635,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar ValueList res; // `doneKeys' doesn't need to be a GC root, because its values are // reachable from res. - set<Value *, CompareValues> doneKeys; + auto cmp = CompareValues(state); + set<Value *, decltype(cmp)> doneKeys(cmp); while (!workSet.empty()) { Value * e = *(workSet.begin()); workSet.pop_front(); @@ -643,9 +661,9 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar state.forceList(call, pos); /* Add the values returned by the operator to the work set. */ - for (unsigned int n = 0; n < call.listSize(); ++n) { - state.forceValue(*call.listElems()[n], pos); - workSet.push_back(call.listElems()[n]); + for (auto elem : call.listItems()) { + state.forceValue(*elem, pos); + workSet.push_back(elem); } } @@ -985,16 +1003,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } if (i->name == state.sContentAddressed) { - settings.requireExperimentalFeature(Xp::CaDerivations); contentAddressed = state.forceBool(*i->value, pos); + if (contentAddressed) + settings.requireExperimentalFeature(Xp::CaDerivations); } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { state.forceList(*i->value, pos); - for (unsigned int n = 0; n < i->value->listSize(); ++n) { - string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); + for (auto elem : i->value->listItems()) { + string s = state.coerceToString(posDrvName, *elem, context, true); drv.args.push_back(s); } } @@ -1008,7 +1027,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (i->name == state.sStructuredAttrs) continue; auto placeholder(jsonObject->placeholder(key)); - printValueAsJSON(state, true, *i->value, placeholder, context); + printValueAsJSON(state, true, *i->value, pos, placeholder, context); if (i->name == state.sBuilder) drv.builder = state.forceString(*i->value, context, posDrvName); @@ -1024,8 +1043,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Require ‘outputs’ to be a list of strings. */ state.forceList(*i->value, posDrvName); Strings ss; - for (unsigned int n = 0; n < i->value->listSize(); ++n) - ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName)); + for (auto elem : i->value->listItems()) + ss.emplace_back(state.forceStringNoCtx(*elem, posDrvName)); handleOutputs(ss); } @@ -1440,20 +1459,19 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va SearchPath searchPath; - for (unsigned int n = 0; n < args[0]->listSize(); ++n) { - Value & v2(*args[0]->listElems()[n]); - state.forceAttrs(v2, pos); + for (auto v2 : args[0]->listItems()) { + state.forceAttrs(*v2, pos); string prefix; - Bindings::iterator i = v2.attrs->find(state.symbols.create("prefix")); - if (i != v2.attrs->end()) + Bindings::iterator i = v2->attrs->find(state.symbols.create("prefix")); + if (i != v2->attrs->end()) prefix = state.forceStringNoCtx(*i->value, pos); i = getAttr( state, "findFile", "path", - v2.attrs, + v2->attrs, pos ); @@ -1579,7 +1597,7 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value { std::ostringstream out; PathSet context; - printValueAsXML(state, true, false, *args[0], out, context); + printValueAsXML(state, true, false, *args[0], out, context, pos); mkString(v, out.str(), context); } @@ -1687,7 +1705,7 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu { std::ostringstream out; PathSet context; - printValueAsJSON(state, true, *args[0], out, context); + printValueAsJSON(state, true, *args[0], pos, out, context); mkString(v, out.str(), context); } @@ -1859,12 +1877,12 @@ static void addPath( // be rewritten to the actual output). state.realiseContext(context); + StorePathSet refs; + if (state.store->isInStore(path)) { auto [storePath, subPath] = state.store->toStorePath(path); - auto info = state.store->queryPathInfo(storePath); - if (!info->references.empty()) - throw EvalError("store path '%s' is not allowed to have references", - state.store->printStorePath(storePath)); + // FIXME: we should scanForReferences on the path before adding it + refs = state.store->queryPathInfo(storePath)->references; path = state.store->toRealPath(storePath) + subPath; } @@ -1880,9 +1898,6 @@ static void addPath( Value arg1; mkString(arg1, path); - Value fun2; - state.callFunction(*filterFun, arg1, fun2, noPos); - Value arg2; mkString(arg2, S_ISREG(st.st_mode) ? "regular" : @@ -1890,8 +1905,9 @@ static void addPath( S_ISLNK(st.st_mode) ? "symlink" : "unknown" /* not supported, will fail! */); + Value * args []{&arg1, &arg2}; Value res; - state.callFunction(fun2, arg2, res, noPos); + state.callFunction(*filterFun, 2, args, res, pos); return state.forceBool(res, pos); }) : defaultPathFilter; @@ -1904,7 +1920,7 @@ static void addPath( if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs)); if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); } else @@ -2221,9 +2237,9 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, /* Get the attribute names to be removed. */ std::set<Symbol> names; - for (unsigned int i = 0; i < args[1]->listSize(); ++i) { - state.forceStringNoCtx(*args[1]->listElems()[i], pos); - names.insert(state.symbols.create(args[1]->listElems()[i]->string.s)); + for (auto elem : args[1]->listItems()) { + state.forceStringNoCtx(*elem, pos); + names.insert(state.symbols.create(elem->string.s)); } /* Copy all attributes not in that set. Note that we don't need @@ -2231,7 +2247,7 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, vector. */ state.mkAttrs(v, args[0]->attrs->size()); for (auto & i : *args[0]->attrs) { - if (names.find(i.name) == names.end()) + if (!names.count(i.name)) v.attrs->push_back(i); } } @@ -2265,15 +2281,14 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, std::set<Symbol> seen; - for (unsigned int i = 0; i < args[0]->listSize(); ++i) { - Value & v2(*args[0]->listElems()[i]); - state.forceAttrs(v2, pos); + for (auto v2 : args[0]->listItems()) { + state.forceAttrs(*v2, pos); Bindings::iterator j = getAttr( state, "listToAttrs", state.sName, - v2.attrs, + v2->attrs, pos ); @@ -2285,7 +2300,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, state, "listToAttrs", state.sValue, - v2.attrs, + v2->attrs, pos ); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); @@ -2352,11 +2367,10 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va Value * res[args[1]->listSize()]; unsigned int found = 0; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - Value & v2(*args[1]->listElems()[n]); - state.forceAttrs(v2, pos); - Bindings::iterator i = v2.attrs->find(attrName); - if (i != v2.attrs->end()) + for (auto v2 : args[1]->listItems()) { + state.forceAttrs(*v2, pos); + Bindings::iterator i = v2->attrs->find(attrName); + if (i != v2->attrs->end()) res[found++] = i->value; } @@ -2631,8 +2645,8 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value { bool res = false; state.forceList(*args[1], pos); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) - if (state.eqValues(*args[0], *args[1]->listElems()[n])) { + for (auto elem : args[1]->listItems()) + if (state.eqValues(*args[0], *elem)) { res = true; break; } @@ -2691,11 +2705,10 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, if (args[2]->listSize()) { Value * vCur = args[1]; - for (unsigned int n = 0; n < args[2]->listSize(); ++n) { - Value vTmp; - state.callFunction(*args[0], *vCur, vTmp, pos); + for (auto [n, elem] : enumerate(args[2]->listItems())) { + Value * vs []{vCur, elem}; vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); - state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); + state.callFunction(*args[0], 2, vs, *vCur, pos); } state.forceValue(v, pos); } else { @@ -2709,9 +2722,9 @@ static RegisterPrimOp primop_foldlStrict({ .args = {"op", "nul", "list"}, .doc = R"( Reduce a list by applying a binary operator, from left to right, - e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) + e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]` + evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. )", .fun = prim_foldlStrict, @@ -2723,8 +2736,8 @@ static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * arg state.forceList(*args[1], pos); Value vTmp; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos); + for (auto elem : args[1]->listItems()) { + state.callFunction(*args[0], *elem, vTmp, pos); bool res = state.forceBool(vTmp, pos); if (res == any) { mkBool(v, any); @@ -2816,17 +2829,16 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value v.listElems()[n] = args[1]->listElems()[n]; } - auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) - return CompareValues()(a, b); + return CompareValues(state)(a, b); - Value vTmp1, vTmp2; - state.callFunction(*args[0], *a, vTmp1, pos); - state.callFunction(vTmp1, *b, vTmp2, pos); - return state.forceBool(vTmp2, pos); + Value * vs[] = {a, b}; + Value vBool; + state.callFunction(*args[0], 2, vs, vBool, pos); + return state.forceBool(vBool, pos); }; /* FIXME: std::sort can segfault if the comparator is not a strict @@ -2916,6 +2928,56 @@ static RegisterPrimOp primop_partition({ .fun = prim_partition, }); +static void prim_groupBy(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + ValueVectorMap attrs; + + for (auto vElem : args[1]->listItems()) { + Value res; + state.callFunction(*args[0], *vElem, res, pos); + string name = state.forceStringNoCtx(res, pos); + Symbol sym = state.symbols.create(name); + auto vector = attrs.try_emplace(sym, ValueVector()).first; + vector->second.push_back(vElem); + } + + state.mkAttrs(v, attrs.size()); + + for (auto & i : attrs) { + Value * list = state.allocAttr(v, i.first); + auto size = i.second.size(); + state.mkList(*list, size); + memcpy(list->listElems(), i.second.data(), sizeof(Value *) * size); + } +} + +static RegisterPrimOp primop_groupBy({ + .name = "__groupBy", + .args = {"f", "list"}, + .doc = R"( + Groups elements of *list* together by the string returned from the + function *f* called on each element. It returns an attribute set + where each attribute value contains the elements of *list* that are + mapped to the same corresponding attribute name returned by *f*. + + For example, + + ```nix + builtins.groupBy (builtins.substring 0 1) ["foo" "bar" "baz"] + ``` + + evaluates to + + ```nix + { b = [ "bar" "baz" ]; f = [ "foo" ]; } + ``` + )", + .fun = prim_groupBy, +}); + static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceFunction(*args[0], pos); @@ -3103,7 +3165,7 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - CompareValues comp; + CompareValues comp{state}; mkBool(v, comp(args[0], args[1])); } @@ -3454,9 +3516,9 @@ static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * * res.reserve((args[1]->listSize() + 32) * sep.size()); bool first = true; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + for (auto elem : args[1]->listItems()) { if (first) first = false; else res += sep; - res += state.coerceToString(pos, *args[1]->listElems()[n], context); + res += state.coerceToString(pos, *elem, context); } mkString(v, res, context); @@ -3485,14 +3547,14 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar vector<string> from; from.reserve(args[0]->listSize()); - for (unsigned int n = 0; n < args[0]->listSize(); ++n) - from.push_back(state.forceString(*args[0]->listElems()[n], pos)); + for (auto elem : args[0]->listItems()) + from.push_back(state.forceString(*elem, pos)); vector<std::pair<string, PathSet>> to; to.reserve(args[1]->listSize()); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + for (auto elem : args[1]->listItems()) { PathSet ctx; - auto s = state.forceString(*args[1]->listElems()[n], ctx, pos); + auto s = state.forceString(*elem, ctx, pos); to.push_back(std::make_pair(std::move(s), std::move(ctx))); } @@ -3693,7 +3755,7 @@ void EvalState::createBaseEnv() language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ - mkInt(v, 5); + mkInt(v, 6); addConstant("__langVersion", v); // Miscellaneous @@ -3720,21 +3782,27 @@ void EvalState::createBaseEnv() .fun = primOp.fun, .arity = std::max(primOp.args.size(), primOp.arity), .name = symbols.create(primOp.name), - .args = std::move(primOp.args), + .args = primOp.args, .doc = primOp.doc, }); /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ sDerivationNix = symbols.create("//builtin/derivation.nix"); - eval(parse( - #include "primops/derivation.nix.gen.hh" - , foFile, sDerivationNix, "/", staticBaseEnv), v); - addConstant("derivation", v); + auto vDerivation = allocValue(); + addConstant("derivation", vDerivation); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); + + staticBaseEnv.sort(); + + /* Note: we have to initialize the 'derivation' constant *after* + building baseEnv/staticBaseEnv because it uses 'builtins'. */ + eval(parse( + #include "primops/derivation.nix.gen.hh" + , foFile, sDerivationNix, "/", staticBaseEnv), *vDerivation); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 31cf812b4..20545afd0 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -118,9 +118,8 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); state.mkList(outputsVal, info.second.outputs.size()); size_t i = 0; - for (const auto & output : info.second.outputs) { + for (const auto & output : info.second.outputs) mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); - } } infoVal.attrs->sort(); } @@ -181,8 +180,8 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg .errPos = *i.pos }); } - for (unsigned int n = 0; n < iter->value->listSize(); ++n) { - auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); + for (auto elem : iter->value->listItems()) { + auto name = state.forceStringNoCtx(*elem, *iter->pos); context.insert("!" + name + "!" + string(i.name)); } } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e6becdafc..079513873 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -74,7 +74,10 @@ std::string fixURI(std::string uri, EvalState & state, const std::string & defau std::string fixURIForGit(std::string uri, EvalState & state) { - static std::regex scp_uri("([^/].*)@(.*):(.*)"); + /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + * them by removing the `:` and assuming a scheme of `ssh://` + * */ + static std::regex scp_uri("([^/]*)@(.*):(.*)"); if (uri[0] != '/' && std::regex_match(uri, scp_uri)) return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); else diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index bfea24d40..517da4c01 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -10,11 +10,11 @@ namespace nix { void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context) + Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context) { checkInterrupt(); - if (strict) state.forceValue(v); + if (strict) state.forceValue(v, pos); switch (v.type()) { @@ -40,7 +40,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nAttrs: { - auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); + auto maybeString = state.tryAttrsToString(pos, v, context, false, false); if (maybeString) { out.write(*maybeString); break; @@ -54,18 +54,18 @@ void printValueAsJSON(EvalState & state, bool strict, for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, placeholder, context); + printValueAsJSON(state, strict, *a.value, *a.pos, placeholder, context); } } else - printValueAsJSON(state, strict, *i->value, out, context); + printValueAsJSON(state, strict, *i->value, *i->pos, out, context); break; } case nList: { auto list(out.list()); - for (unsigned int n = 0; n < v.listSize(); ++n) { + for (auto elem : v.listItems()) { auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context); + printValueAsJSON(state, strict, *elem, pos, placeholder, context); } break; } @@ -79,18 +79,20 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nThunk: - throw TypeError("cannot convert %1% to JSON", showType(v)); - case nFunction: - throw TypeError("cannot convert %1% to JSON", showType(v)); + auto e = TypeError({ + .msg = hintfmt("cannot convert %1% to JSON", showType(v)), + .errPos = v.determinePos(pos) + }); + throw e.addTrace(pos, hintfmt("message for the trace")); } } void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context) + Value & v, const Pos & pos, std::ostream & str, PathSet & context) { JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, out, context); + printValueAsJSON(state, strict, v, pos, out, context); } void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 67fed6487..c2f797b29 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { class JSONPlaceholder; void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context); + Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context); void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context); + Value & v, const Pos & pos, std::ostream & str, PathSet & context); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index b44455f5f..a875f82d7 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -18,7 +18,8 @@ static XMLAttrs singletonAttrs(const string & name, const string & value) static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos); static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) @@ -46,17 +47,18 @@ static void showAttrs(EvalState & state, bool strict, bool location, XMLOpenElement _(doc, "attr", xmlAttrs); printValueAsXML(state, strict, location, - *a.value, doc, context, drvsSeen); + *a.value, doc, context, drvsSeen, *a.pos); } } static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) { checkInterrupt(); - if (strict) state.forceValue(v); + if (strict) state.forceValue(v, pos); switch (v.type()) { @@ -91,14 +93,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, Path drvPath; a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); + if (strict) state.forceValue(*a->value, *a->pos); if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); + if (strict) state.forceValue(*a->value, *a->pos); if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->string.s; } @@ -120,8 +122,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nList: { XMLOpenElement _(doc, "list"); - for (unsigned int n = 0; n < v.listSize(); ++n) - printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); + for (auto v2 : v.listItems()) + printValueAsXML(state, strict, location, *v2, doc, context, drvsSeen, pos); break; } @@ -149,7 +151,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, } case nExternal: - v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); + v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); break; case nFloat: @@ -163,19 +165,20 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const + bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) const { doc.writeEmptyElement("unevaluated"); } void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context) + Value & v, std::ostream & out, PathSet & context, const Pos & pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); PathSet drvsSeen; - printValueAsXML(state, strict, location, v, doc, context, drvsSeen); + printValueAsXML(state, strict, location, v, doc, context, drvsSeen, pos); } diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..cc778a2cb 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -9,6 +9,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context); - + Value & v, std::ostream & out, PathSet & context, const Pos & pos); + } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index a1f131f9e..6b4f3c0ae 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,5 +1,7 @@ #pragma once +#include <cassert> + #include "symbol-table.hh" #if HAVE_BOEHMGC @@ -94,7 +96,8 @@ class ExternalValueBase /* Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const; + XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) const; virtual ~ExternalValueBase() { @@ -349,6 +352,34 @@ public: bool isTrivial() const; std::vector<std::pair<Path, std::string>> getContext(); + + auto listItems() + { + struct ListIterable + { + typedef Value * const * iterator; + iterator _begin, _end; + iterator begin() const { return _begin; } + iterator end() const { return _end; } + }; + assert(isList()); + auto begin = listElems(); + return ListIterable { begin, begin + listSize() }; + } + + auto listItems() const + { + struct ConstListIterable + { + typedef const Value * const * iterator; + iterator _begin, _end; + iterator begin() const { return _begin; } + iterator end() const { return _end; } + }; + assert(isList()); + auto begin = listElems(); + return ConstListIterable { begin, begin + listSize() }; + } }; @@ -394,9 +425,11 @@ void mkPath(Value & v, const char * s); #if HAVE_BOEHMGC typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector; typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap; +typedef std::map<Symbol, ValueVector, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, ValueVector> > > ValueVectorMap; #else typedef std::vector<Value *> ValueVector; typedef std::map<Symbol, Value *> ValueMap; +typedef std::map<Symbol, ValueVector> ValueVectorMap; #endif diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8468d2afc..544d2ffbf 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -51,7 +51,7 @@ struct GitInputScheme : InputScheme for (auto &[name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); - else if (name == "shallow") + else if (name == "shallow" || name == "submodules") attrs.emplace(name, Explicit<bool> { value == "1" }); else url2.query.emplace(name, value); @@ -324,17 +324,13 @@ struct GitInputScheme : InputScheme Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); repoDir = cacheDir; - Path cacheDirLock = cacheDir + ".lock"; createDirs(dirOf(cacheDir)); - AutoCloseFD lock = openLockFile(cacheDirLock, true); - lockFile(lock.get(), ltWrite, true); + PathLocks cacheDirLock({cacheDir + ".lock"}); if (!pathExists(cacheDir)) { runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); } - deleteLockFile(cacheDirLock, lock.get()); - Path localRefFile = input.getRef()->compare(0, 5, "refs/") == 0 ? cacheDir + "/" + *input.getRef() @@ -399,6 +395,8 @@ struct GitInputScheme : InputScheme if (!input.getRev()) input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + + // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ffc44e9e2..1c539b80e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -300,7 +300,7 @@ struct GitLabInputScheme : GitArchiveInputScheme if ("PAT" == token.substr(0, fldsplit)) return std::make_pair("Private-token", token.substr(fldsplit+1)); warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit)); - return std::nullopt; + return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1)); } Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index fb5702c4c..07e543c53 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -97,7 +97,7 @@ struct PathInputScheme : InputScheme // for security, ensure that if the parent is a store path, it's inside it if (store->isInStore(parent)) { auto storePath = store->printStorePath(store->toStorePath(parent).first); - if (!isInDir(absPath, storePath)) + if (!isDirOrInDir(absPath, storePath)) throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); } } else diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index b2a6e2a82..f4306ab91 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -11,7 +11,7 @@ namespace nix { -static std::string getS(const std::vector<Logger::Field> & fields, size_t n) +static std::string_view getS(const std::vector<Logger::Field> & fields, size_t n) { assert(n < fields.size()); assert(fields[n].type == Logger::Field::tString); @@ -103,17 +103,19 @@ public: ~ProgressBar() { stop(); - updateThread.join(); } void stop() override { - auto state(state_.lock()); - if (!state->active) return; - state->active = false; - writeToStderr("\r\e[K"); - updateCV.notify_one(); - quitCV.notify_one(); + { + auto state(state_.lock()); + if (!state->active) return; + state->active = false; + writeToStderr("\r\e[K"); + updateCV.notify_one(); + quitCV.notify_one(); + } + updateThread.join(); } bool isVerbose() override { diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 85f9f0d58..4404e0195 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -15,9 +15,14 @@ #include <sys/stat.h> #include <unistd.h> #include <signal.h> -#include <sys/types.h> -#include <sys/socket.h> -#include <netdb.h> +#ifdef __linux__ +#include <features.h> +#endif +#ifdef __GLIBC__ +#include <gnu/lib-names.h> +#include <nss.h> +#include <dlfcn.h> +#endif #include <openssl/crypto.h> @@ -121,21 +126,30 @@ static void preloadNSS() { been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to load its lookup libraries in the parent before any child gets a chance to. */ std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN. - We need the resolution to be done locally, as nscd socket will not be accessible in the - sandbox. */ - char * previous_env = getenv("LOCALDOMAIN"); - setenv("LOCALDOMAIN", "invalid", 1); - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) { - if (res) freeaddrinfo(res); - } - if (previous_env) { - setenv("LOCALDOMAIN", previous_env, 1); - } else { - unsetenv("LOCALDOMAIN"); +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (dlopen (LIBNSS_DNS_SO, RTLD_NOW) == NULL) { + printMsg(Verbosity::lvlWarn, fmt("Unable to load nss_dns backend")); } + __nss_configure_lookup ("hosts", "dns"); +#endif }); } @@ -413,7 +427,7 @@ RunPager::RunPager() }); pid.setKillSignal(SIGINT); - + stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); } @@ -424,7 +438,7 @@ RunPager::~RunPager() try { if (pid != -1) { std::cout.flush(); - close(STDOUT_FILENO); + dup2(stdout, STDOUT_FILENO); pid.wait(); } } catch (...) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 05277d90a..ed012959b 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -88,6 +88,7 @@ public: private: Pid pid; + int stdout; }; extern volatile ::sig_atomic_t blockInt; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 8fce94264..13c086a46 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - std::string hashPart(narInfo->path.hashPart()); - { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) }); + state_->pathInfoCache.upsert( + std::string(narInfo->path.to_string()), + PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) }); } if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); + diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo)); } AutoCloseFD openFile(const Path & path) @@ -308,16 +308,17 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name), + makeFixedOutputPath(method, nar.first, name, references), nar.first, }; info.narSize = nar.second; + info.references = references; return info; })->path; } @@ -385,7 +386,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default @@ -404,10 +405,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name), + makeFixedOutputPath(method, h, name, references), nar.first, }; info.narSize = nar.second; + info.references = references; info.ca = FixedOutputHash { .method = method, .hash = h, @@ -437,40 +439,29 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s })->path; } -std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id) +void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { - if (diskCache) { - auto [cacheOutcome, maybeCachedRealisation] = - diskCache->lookupRealisation(getUri(), id); - switch (cacheOutcome) { - case NarInfoDiskCache::oValid: - debug("Returning a cached realisation for %s", id.to_string()); - return *maybeCachedRealisation; - case NarInfoDiskCache::oInvalid: - debug("Returning a cached missing realisation for %s", id.to_string()); - return {}; - case NarInfoDiskCache::oUnknown: - break; - } - } - auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; - auto rawOutputInfo = getFile(outputInfoFilePath); - if (rawOutputInfo) { - auto realisation = Realisation::fromJSON( - nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath); + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); - if (diskCache) - diskCache->upsertRealisation( - getUri(), realisation); + Callback<std::shared_ptr<std::string>> newCallback = { + [=](std::future<std::shared_ptr<std::string>> fut) { + try { + auto data = fut.get(); + if (!data) return (*callbackPtr)(nullptr); - return {realisation}; - } else { - if (diskCache) - diskCache->upsertAbsentRealisation(getUri(), id); - return std::nullopt; - } + auto realisation = Realisation::fromJSON( + nlohmann::json::parse(*data), outputInfoFilePath); + return (*callbackPtr)(std::make_shared<const Realisation>(realisation)); + } catch (...) { + callbackPtr->rethrow(); + } + } + }; + + getFile(outputInfoFilePath, std::move(newCallback)); } void BinaryCacheStore::registerDrvOutput(const Realisation& info) { diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 723f2e805..9815af591 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -97,18 +97,19 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references ) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; + PathFilter & filter, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; void registerDrvOutput(const Realisation & info) override; - std::optional<const Realisation> queryRealisation(const DrvOutput &) override; + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; void narFromPath(const StorePath & path, Sink & sink) override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b924d23b2..60945403e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -655,7 +655,7 @@ void DerivationGoal::tryLocalBuild() { throw Error( "unable to build with a primary store that isn't a local store; " "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); } diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index be270d079..b9602e696 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -1,6 +1,8 @@ #include "drv-output-substitution-goal.hh" +#include "finally.hh" #include "worker.hh" #include "substitution-goal.hh" +#include "callback.hh" namespace nix { @@ -50,14 +52,42 @@ void DrvOutputSubstitutionGoal::tryNext() return; } - auto sub = subs.front(); + sub = subs.front(); subs.pop_front(); // FIXME: Make async - outputInfo = sub->queryRealisation(id); + // outputInfo = sub->queryRealisation(id); + outPipe.create(); + promise = decltype(promise)(); + + sub->queryRealisation( + id, { [&](std::future<std::shared_ptr<const Realisation>> res) { + try { + Finally updateStats([this]() { outPipe.writeSide.close(); }); + promise.set_value(res.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + } }); + + worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + + state = &DrvOutputSubstitutionGoal::realisationFetched; +} + +void DrvOutputSubstitutionGoal::realisationFetched() +{ + worker.childTerminated(this); + + try { + outputInfo = promise.get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + substituterFailed = true; + } + if (!outputInfo) { - tryNext(); - return; + return tryNext(); } for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { @@ -119,4 +149,10 @@ void DrvOutputSubstitutionGoal::work() (this->*state)(); } +void DrvOutputSubstitutionGoal::handleEOF(int fd) +{ + if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); +} + + } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 63ab53d89..67ae2624a 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -3,6 +3,8 @@ #include "store-api.hh" #include "goal.hh" #include "realisation.hh" +#include <thread> +#include <future> namespace nix { @@ -20,11 +22,18 @@ private: // The realisation corresponding to the given output id. // Will be filled once we can get it. - std::optional<Realisation> outputInfo; + std::shared_ptr<const Realisation> outputInfo; /* The remaining substituters. */ std::list<ref<Store>> subs; + /* The current substituter. */ + std::shared_ptr<Store> sub; + + Pipe outPipe; + std::thread thr; + std::promise<std::shared_ptr<const Realisation>> promise; + /* Whether a substituter failed. */ bool substituterFailed = false; @@ -36,6 +45,7 @@ public: void init(); void tryNext(); + void realisationFetched(); void outPathValid(); void finished(); @@ -44,7 +54,7 @@ public: string key() override; void work() override; - + void handleEOF(int fd) override; }; } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 065efc855..9b4cfd835 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,4 +1,3 @@ -#include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" #include "derivation-goal.hh" diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index fab6c3a08..c9a4a31e7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -342,7 +342,7 @@ int childEntry(void * arg) return 1; } - +#if __linux__ static void linkOrCopy(const Path & from, const Path & to) { if (link(from.c_str(), to.c_str()) == -1) { @@ -358,6 +358,7 @@ static void linkOrCopy(const Path & from, const Path & to) copyPath(from, to); } } +#endif void LocalDerivationGoal::startBuilder() @@ -917,7 +918,9 @@ void LocalDerivationGoal::startBuilder() } else #endif { +#if __linux__ fallback: +#endif pid = startProcess([&]() { runChild(); }); @@ -1179,7 +1182,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1198,9 +1202,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo } StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) override { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); goal.addDependency(path); return path; } @@ -1224,13 +1229,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // corresponds to an allowed derivation { throw Error("registerDrvOutput"); } - std::optional<const Realisation> queryRealisation(const DrvOutput & id) override + void queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override // XXX: This should probably be allowed if the realisation corresponds to // an allowed derivation { if (!goal.isAllowed(id)) - throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string()); - return next->queryRealisation(id); + callback(nullptr); + next->queryRealisation(id, std::move(callback)); } void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override @@ -1353,7 +1359,7 @@ void LocalDerivationGoal::startDaemon() AutoCloseFD remote = accept(daemonSocket.get(), (struct sockaddr *) &remoteAddr, &remoteAddrLen); if (!remote) { - if (errno == EINTR) continue; + if (errno == EINTR || errno == EAGAIN) continue; if (errno == EINVAL) break; throw SysError("accepting connection"); } @@ -1991,7 +1997,7 @@ void LocalDerivationGoal::runChild() else if (drv->builder == "builtin:unpack-channel") builtinUnpackChannel(drv2); else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); + throw Error("unsupported builtin builder '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 55afb5cca..f11c5ce68 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -281,11 +281,11 @@ void Worker::run(const Goals & _topGoals) if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); else throw Error("unable to start any build; remote machines may not have " "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); } assert(!awake.empty()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 974d1c471..cf32ccdc4 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -120,8 +120,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) { ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) { - std::string_view asPrefix {std::string{caMethod} + ":"}; - return parseContentAddressMethodPrefix(asPrefix); + std::string asPrefix = std::string{caMethod} + ":"; + // parseContentAddressMethodPrefix takes its argument by reference + std::string_view asPrefixView = asPrefix; + return parseContentAddressMethodPrefix(asPrefixView); } std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4b3465da3..bafab6fd5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -403,9 +403,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, return store->queryPathInfo(path); }, [&](FixedOutputHashMethod & fohm) { - if (!refs.empty()) - throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); - auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); + auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); return store->queryPathInfo(path); }, }, contentAddressMethod); @@ -433,25 +431,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store, hashAlgo = parseHashType(hashAlgoRaw); } - StringSink saved; - TeeSource savedNARSource(from, saved); - RetrieveRegularNARSink savedRegular { saved }; - - if (method == FileIngestionMethod::Recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNARSource); - } else - parseDump(savedRegular, from); - + auto dumpSource = sinkToSource([&](Sink & saved) { + if (method == FileIngestionMethod::Recursive) { + /* We parse the NAR dump through into `saved` unmodified, + so why all this extra work? We still parse the NAR so + that we aren't sending arbitrary data to `saved` + unwittingly`, and we know when the NAR ends so we don't + consume the rest of `from` and can't parse another + command. (We don't trust `addToStoreFromDump` to not + eagerly consume the entire stream it's given, past the + length of the Nar. */ + TeeSource savedNARSource(from, saved); + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNARSource); + } else { + /* Incrementally parse the NAR file, stripping the + metadata, and streaming the sole file we expect into + `saved`. */ + RetrieveRegularNARSink savedRegular { saved }; + parseDump(savedRegular, from); + if (!savedRegular.regular) throw Error("regular file expected"); + } + }); logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - - // FIXME: try to stream directly from `from`. - StringSource dumpSource { *saved.s }; - auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); + auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -625,9 +628,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } + // Obsolete. case wopSyncWithGC: { logger->startWork(); - store->syncWithGC(); logger->stopWork(); to << 1; break; @@ -953,7 +956,7 @@ void processConnection( Finally finally([&]() { _isInterrupted = false; - prevLogger->log(lvlDebug, fmt("%d operations", opCount)); + printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { @@ -986,6 +989,8 @@ void processConnection( break; } + printMsgUsing(prevLogger, lvlDebug, "received daemon op %d", op); + opCount++; try { diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 36c6e725c..62dc21c59 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -50,8 +50,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } - std::optional<const Realisation> queryRealisation(const DrvOutput&) override - { unsupported("queryRealisation"); } + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override + { callback(nullptr); } }; static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 37e17b397..4621a8217 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -544,6 +544,14 @@ struct curlFileTransfer : public FileTransfer stopWorkerThread(); }); +#ifdef __linux__ + /* Cause this thread to not share any FS attributes with the main thread, + because this causes setns() in restoreMountNamespace() to fail. + Ideally, this would happen in the std::thread() constructor. */ + if (unshare(CLONE_FS) != 0) + throw SysError("unsharing filesystem state in download thread"); +#endif + std::map<CURL *, std::shared_ptr<TransferItem>> items; bool quit = false; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 5a62c6529..7a414da6b 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -10,48 +10,22 @@ #include <regex> #include <random> -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/statvfs.h> +#include <climits> #include <errno.h> #include <fcntl.h> +#include <poll.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/types.h> +#include <sys/un.h> #include <unistd.h> -#include <climits> namespace nix { -static string gcLockName = "gc.lock"; -static string gcRootsDir = "gcroots"; - - -/* Acquire the global GC lock. This is used to prevent new Nix - processes from starting after the temporary root files have been - read. To be precise: when they try to create a new temporary root - file, they will block until the garbage collector has finished / - yielded the GC lock. */ -AutoCloseFD LocalStore::openGCLock(LockType lockType) -{ - Path fnGCLock = (format("%1%/%2%") - % stateDir % gcLockName).str(); - - debug(format("acquiring global GC lock '%1%'") % fnGCLock); - - AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fdGCLock) - throw SysError("opening global GC lock '%1%'", fnGCLock); - - if (!lockFile(fdGCLock.get(), lockType, false)) { - printInfo("waiting for the big garbage collector lock..."); - lockFile(fdGCLock.get(), lockType, true); - } - - /* !!! Restrict read permission on the GC root. Otherwise any - process that can open the file for reading can DoS the - collector. */ - - return fdGCLock; -} +static std::string gcSocketPath = "/gc-socket/socket"; +static std::string gcRootsDir = "gcroots"; static void makeSymlink(const Path & link, const Path & target) @@ -71,12 +45,6 @@ static void makeSymlink(const Path & link, const Path & target) } -void LocalStore::syncWithGC() -{ - AutoCloseFD fdGCLock = openGCLock(ltRead); -} - - void LocalStore::addIndirectRoot(const Path & path) { string hash = hashString(htSHA1, path).to_string(Base32, false); @@ -95,6 +63,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot "creating a garbage collector root (%1%) in the Nix store is forbidden " "(are you running nix-build inside the store?)", gcRoot); + /* Register this root with the garbage collector, if it's + running. This should be superfluous since the caller should + have registered this root yet, but let's be on the safe + side. */ + addTempRoot(storePath); + /* Don't clobber the link if it already exists and doesn't point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) @@ -102,11 +76,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); - /* Grab the global GC root, causing us to block while a GC is in - progress. This prevents the set of permanent roots from - increasing while a GC is in progress. */ - syncWithGC(); - return gcRoot; } @@ -119,8 +88,6 @@ void LocalStore::addTempRoot(const StorePath & path) if (!state->fdTempRoots) { while (1) { - AutoCloseFD fdGCLock = openGCLock(ltRead); - if (pathExists(fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ @@ -128,10 +95,8 @@ void LocalStore::addTempRoot(const StorePath & path) state->fdTempRoots = openLockFile(fnTempRoots, true); - fdGCLock = -1; - - debug(format("acquiring read lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltRead, true); + debug("acquiring write lock on '%s'", fnTempRoots); + lockFile(state->fdTempRoots.get(), ltWrite, true); /* Check whether the garbage collector didn't get in our way. */ @@ -147,24 +112,55 @@ void LocalStore::addTempRoot(const StorePath & path) } - /* Upgrade the lock to a write lock. This will cause us to block - if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltWrite, true); + if (!state->fdGCLock) + state->fdGCLock = openGCLock(); + + restart: + FdLock gcLock(state->fdGCLock.get(), ltRead, false, ""); + + if (!gcLock.acquired) { + /* We couldn't get a shared global GC lock, so the garbage + collector is running. So we have to connect to the garbage + collector and inform it about our root. */ + if (!state->fdRootsSocket) { + auto socketPath = stateDir.get() + gcSocketPath; + debug("connecting to '%s'", socketPath); + state->fdRootsSocket = createUnixDomainSocket(); + nix::connect(state->fdRootsSocket.get(), socketPath); + } + + try { + debug("sending GC root '%s'", printStorePath(path)); + writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false); + char c; + readFull(state->fdRootsSocket.get(), &c, 1); + assert(c == '1'); + debug("got ack for GC root '%s'", printStorePath(path)); + } catch (SysError & e) { + /* The garbage collector may have exited, so we need to + restart. */ + if (e.errNo == EPIPE) { + debug("GC socket disconnected"); + state->fdRootsSocket.close(); + goto restart; + } + } catch (EndOfFile & e) { + debug("GC socket disconnected"); + state->fdRootsSocket.close(); + goto restart; + } + } + /* Append the store path to the temporary roots file. */ string s = printStorePath(path) + '\0'; writeFull(state->fdTempRoots.get(), s); - - /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltRead, true); } static std::string censored = "{censored}"; -void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) +void LocalStore::findTempRoots(Roots & tempRoots, bool censor) { /* Read the `temproots' directory for per-process temporary root files. */ @@ -179,35 +175,25 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) pid_t pid = std::stoi(i.name); debug(format("reading temporary root file '%1%'") % path); - FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666))); - if (!*fd) { + AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)); + if (!fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; throw SysError("opening temporary roots file '%1%'", path); } - /* This should work, but doesn't, for some reason. */ - //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); - //if (*fd == -1) continue; - /* Try to acquire a write lock without blocking. This can only succeed if the owning process has died. In that case we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { + if (lockFile(fd.get(), ltWrite, false)) { printInfo("removing stale temporary roots file '%1%'", path); unlink(path.c_str()); - writeFull(fd->get(), "d"); + writeFull(fd.get(), "d"); continue; } - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - /* Read the entire file. */ - string contents = readFile(fd->get()); + string contents = readFile(fd.get()); /* Extract the roots. */ string::size_type pos = 0, end; @@ -218,8 +204,6 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } - - fds.push_back(fd); /* keep open */ } } @@ -304,8 +288,7 @@ Roots LocalStore::findRoots(bool censor) Roots roots; findRootsNoTemp(roots, censor); - FDs fds; - findTempRoots(fds, roots, censor); + findTempRoots(roots, censor); return roots; } @@ -341,6 +324,7 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } +#if __linux__ static void readFileRoots(const char * path, UncheckedRoots & roots) { try { @@ -350,6 +334,7 @@ static void readFileRoots(const char * path, UncheckedRoots & roots) throw; } } +#endif void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { @@ -431,7 +416,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } #endif -#if defined(__linux__) +#if __linux__ readFileRoots("/proc/sys/kernel/modprobe", unchecked); readFileRoots("/proc/sys/kernel/fbsplash", unchecked); readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); @@ -455,391 +440,397 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) struct GCLimitReached { }; -struct LocalStore::GCState -{ - const GCOptions & options; - GCResults & results; - StorePathSet roots; - StorePathSet tempRoots; - StorePathSet dead; - StorePathSet alive; - bool gcKeepOutputs; - bool gcKeepDerivations; - uint64_t bytesInvalidated; - bool moveToTrash = true; - bool shouldDelete; - GCState(const GCOptions & options, GCResults & results) - : options(options), results(results), bytesInvalidated(0) { } -}; - - -bool LocalStore::isActiveTempFile(const GCState & state, - const Path & path, const string & suffix) +void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { - return hasSuffix(path, suffix) - && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size()))); -} + bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + bool gcKeepOutputs = settings.gcKeepOutputs; + bool gcKeepDerivations = settings.gcKeepDerivations; + StorePathSet roots, dead, alive; -void LocalStore::deleteGarbage(GCState & state, const Path & path) -{ - uint64_t bytesFreed; - deletePath(path, bytesFreed); - state.results.bytesFreed += bytesFreed; -} + struct Shared + { + // The temp roots only store the hash part to make it easier to + // ignore suffixes like '.lock', '.chroot' and '.check'. + std::unordered_set<std::string> tempRoots; + // Hash part of the store path currently being deleted, if + // any. + std::optional<std::string> pending; + }; -void LocalStore::deletePathRecursive(GCState & state, const Path & path) -{ - checkInterrupt(); - - uint64_t size = 0; - - auto storePath = maybeParseStorePath(path); - if (storePath && isValidPath(*storePath)) { - StorePathSet referrers; - queryReferrers(*storePath, referrers); - for (auto & i : referrers) - if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i)); - size = queryPathInfo(*storePath)->narSize; - invalidatePathChecked(*storePath); - } + Sync<Shared> _shared; - Path realPath = realStoreDir + "/" + std::string(baseNameOf(path)); + std::condition_variable wakeup; - struct stat st; - if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT) return; - throw SysError("getting status of %1%", realPath); + /* Using `--ignore-liveness' with `--delete' can have unintended + consequences if `keep-outputs' or `keep-derivations' are true + (the garbage collector will recurse into deleting the outputs + or derivers, respectively). So disable them. */ + if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + gcKeepOutputs = false; + gcKeepDerivations = false; } - printInfo(format("deleting '%1%'") % path); - - state.results.paths.insert(path); + if (shouldDelete) + deletePath(reservedPath); - /* If the path is not a regular file or symlink, move it to the - trash directory. The move is to ensure that later (when we're - not holding the global GC lock) we can delete the path without - being afraid that the path has become alive again. Otherwise - delete it right away. */ - if (state.moveToTrash && S_ISDIR(st.st_mode)) { - // Estimate the amount freed using the narSize field. FIXME: - // if the path was not valid, need to determine the actual - // size. - try { - if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError("making '%1%' writable", realPath); - Path tmp = trashDir + "/" + std::string(baseNameOf(path)); - if (rename(realPath.c_str(), tmp.c_str())) - throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp); - state.bytesInvalidated += size; - } catch (SysError & e) { - if (e.errNo == ENOSPC) { - printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg()); - deleteGarbage(state, realPath); + /* Acquire the global GC root. Note: we don't use fdGCLock + here because then in auto-gc mode, another thread could + downgrade our exclusive lock. */ + auto fdGCLock = openGCLock(); + FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock..."); + + /* Start the server for receiving new roots. */ + auto socketPath = stateDir.get() + gcSocketPath; + createDirs(dirOf(socketPath)); + auto fdServer = createUnixDomainSocket(socketPath, 0666); + + if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1) + throw SysError("making socket '%1%' non-blocking", socketPath); + + Pipe shutdownPipe; + shutdownPipe.create(); + + std::thread serverThread([&]() { + Sync<std::map<int, std::thread>> connections; + + Finally cleanup([&]() { + debug("GC roots server shutting down"); + while (true) { + auto item = remove_begin(*connections.lock()); + if (!item) break; + auto & [fd, thread] = *item; + shutdown(fd, SHUT_RDWR); + thread.join(); } - } - } else - deleteGarbage(state, realPath); - - if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { - printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); - throw GCLimitReached(); - } -} - - -bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path) -{ - if (visited.count(path)) return false; - - if (state.alive.count(path)) return true; - - if (state.dead.count(path)) return false; - - if (state.roots.count(path)) { - debug("cannot delete '%1%' because it's a root", printStorePath(path)); - state.alive.insert(path); - return true; - } - - visited.insert(path); - - if (!isValidPath(path)) return false; - - StorePathSet incoming; - - /* Don't delete this path if any of its referrers are alive. */ - queryReferrers(path, incoming); - - /* If keep-derivations is set and this is a derivation, then - don't delete the derivation if any of the outputs are alive. */ - if (state.gcKeepDerivations && path.isDerivation()) { - for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path)) - if (maybeOutPath && - isValidPath(*maybeOutPath) && - queryPathInfo(*maybeOutPath)->deriver == path - ) - incoming.insert(*maybeOutPath); - } - - /* If keep-outputs is set, then don't delete this path if there - are derivers of this path that are not garbage. */ - if (state.gcKeepOutputs) { - auto derivers = queryValidDerivers(path); - for (auto & i : derivers) - incoming.insert(i); - } + }); + + while (true) { + std::vector<struct pollfd> fds; + fds.push_back({.fd = shutdownPipe.readSide.get(), .events = POLLIN}); + fds.push_back({.fd = fdServer.get(), .events = POLLIN}); + auto count = poll(fds.data(), fds.size(), -1); + assert(count != -1); + + if (fds[0].revents) + /* Parent is asking us to quit. */ + break; + + if (fds[1].revents) { + /* Accept a new connection. */ + assert(fds[1].revents & POLLIN); + AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr); + if (!fdClient) continue; + + /* Process the connection in a separate thread. */ + auto fdClient_ = fdClient.get(); + std::thread clientThread([&, fdClient = std::move(fdClient)]() { + Finally cleanup([&]() { + auto conn(connections.lock()); + auto i = conn->find(fdClient.get()); + if (i != conn->end()) { + i->second.detach(); + conn->erase(i); + } + }); + + while (true) { + try { + auto path = readLine(fdClient.get()); + auto storePath = maybeParseStorePath(path); + if (storePath) { + debug("got new GC root '%s'", path); + auto hashPart = std::string(storePath->hashPart()); + auto shared(_shared.lock()); + shared->tempRoots.insert(hashPart); + /* If this path is currently being + deleted, then we have to wait until + deletion is finished to ensure that + the client doesn't start + re-creating it before we're + done. FIXME: ideally we would use a + FD for this so we don't block the + poll loop. */ + while (shared->pending == hashPart) { + debug("synchronising with deletion of path '%s'", path); + shared.wait(wakeup); + } + } else + printError("received garbage instead of a root from client"); + writeFull(fdClient.get(), "1", false); + } catch (Error &) { break; } + } + }); - for (auto & i : incoming) - if (i != path) - if (canReachRoot(state, visited, i)) { - state.alive.insert(path); - return true; + connections.lock()->insert({fdClient_, std::move(clientThread)}); } + } + }); - return false; -} - - -void LocalStore::tryToDelete(GCState & state, const Path & path) -{ - checkInterrupt(); - - auto realPath = realStoreDir + "/" + std::string(baseNameOf(path)); - if (realPath == linksDir || realPath == trashDir) return; - - //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); - - auto storePath = maybeParseStorePath(path); - - if (!storePath || !isValidPath(*storePath)) { - /* A lock file belonging to a path that we're building right - now isn't garbage. */ - if (isActiveTempFile(state, path, ".lock")) return; + Finally stopServer([&]() { + writeFull(shutdownPipe.writeSide.get(), "x", false); + wakeup.notify_all(); + if (serverThread.joinable()) serverThread.join(); + }); - /* Don't delete .chroot directories for derivations that are - currently being built. */ - if (isActiveTempFile(state, path, ".chroot")) return; + /* Find the roots. Since we've grabbed the GC lock, the set of + permanent roots cannot increase now. */ + printInfo("finding garbage collector roots..."); + Roots rootMap; + if (!options.ignoreLiveness) + findRootsNoTemp(rootMap, true); - /* Don't delete .check directories for derivations that are - currently being built, because we may need to run - diff-hook. */ - if (isActiveTempFile(state, path, ".check")) return; - } + for (auto & i : rootMap) roots.insert(i.first); - StorePathSet visited; - - if (storePath && canReachRoot(state, visited, *storePath)) { - debug("cannot delete '%s' because it's still reachable", path); - } else { - /* No path we visited was a root, so everything is garbage. - But we only delete ‘path’ and its referrers here so that - ‘nix-store --delete’ doesn't have the unexpected effect of - recursing into derivations and outputs. */ - for (auto & i : visited) - state.dead.insert(i); - if (state.shouldDelete) - deletePathRecursive(state, path); + /* Read the temporary roots created before we acquired the global + GC root. Any new roots will be sent to our socket. */ + Roots tempRoots; + findTempRoots(tempRoots, true); + for (auto & root : tempRoots) { + _shared.lock()->tempRoots.insert(std::string(root.first.hashPart())); + roots.insert(root.first); } -} + /* Helper function that deletes a path from the store and throws + GCLimitReached if we've deleted enough garbage. */ + auto deleteFromStore = [&](std::string_view baseName) + { + Path path = storeDir + "/" + std::string(baseName); + Path realPath = realStoreDir + "/" + std::string(baseName); -/* Unlink all files in /nix/store/.links that have a link count of 1, - which indicates that there are no other links and so they can be - safely deleted. FIXME: race condition with optimisePath(): we - might see a link count of 1 just before optimisePath() increases - the link count. */ -void LocalStore::removeUnusedLinks(const GCState & state) -{ - AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError("opening directory '%1%'", linksDir); - - int64_t actualSize = 0, unsharedSize = 0; + printInfo("deleting '%1%'", path); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = linksDir + "/" + name; + results.paths.insert(path); - auto st = lstat(path); + uint64_t bytesFreed; + deletePath(realPath, bytesFreed); + results.bytesFreed += bytesFreed; - if (st.st_nlink != 1) { - actualSize += st.st_size; - unsharedSize += (st.st_nlink - 1) * st.st_size; - continue; + if (results.bytesFreed > options.maxFreed) { + printInfo("deleted more than %d bytes; stopping", options.maxFreed); + throw GCLimitReached(); } + }; - printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); - - if (unlink(path.c_str()) == -1) - throw SysError("deleting '%1%'", path); + std::map<StorePath, StorePathSet> referrersCache; - state.results.bytesFreed += st.st_size; - } + /* Helper function that visits all paths reachable from `start` + via the referrers edges and optionally derivers and derivation + output edges. If none of those paths are roots, then all + visited paths are garbage and are deleted. */ + auto deleteReferrersClosure = [&](const StorePath & start) { + StorePathSet visited; + std::queue<StorePath> todo; - struct stat st; - if (stat(linksDir.c_str(), &st) == -1) - throw SysError("statting '%1%'", linksDir); - int64_t overhead = st.st_blocks * 512ULL; + /* Wake up any GC client waiting for deletion of the paths in + 'visited' to finish. */ + Finally releasePending([&]() { + auto shared(_shared.lock()); + shared->pending.reset(); + wakeup.notify_all(); + }); - printInfo("note: currently hard linking saves %.2f MiB", - ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); -} + auto enqueue = [&](const StorePath & path) { + if (visited.insert(path).second) + todo.push(path); + }; + enqueue(start); -void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) -{ - GCState state(options, results); - state.gcKeepOutputs = settings.gcKeepOutputs; - state.gcKeepDerivations = settings.gcKeepDerivations; + while (auto path = pop(todo)) { + checkInterrupt(); - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `keep-outputs' or `keep-derivations' are true - (the garbage collector will recurse into deleting the outputs - or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - state.gcKeepOutputs = false; - state.gcKeepDerivations = false; - } + /* Bail out if we've previously discovered that this path + is alive. */ + if (alive.count(*path)) { + alive.insert(start); + return; + } - state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + /* If we've previously deleted this path, we don't have to + handle it again. */ + if (dead.count(*path)) continue; - if (state.shouldDelete) - deletePath(reservedPath); + auto markAlive = [&]() + { + alive.insert(*path); + alive.insert(start); + try { + StorePathSet closure; + computeFSClosure(*path, closure); + for (auto & p : closure) + alive.insert(p); + } catch (InvalidPath &) { } + }; + + /* If this is a root, bail out. */ + if (roots.count(*path)) { + debug("cannot delete '%s' because it's a root", printStorePath(*path)); + return markAlive(); + } - /* Acquire the global GC root. This prevents - a) New roots from being added. - b) Processes from creating new temporary root files. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); + if (options.action == GCOptions::gcDeleteSpecific + && !options.pathsToDelete.count(*path)) + return; - /* Find the roots. Since we've grabbed the GC lock, the set of - permanent roots cannot increase now. */ - printInfo("finding garbage collector roots..."); - Roots rootMap; - if (!options.ignoreLiveness) - findRootsNoTemp(rootMap, true); + { + auto hashPart = std::string(path->hashPart()); + auto shared(_shared.lock()); + if (shared->tempRoots.count(hashPart)) { + debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); + return markAlive(); + } + shared->pending = hashPart; + } - for (auto & i : rootMap) state.roots.insert(i.first); + if (isValidPath(*path)) { - /* Read the temporary roots. This acquires read locks on all - per-process temporary root files. So after this point no paths - can be added to the set of temporary roots. */ - FDs fds; - Roots tempRoots; - findTempRoots(fds, tempRoots, true); - for (auto & root : tempRoots) { - state.tempRoots.insert(root.first); - state.roots.insert(root.first); - } + /* Visit the referrers of this path. */ + auto i = referrersCache.find(*path); + if (i == referrersCache.end()) { + StorePathSet referrers; + queryReferrers(*path, referrers); + referrersCache.emplace(*path, std::move(referrers)); + i = referrersCache.find(*path); + } + for (auto & p : i->second) + enqueue(p); + + /* If keep-derivations is set and this is a + derivation, then visit the derivation outputs. */ + if (gcKeepDerivations && path->isDerivation()) { + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) + if (maybeOutPath && + isValidPath(*maybeOutPath) && + queryPathInfo(*maybeOutPath)->deriver == *path) + enqueue(*maybeOutPath); + } - /* After this point the set of roots or temporary roots cannot - increase, since we hold locks on everything. So everything - that is not reachable from `roots' is garbage. */ + /* If keep-outputs is set, then visit the derivers. */ + if (gcKeepOutputs) { + auto derivers = queryValidDerivers(*path); + for (auto & i : derivers) + enqueue(i); + } + } + } - if (state.shouldDelete) { - if (pathExists(trashDir)) deleteGarbage(state, trashDir); - try { - createDirs(trashDir); - } catch (SysError & e) { - if (e.errNo == ENOSPC) { - printInfo("note: can't create trash directory: %s", e.msg()); - state.moveToTrash = false; + for (auto & path : topoSortPaths(visited)) { + if (!dead.insert(path).second) continue; + if (shouldDelete) { + invalidatePathChecked(path); + deleteFromStore(path.to_string()); + referrersCache.erase(path); } } - } + }; - /* Now either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ + /* Synchronisation point for testing, see tests/gc-concurrent.sh. */ + if (auto p = getEnv("_NIX_TEST_GC_SYNC")) + readFile(*p); + /* Either delete all garbage paths, or just the specified + paths (for gcDeleteSpecific). */ if (options.action == GCOptions::gcDeleteSpecific) { for (auto & i : options.pathsToDelete) { - tryToDelete(state, printStorePath(i)); - if (state.dead.find(i) == state.dead.end()) + deleteReferrersClosure(i); + if (!dead.count(i)) throw Error( - "cannot delete path '%1%' since it is still alive. " - "To find out why use: " + "Cannot delete path '%1%' since it is still alive. " + "To find out why, use: " "nix-store --query --roots", printStorePath(i)); } } else if (options.maxFreed > 0) { - if (state.shouldDelete) + if (shouldDelete) printInfo("deleting garbage..."); else printInfo("determining live/dead paths..."); try { - AutoCloseDir dir(opendir(realStoreDir.get().c_str())); if (!dir) throw SysError("opening directory '%1%'", realStoreDir); - /* Read the store and immediately delete all paths that - aren't valid. When using --max-freed etc., deleting - invalid paths is preferred over deleting unreachable - paths, since unreachable paths could become reachable - again. We don't use readDirectory() here so that GCing - can start faster. */ + /* Read the store and delete all paths that are invalid or + unreachable. We don't use readDirectory() here so that + GCing can start faster. */ + auto linksName = baseNameOf(linksDir); Paths entries; struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { checkInterrupt(); string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = storeDir + "/" + name; - auto storePath = maybeParseStorePath(path); - if (storePath && isValidPath(*storePath)) - entries.push_back(path); - else - tryToDelete(state, path); - } - - dir.reset(); - - /* Now delete the unreachable valid paths. Randomise the - order in which we delete entries to make the collector - less biased towards deleting paths that come - alphabetically first (e.g. /nix/store/000...). This - matters when using --max-freed etc. */ - vector<Path> entries_(entries.begin(), entries.end()); - std::mt19937 gen(1); - std::shuffle(entries_.begin(), entries_.end(), gen); + if (name == "." || name == ".." || name == linksName) continue; - for (auto & i : entries_) - tryToDelete(state, i); + if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) + deleteReferrersClosure(*storePath); + else + deleteFromStore(name); + } } catch (GCLimitReached & e) { } } - if (state.options.action == GCOptions::gcReturnLive) { - for (auto & i : state.alive) - state.results.paths.insert(printStorePath(i)); + if (options.action == GCOptions::gcReturnLive) { + for (auto & i : alive) + results.paths.insert(printStorePath(i)); return; } - if (state.options.action == GCOptions::gcReturnDead) { - for (auto & i : state.dead) - state.results.paths.insert(printStorePath(i)); + if (options.action == GCOptions::gcReturnDead) { + for (auto & i : dead) + results.paths.insert(printStorePath(i)); return; } - /* Allow other processes to add to the store from here on. */ - fdGCLock = -1; - fds.clear(); - - /* Delete the trash directory. */ - printInfo(format("deleting '%1%'") % trashDir); - deleteGarbage(state, trashDir); - - /* Clean up the links directory. */ + /* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { printInfo("deleting unused links..."); - removeUnusedLinks(state); + + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError("opening directory '%1%'", linksDir); + + int64_t actualSize = 0, unsharedSize = 0; + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + auto st = lstat(path); + + if (st.st_nlink != 1) { + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; + continue; + } + + printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError("deleting '%1%'", path); + + results.bytesFreed += st.st_size; + } + + struct stat st; + if (stat(linksDir.c_str(), &st) == -1) + throw SysError("statting '%1%'", linksDir); + int64_t overhead = st.st_blocks * 512ULL; + + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } /* While we're at it, vacuum the database. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 9f1a88130..81ca9cc0f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -122,7 +122,7 @@ StringSet Settings::getDefaultSystemFeatures() /* For backwards compatibility, accept some "features" that are used in Nixpkgs to route builds to certain machines but don't actually require anything special on the machines. */ - StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"}; + StringSet features{"nixos-test", "benchmark", "big-parallel"}; #if __linux__ if (access("/dev/kvm", R_OK | W_OK) == 0) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 165639261..433deaf0f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -797,6 +797,15 @@ public: may be useful in certain scenarios (e.g. to spin up containers or set up userspace network interfaces in tests). )"}; + + Setting<StringSet> ignoredAcls{ + this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls", + R"( + A list of ACLs that should be ignored, normally Nix attempts to + remove all ACLs from files and directories in the Nix store, but + some ACLs like `security.selinux` or `system.nfs4_acl` can't be + removed even by root. Therefore it's best to just ignore them. + )"}; #endif Setting<Strings> hashedMirrors{ @@ -951,6 +960,9 @@ public: Setting<bool> useRegistries{this, true, "use-registries", "Whether to use flake registries to resolve flake references."}; + + Setting<bool> acceptFlakeConfig{this, false, "accept-flake-config", + "Whether to accept nix configuration from a flake without prompting."}; }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 814960bb5..4861d185e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -227,7 +227,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override + PathFilter & filter, RepairFlag repair, const StorePathSet & references) override { unsupported("addToStore"); } StorePath addTextToStore(const string & name, const string & s, @@ -367,7 +367,8 @@ public: return conn->remoteVersion; } - std::optional<const Realisation> queryRealisation(const DrvOutput&) override + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override // TODO: Implement { unsupported("queryRealisation"); } }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1e3b5482e..79011b522 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -8,6 +8,7 @@ #include "references.hh" #include "callback.hh" #include "topo-sort.hh" +#include "finally.hh" #include <iostream> #include <algorithm> @@ -145,7 +146,6 @@ LocalStore::LocalStore(const Params & params) , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") - , trashDir(realStoreDir + "/trash") , tempRootsDir(stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) , locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or(""))) @@ -386,6 +386,16 @@ LocalStore::LocalStore(const Params & params) } +AutoCloseFD LocalStore::openGCLock() +{ + Path fnGCLock = stateDir + "/gc.lock"; + auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fdGCLock) + throw SysError("opening global GC lock '%1%'", fnGCLock); + return fdGCLock; +} + + LocalStore::~LocalStore() { std::shared_future<void> future; @@ -495,9 +505,6 @@ void LocalStore::makeStoreWritable() throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) throw SysError("remounting %1% writable", realStoreDir); } @@ -583,9 +590,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe throw SysError("querying extended attributes of '%s'", path); for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - /* Ignore SELinux security labels since these cannot be - removed even by root. */ - if (eaName == "security.selinux") continue; + if (settings.ignoredAcls.get().count(eaName)) continue; if (lremovexattr(path.c_str(), eaName.c_str()) == -1) throw SysError("removing extended attribute '%s' from '%s'", eaName, path); } @@ -825,7 +830,7 @@ uint64_t LocalStore::addValidPath(State & state, { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(std::string(info.path.hashPart()), + state_->pathInfoCache.upsert(std::string(info.path.to_string()), PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) }); } @@ -1198,7 +1203,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(std::string(path.hashPart())); + state_->pathInfoCache.erase(std::string(path.to_string())); } } @@ -1302,7 +1307,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { /* For computing the store path. */ auto hashSink = std::make_unique<HashSink>(hashAlgo); @@ -1327,13 +1332,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto want = std::min(chunkSize, settings.narBufferSize - oldSize); dump.resize(oldSize + want); auto got = 0; + Finally cleanup([&]() { + dump.resize(oldSize + got); + }); try { got = source.read(dump.data() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; } - dump.resize(oldSize + got); } std::unique_ptr<AutoDelete> delTempDir; @@ -1358,7 +1365,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name); + auto dstPath = makeFixedOutputPath(method, hash, name, references); addTempRoot(dstPath); @@ -1405,6 +1412,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, ValidPathInfo info { dstPath, narHash.first }; info.narSize = narHash.second; + info.references = references; info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1505,7 +1513,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Acquire the global GC lock to get a consistent snapshot of existing and valid paths. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); + auto fdGCLock = openGCLock(); + FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); StringSet store; for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); @@ -1516,8 +1525,6 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) StorePathSet validPaths; PathSet done; - fdGCLock = -1; - for (auto & i : queryAllValidPaths()) verifyPath(printStorePath(i), store, done, validPaths, repair, errors); @@ -1830,13 +1837,24 @@ std::optional<const Realisation> LocalStore::queryRealisation_( return { res }; } -std::optional<const Realisation> -LocalStore::queryRealisation(const DrvOutput & id) +void LocalStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { - return retrySQLite<std::optional<const Realisation>>([&]() { - auto state(_state.lock()); - return queryRealisation_(*state, id); - }); + try { + auto maybeRealisation + = retrySQLite<std::optional<const Realisation>>([&]() { + auto state(_state.lock()); + return queryRealisation_(*state, id); + }); + if (maybeRealisation) + callback( + std::make_shared<const Realisation>(maybeRealisation.value())); + else + callback(nullptr); + + } catch (...) { + callback.rethrow(); + } } FixedOutputHash LocalStore::hashCAPath( diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a01d48c4b..115ea046a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,9 +58,15 @@ private: struct Stmts; std::unique_ptr<Stmts> stmts; + /* The global GC lock */ + AutoCloseFD fdGCLock; + /* The file to which we write our temporary roots. */ AutoCloseFD fdTempRoots; + /* Connection to the garbage collector. */ + AutoCloseFD fdRootsSocket; + /* The last time we checked whether to do an auto-GC, or an auto-GC finished. */ std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; @@ -87,7 +93,6 @@ public: const Path linksDir; const Path reservedPath; const Path schemaPath; - const Path trashDir; const Path tempRootsDir; const Path fnTempRoots; @@ -140,7 +145,7 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; @@ -149,14 +154,11 @@ public: void addIndirectRoot(const Path & path) override; - void syncWithGC() override; - private: - typedef std::shared_ptr<AutoCloseFD> FDPtr; - typedef list<FDPtr> FDs; + void findTempRoots(Roots & roots, bool censor); - void findTempRoots(FDs & fds, Roots & roots, bool censor); + AutoCloseFD openGCLock(); public: @@ -205,7 +207,8 @@ public: std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id); std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id); - std::optional<const Realisation> queryRealisation(const DrvOutput&) override; + void queryRealisationUncached(const DrvOutput&, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; private: @@ -236,29 +239,12 @@ private: PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); - struct GCState; - - void deleteGarbage(GCState & state, const Path & path); - - void tryToDelete(GCState & state, const Path & path); - - bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path); - - void deletePathRecursive(GCState & state, const Path & path); - - bool isActiveTempFile(const GCState & state, - const Path & path, const string & suffix); - - AutoCloseFD openGCLock(LockType lockType); - void findRoots(const Path & path, unsigned char type, Roots & roots); void findRootsNoTemp(Roots & roots, bool censor); void findRuntimeRoots(Roots & roots, bool censor); - void removeUnusedLinks(const GCState & state); - Path createTempDirInStore(); void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 9843ccf04..b6270a81b 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -39,7 +39,8 @@ Machine::Machine(decltype(storeUri) storeUri, sshPublicHostKey(sshPublicHostKey) {} -bool Machine::allSupported(const std::set<string> & features) const { +bool Machine::allSupported(const std::set<string> & features) const +{ return std::all_of(features.begin(), features.end(), [&](const string & feature) { return supportedFeatures.count(feature) || @@ -47,14 +48,16 @@ bool Machine::allSupported(const std::set<string> & features) const { }); } -bool Machine::mandatoryMet(const std::set<string> & features) const { +bool Machine::mandatoryMet(const std::set<string> & features) const +{ return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), [&](const string & feature) { return features.count(feature); }); } -ref<Store> Machine::openStore() const { +ref<Store> Machine::openStore() const +{ Store::Params storeParams; if (hasPrefix(storeUri, "ssh://")) { storeParams["max-connections"] = "1"; @@ -83,53 +86,86 @@ ref<Store> Machine::openStore() const { return nix::openStore(storeUri, storeParams); } -void parseMachines(const std::string & s, Machines & machines) +static std::vector<std::string> expandBuilderLines(const std::string & builders) { - for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { + std::vector<std::string> result; + for (auto line : tokenizeString<std::vector<string>>(builders, "\n;")) { trim(line); line.erase(std::find(line.begin(), line.end(), '#'), line.end()); if (line.empty()) continue; if (line[0] == '@') { - auto file = trim(std::string(line, 1)); + const std::string path = trim(std::string(line, 1)); + std::string text; try { - parseMachines(readFile(file), machines); + text = readFile(path); } catch (const SysError & e) { if (e.errNo != ENOENT) throw; - debug("cannot find machines file '%s'", file); + debug("cannot find machines file '%s'", path); } + + const auto lines = expandBuilderLines(text); + result.insert(end(result), begin(lines), end(lines)); continue; } - auto tokens = tokenizeString<std::vector<string>>(line); - auto sz = tokens.size(); - if (sz < 1) - throw FormatError("bad machine specification '%s'", line); + result.emplace_back(line); + } + return result; +} - auto isSet = [&](size_t n) { - return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; - }; +static Machine parseBuilderLine(const std::string & line) +{ + const auto tokens = tokenizeString<std::vector<string>>(line); - machines.emplace_back(tokens[0], - isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, - isSet(2) ? tokens[2] : "", - isSet(3) ? std::stoull(tokens[3]) : 1LL, - isSet(4) ? std::stoull(tokens[4]) : 1LL, - isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, - isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, - isSet(7) ? tokens[7] : ""); - } + auto isSet = [&](size_t fieldIndex) { + return tokens.size() > fieldIndex && tokens[fieldIndex] != "" && tokens[fieldIndex] != "-"; + }; + + auto parseUnsignedIntField = [&](size_t fieldIndex) { + const auto result = string2Int<unsigned int>(tokens[fieldIndex]); + if (!result) { + throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'unsigned int'", fieldIndex, line); + } + return result.value(); + }; + + auto ensureBase64 = [&](size_t fieldIndex) { + const auto & str = tokens[fieldIndex]; + try { + base64Decode(str); + } catch (const Error & e) { + throw FormatError("bad machine specification: a column #%lu in a row: '%s' is not valid base64 string: %s", fieldIndex, line, e.what()); + } + return str; + }; + + if (!isSet(0)) + throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line); + + return { + tokens[0], + isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, + isSet(2) ? tokens[2] : "", + isSet(3) ? parseUnsignedIntField(3) : 1U, + isSet(4) ? parseUnsignedIntField(4) : 1U, + isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, + isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, + isSet(7) ? ensureBase64(7) : "" + }; +} + +static Machines parseBuilderLines(const std::vector<std::string>& builders) { + Machines result; + std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine); + return result; } Machines getMachines() { - static auto machines = [&]() { - Machines machines; - parseMachines(settings.builders, machines); - return machines; - }(); - return machines; + const auto builderLines = expandBuilderLines(settings.builders); + return parseBuilderLines(builderLines); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f184dd857..32786e963 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -239,12 +239,11 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) { return topoSort(paths, {[&](const StorePath & path) { - StorePathSet references; try { - references = queryPathInfo(path)->references; + return queryPathInfo(path)->references; } catch (InvalidPath &) { + return StorePathSet(); } - return references; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( diff --git a/src/libstore/names.cc b/src/libstore/names.cc index ce808accc..54c95055d 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -42,7 +42,7 @@ DrvName::~DrvName() { } -bool DrvName::matches(DrvName & n) +bool DrvName::matches(const DrvName & n) { if (name != "*") { if (!regex) { diff --git a/src/libstore/names.hh b/src/libstore/names.hh index bc62aac93..3f861bc44 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -19,7 +19,7 @@ struct DrvName DrvName(std::string_view s); ~DrvName(); - bool matches(DrvName & n); + bool matches(const DrvName & n); private: std::unique_ptr<Regex> regex; diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 926f4ea1e..2da74e262 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -176,4 +176,17 @@ void PathLocks::setDeletion(bool deletePaths) } +FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg) + : fd(fd) +{ + if (wait) { + if (!lockFile(fd, lockType, false)) { + printInfo("%s", waitMsg); + acquired = lockFile(fd, lockType, true); + } + } else + acquired = lockFile(fd, lockType, false); +} + + } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 411da0222..919c8904c 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -35,4 +35,18 @@ public: void setDeletion(bool deletePaths); }; +struct FdLock +{ + int fd; + bool acquired = false; + + FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg); + + ~FdLock() + { + if (acquired) + lockFile(fd, ltNone, false); + } +}; + } diff --git a/src/libstore/references.cc b/src/libstore/references.cc index c369b14ac..91b3fc142 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -54,12 +54,12 @@ void RefScanSink::operator () (std::string_view data) fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ auto s = tail; - s.append(data.data(), refLength); + auto tailLen = std::min(data.size(), refLength); + s.append(data.data(), tailLen); search(s, hashes, seen); search(data, hashes, seen); - auto tailLen = std::min(data.size(), refLength); auto rest = refLength - tailLen; if (rest < tail.size()) tail = tail.substr(tail.size() - rest); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index fa5ea8af7..7f7e973e9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -290,6 +290,10 @@ ConnectionHandle RemoteStore::getConnection() return ConnectionHandle(connections->get()); } +void RemoteStore::setOptions() +{ + setOptions(*(getConnection().handle)); +} bool RemoteStore::isValidPathUncached(const StorePath & path) { @@ -578,9 +582,8 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashType, RepairFlag repair) + FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) { - StorePathSet references; return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; } @@ -677,23 +680,41 @@ void RemoteStore::registerDrvOutput(const Realisation & info) conn.processStderr(); } -std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id) +void RemoteStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); + try { + callback(nullptr); + } catch (...) { return callback.rethrow(); } + } + conn->to << wopQueryRealisation; conn->to << id.to_string(); conn.processStderr(); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { - auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{}); - if (outPaths.empty()) - return std::nullopt; - return {Realisation{.id = id, .outPath = *outPaths.begin()}}; - } else { - auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{}); - if (realisations.empty()) - return std::nullopt; - return *realisations.begin(); - } + + auto real = [&]() -> std::shared_ptr<const Realisation> { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + auto outPaths = worker_proto::read( + *this, conn->from, Phantom<std::set<StorePath>> {}); + if (outPaths.empty()) + return nullptr; + return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() }); + } else { + auto realisations = worker_proto::read( + *this, conn->from, Phantom<std::set<Realisation>> {}); + if (realisations.empty()) + return nullptr; + return std::make_shared<const Realisation>(*realisations.begin()); + } + }(); + + try { + callback(std::shared_ptr<const Realisation>(real)); + } catch (...) { return callback.rethrow(); } } static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs) @@ -797,15 +818,6 @@ void RemoteStore::addIndirectRoot(const Path & path) } -void RemoteStore::syncWithGC() -{ - auto conn(getConnection()); - conn->to << wopSyncWithGC; - conn.processStderr(); - readInt(conn->from); -} - - Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index ac1eaa19e..0fd67f371 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -73,7 +73,7 @@ public: /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; @@ -88,7 +88,8 @@ public: void registerDrvOutput(const Realisation & info) override; - std::optional<const Realisation> queryRealisation(const DrvOutput &) override; + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override; @@ -101,8 +102,6 @@ public: void addIndirectRoot(const Path & path) override; - void syncWithGC() override; - Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -149,6 +148,8 @@ protected: virtual void setOptions(Connection & conn); + void setOptions() override; + ConnectionHandle getConnection(); friend struct ConnectionHandle; diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 41893e6dd..56b35c3fe 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -100,4 +100,5 @@ ; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin. (allow file-read* - (subpath "/Library/Apple/usr/libexec/oah")) + (subpath "/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/Apple/usr/libexec/oah")) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 447b4179b..1d6baf02d 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,4 +1,5 @@ #include "sqlite.hh" +#include "globals.hh" #include "util.hh" #include <sqlite3.h> @@ -27,8 +28,12 @@ namespace nix { SQLite::SQLite(const Path & path, bool create) { + // useSQLiteWAL also indicates what virtual file system we need. Using + // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem + // for Linux (WSL) where useSQLiteWAL should be false by default. + const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK) throw Error("cannot open SQLite database '%s'", path); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3338cdc1b..aab4ce94c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -237,7 +237,7 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, StorePath Store::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references) { Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { @@ -246,7 +246,7 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath, else readFile(srcPath, sink); }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair); + return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); } @@ -355,8 +355,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, StringSet StoreConfig::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) res.insert("ca-derivations"); + + if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix)) + res.insert("recursive-nix"); + return res; } @@ -414,11 +419,9 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) bool Store::isValidPath(const StorePath & storePath) { - std::string hashPart(storePath.hashPart()); - { auto state_(state.lock()); - auto res = state_->pathInfoCache.get(hashPart); + auto res = state_->pathInfoCache.get(std::string(storePath.to_string())); if (res && res->isKnownNow()) { stats.narInfoReadAverted++; return res->didExist(); @@ -426,11 +429,11 @@ bool Store::isValidPath(const StorePath & storePath) } if (diskCache) { - auto res = diskCache->lookupNarInfo(getUri(), hashPart); + auto res = diskCache->lookupNarInfo(getUri(), std::string(storePath.hashPart())); if (res.first != NarInfoDiskCache::oUnknown) { stats.narInfoReadAverted++; auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, + state_->pathInfoCache.upsert(std::string(storePath.to_string()), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second }); return res.first == NarInfoDiskCache::oValid; } @@ -440,7 +443,7 @@ bool Store::isValidPath(const StorePath & storePath) if (diskCache && !valid) // FIXME: handle valid = true case. - diskCache->upsertNarInfo(getUri(), hashPart, 0); + diskCache->upsertNarInfo(getUri(), std::string(storePath.hashPart()), 0); return valid; } @@ -487,13 +490,11 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual) void Store::queryPathInfo(const StorePath & storePath, Callback<ref<const ValidPathInfo>> callback) noexcept { - std::string hashPart; + auto hashPart = std::string(storePath.hashPart()); try { - hashPart = storePath.hashPart(); - { - auto res = state.lock()->pathInfoCache.get(hashPart); + auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string())); if (res && res->isKnownNow()) { stats.narInfoReadAverted++; if (!res->didExist()) @@ -508,7 +509,7 @@ void Store::queryPathInfo(const StorePath & storePath, stats.narInfoReadAverted++; { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, + state_->pathInfoCache.upsert(std::string(storePath.to_string()), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || !goodStorePath(storePath, res.second->path)) @@ -523,7 +524,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { + {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -533,14 +534,12 @@ void Store::queryPathInfo(const StorePath & storePath, { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); + state_->pathInfoCache.upsert(std::string(storePath.to_string()), PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref<const ValidPathInfo>(info)); @@ -548,6 +547,74 @@ void Store::queryPathInfo(const StorePath & storePath, }}); } +void Store::queryRealisation(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept +{ + + try { + if (diskCache) { + auto [cacheOutcome, maybeCachedRealisation] + = diskCache->lookupRealisation(getUri(), id); + switch (cacheOutcome) { + case NarInfoDiskCache::oValid: + debug("Returning a cached realisation for %s", id.to_string()); + callback(maybeCachedRealisation); + return; + case NarInfoDiskCache::oInvalid: + debug( + "Returning a cached missing realisation for %s", + id.to_string()); + callback(nullptr); + return; + case NarInfoDiskCache::oUnknown: + break; + } + } + } catch (...) { + return callback.rethrow(); + } + + auto callbackPtr + = std::make_shared<decltype(callback)>(std::move(callback)); + + queryRealisationUncached( + id, + { [this, id, callbackPtr]( + std::future<std::shared_ptr<const Realisation>> fut) { + try { + auto info = fut.get(); + + if (diskCache) { + if (info) + diskCache->upsertRealisation(getUri(), *info); + else + diskCache->upsertAbsentRealisation(getUri(), id); + } + + (*callbackPtr)(std::shared_ptr<const Realisation>(info)); + + } catch (...) { + callbackPtr->rethrow(); + } + } }); +} + +std::shared_ptr<const Realisation> Store::queryRealisation(const DrvOutput & id) +{ + using RealPtr = std::shared_ptr<const Realisation>; + std::promise<RealPtr> promise; + + queryRealisation(id, + {[&](std::future<RealPtr> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + + return promise.get_future().get(); +} void Store::substitutePaths(const StorePathSet & paths) { @@ -1012,7 +1079,7 @@ std::map<StorePath, StorePath> copyPaths( nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what())); + printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what()); showProgress(); return; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 54471bdf2..aa44651d4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -232,7 +232,6 @@ protected: struct State { - // FIXME: fix key LRUCache<std::string, PathInfoCacheValue> pathInfoCache; }; @@ -370,6 +369,14 @@ public: void queryPathInfo(const StorePath & path, Callback<ref<const ValidPathInfo>> callback) noexcept; + /* Query the information about a realisation. */ + std::shared_ptr<const Realisation> queryRealisation(const DrvOutput &); + + /* Asynchronous version of queryRealisation(). */ + void queryRealisation(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept; + + /* Check whether the given valid path info is sufficiently attested, by either being signed by a trusted public key or content-addressed, in order to be included in the given store. @@ -394,11 +401,11 @@ protected: virtual void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0; + virtual void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept = 0; public: - virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0; - /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) @@ -453,7 +460,7 @@ public: libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()); /* Copy the contents of a path to the store and register the validity the resulting path, using a constant amount of @@ -469,7 +476,8 @@ public: `dump` may be drained */ // FIXME: remove? virtual StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } /* Like addToStore, but the contents written to the output path is @@ -561,26 +569,6 @@ public: virtual void addIndirectRoot(const Path & path) { unsupported("addIndirectRoot"); } - /* Acquire the global GC lock, then immediately release it. This - function must be called after registering a new permanent root, - but before exiting. Otherwise, it is possible that a running - garbage collector doesn't see the new root and deletes the - stuff we've just built. By acquiring the lock briefly, we - ensure that either: - - - The collector is already running, and so we block until the - collector is finished. The collector will know about our - *temporary* locks, which should include whatever it is we - want to register as a permanent lock. - - - The collector isn't running, or it's just started but hasn't - acquired the GC lock yet. In that case we get and release - the lock right away, then exit. The collector scans the - permanent root and sees ours. - - In either case the permanent root is seen by the collector. */ - virtual void syncWithGC() { }; - /* Find the roots of the garbage collector. Each root is a pair (link, storepath) where `link' is the path of the symlink outside of the Nix store that point to `storePath'. If @@ -745,6 +733,11 @@ public: virtual void createUser(const std::string & userName, uid_t userId) { } + /* + * Synchronises the options of the client with those of the daemon + * (a no-op when there’s no daemon) + */ + virtual void setOptions() { } protected: Stats stats; diff --git a/src/libstore/tests/machines.cc b/src/libstore/tests/machines.cc new file mode 100644 index 000000000..f51052b14 --- /dev/null +++ b/src/libstore/tests/machines.cc @@ -0,0 +1,169 @@ +#include "machines.hh" +#include "globals.hh" + +#include <gmock/gmock-matchers.h> + +using testing::Contains; +using testing::ElementsAre; +using testing::EndsWith; +using testing::Eq; +using testing::Field; +using testing::SizeIs; + +using nix::absPath; +using nix::FormatError; +using nix::getMachines; +using nix::Machine; +using nix::Machines; +using nix::pathExists; +using nix::Settings; +using nix::settings; + +class Environment : public ::testing::Environment { + public: + void SetUp() override { settings.thisSystem = "TEST_ARCH-TEST_OS"; } +}; + +testing::Environment* const foo_env = + testing::AddGlobalTestEnvironment(new Environment); + +TEST(machines, getMachinesWithEmptyBuilders) { + settings.builders = ""; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesUriOnly) { + settings.builders = "nix@scratchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); +} + +TEST(machines, getMachinesDefaults) { + settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); +} + +TEST(machines, getMachinesWithNewLineSeparator) { + settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(2)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithSemicolonSeparator) { + settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + EXPECT_THAT(actual, SizeIs(2)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { + settings.builders = "nix@scratchy.labs.cs.uu.nl i686-linux " + "/home/nix/.ssh/id_scratchy_auto 8 3 kvm " + "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark"))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, Eq("SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="))); +} + +TEST(machines, + getMachinesWithCorrectCompleteSingleBuilderWithTabColumnDelimiter) { + settings.builders = + "nix@scratchy.labs.cs.uu.nl\ti686-linux\t/home/nix/.ssh/" + "id_scratchy_auto\t8\t3\tkvm\tbenchmark\tSSH+HOST+PUBLIC+" + "KEY+BASE64+ENCODED=="; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark"))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, Eq("SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="))); +} + +TEST(machines, getMachinesWithMultiOptions) { + settings.builders = "nix@scratchy.labs.cs.uu.nl Arch1,Arch2 - - - " + "SupportedFeature1,SupportedFeature2 " + "MandatoryFeature1,MandatoryFeature2"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2"))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2"))); +} + +TEST(machines, getMachinesWithIncorrectFormat) { + settings.builders = "nix@scratchy.labs.cs.uu.nl - - eight"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - -1"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 three"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 -3"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 3 - - BAD_BASE64"; + EXPECT_THROW(getMachines(), FormatError); +} + +TEST(machines, getMachinesWithCorrectFileReference) { + auto path = absPath("src/libstore/tests/test-data/machines.valid"); + ASSERT_TRUE(pathExists(path)); + + settings.builders = std::string("@") + path; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(3)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { + auto path = "/dev/null"; + ASSERT_TRUE(pathExists(path)); + + settings.builders = std::string("@") + path; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesWithIncorrectFileReference) { + settings.builders = std::string("@") + absPath("/not/a/file"); + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { + settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); + EXPECT_THROW(getMachines(), FormatError); +} diff --git a/src/libstore/tests/test-data/machines.bad_format b/src/libstore/tests/test-data/machines.bad_format new file mode 100644 index 000000000..7255a1216 --- /dev/null +++ b/src/libstore/tests/test-data/machines.bad_format @@ -0,0 +1 @@ +nix@scratchy.labs.cs.uu.nl - - eight diff --git a/src/libstore/tests/test-data/machines.valid b/src/libstore/tests/test-data/machines.valid new file mode 100644 index 000000000..1a6c8017c --- /dev/null +++ b/src/libstore/tests/test-data/machines.valid @@ -0,0 +1,3 @@ +nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm +nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 +nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFDQVFDWWV5R1laNTNzd1VjMUZNSHBWL1BCcXlKaFR5S1JoRkpWWVRpRHlQN2h5c1JGa0w4VDlLOGdhL2Y2L3c3QjN2SjNHSFRIUFkybENiUEdZbGNLd2h6M2ZRbFNNOEViNi95b3ZLajdvM1FsMEx5Y0dzdGJvRmcwWkZKNldncUxsR0ltS0NobUlxOGZ3TW5ZTWUxbnRQeTBUZFZjSU1tOTV3YzF3SjBMd2c3cEVMRmtHazdkeTVvYnM4a3lGZ0pORDVRSmFwQWJjeWp4Z1QzdzdMcktNZ2xzeWhhd01JNVpkMGZsQTVudW5OZ3pid3plYVhLaUsyTW0vdGJXYTU1YTd4QmNYdHpIZGlPSWdSajJlRWxaMGh5bk10YjBmcklsdmxIcEtLaVFaZ3pQdCtIVXQ2bXpRMkRVME52MGYyYnNSU0krOGpJU2pQcmdlcVVHRldMUzVIUTg2N2xSMlpiaWtyclhZNTdqbVFEZk5DRHY1VFBHZU9UekFEd2pjMDc2aFZ3VFJCd3VTZFhtaWNxTS95b3lrWitkV1dnZ25MenE5QU1tdlNZcDhmZkZDcS9CSDBZNUFXWTFHay9vS3hMVTNaOWt3ZDd2UWNFQWFCQ2dxdnVZRGdTaHE1RlhndDM3OVZESWtEL05ZSTg2QXVvajVDRmVNTzlRM2pJSlRadlh6c1VldjVoSnA2djcxSVh5ODVtbTY5R20zcXdicVE1SjVQZDU1Um56SitpaW5BNjZxTEFSc0Y4amNsSnd5ekFXclBoYU9DRVY2bjVMeVhVazhzMW9EVVR4V1pWN25rVkFTbHJ0MllGcjN5dzdjRTRXQVhsemhHcDhocmdLMVVkMUlyeDVnZWRaSnBWcy9uNWVybmJFMUxmb2x5UHUvRUFIWlh6VGd4dHVDUFNobXc9PQo= diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 02e81b022..5c38323cd 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -56,14 +56,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() auto conn = make_ref<Connection>(); /* Connect to a daemon that does the privileged work for us. */ - conn->fd = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!conn->fd) - throw SysError("cannot create Unix domain socket"); - closeOnExec(conn->fd.get()); + conn->fd = createUnixDomainSocket(); nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile); diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 96ad69790..ce9c3dfed 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -189,13 +189,14 @@ extern Verbosity verbosity; /* suppress msgs > this */ /* Print a string message if the current log level is at least the specified level. Note that this has to be implemented as a macro to ensure that the arguments are evaluated lazily. */ -#define printMsg(level, args...) \ +#define printMsgUsing(loggerParam, level, args...) \ do { \ auto __lvl = level; \ if (__lvl <= nix::verbosity) { \ - logger->log(__lvl, fmt(args)); \ + loggerParam->log(__lvl, fmt(args)); \ } \ } while (0) +#define printMsg(level, args...) printMsgUsing(logger, level, args) #define printError(args...) printMsg(lvlError, args) #define notice(args...) printMsg(lvlNotice, args) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 50e691a3d..790bc943a 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -93,9 +93,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir) else archive.check(r); - archive_entry_set_pathname(entry, + archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str()); + // Patch hardlink path + const char *original_hardlink = archive_entry_hardlink(entry); + if (original_hardlink) { + archive_entry_copy_hardlink(entry, + (destDir + "/" + original_hardlink).c_str()); + } + archive.check(archive_read_extract(archive.archive, entry, flags)); } diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 58df9c5ac..92972ed14 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -4,6 +4,8 @@ #include <limits.h> #include <gtest/gtest.h> +#include <numeric> + namespace nix { /* ----------- tests for util.hh ------------------------------------------------*/ @@ -282,6 +284,17 @@ namespace nix { ASSERT_EQ(decoded, s); } + TEST(base64Encode, encodeAndDecodeNonPrintable) { + char s[256]; + std::iota(std::rbegin(s), std::rend(s), 0); + + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + EXPECT_EQ(decoded.length(), 255); + ASSERT_EQ(decoded, s); + } + /* ---------------------------------------------------------------------------- * base64Decode * --------------------------------------------------------------------------*/ @@ -294,6 +307,10 @@ namespace nix { ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); } + TEST(base64Decode, decodeThrowsOnInvalidChar) { + ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); + } + /* ---------------------------------------------------------------------------- * toLower * --------------------------------------------------------------------------*/ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 563a72c12..1b6467eb2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -512,6 +512,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); if (!fd) throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); return {std::move(fd), tmpl}; } @@ -562,7 +563,7 @@ Path getConfigDir() std::vector<Path> getConfigDirs() { Path configHome = getConfigDir(); - string configDirs = getEnv("XDG_CONFIG_DIRS").value_or(""); + string configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":"); result.insert(result.begin(), configHome); return result; @@ -1205,7 +1206,7 @@ void closeOnExec(int fd) ////////////////////////////////////////////////////////////////////// -bool _isInterrupted = false; +std::atomic<bool> _isInterrupted = false; static thread_local bool interruptThrown = false; thread_local std::function<bool()> interruptCheck; @@ -1436,8 +1437,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in } -static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -static std::array<char, 256> base64DecodeChars; +constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; string base64Encode(std::string_view s) { @@ -1462,12 +1462,15 @@ string base64Encode(std::string_view s) string base64Decode(std::string_view s) { - static std::once_flag flag; - std::call_once(flag, [](){ - base64DecodeChars = { (char)-1 }; + constexpr char npos = -1; + constexpr std::array<char, 256> base64DecodeChars = [&]() { + std::array<char, 256> result{}; + for (auto& c : result) + c = npos; for (int i = 0; i < 64; i++) - base64DecodeChars[(int) base64Chars[i]] = i; - }); + result[base64Chars[i]] = i; + return result; + }(); string res; unsigned int d = 0, bits = 0; @@ -1477,7 +1480,7 @@ string base64Decode(std::string_view s) if (c == '\n') continue; char digit = base64DecodeChars[(unsigned char) c]; - if (digit == -1) + if (digit == npos) throw Error("invalid character in Base64 string: '%c'", c); bits += 6; @@ -1630,9 +1633,39 @@ void setStackSize(size_t stackSize) #endif } -void restoreProcessContext() +static AutoCloseFD fdSavedMountNamespace; + +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); + if (!fd) + throw SysError("saving parent mount namespace"); + fdSavedMountNamespace = std::move(fd); + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + try { + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + } catch (Error & e) { + debug(e.msg()); + } +#endif +} + +void restoreProcessContext(bool restoreMounts) { restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } restoreAffinity(); @@ -1670,7 +1703,7 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> } -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +AutoCloseFD createUnixDomainSocket() { AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM #ifdef SOCK_CLOEXEC @@ -1679,8 +1712,14 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) , 0); if (!fdSocket) throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); + return fdSocket; +} + + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); bind(fdSocket.get(), path); @@ -1709,7 +1748,7 @@ void bind(int fd, const std::string & path) std::string base(baseNameOf(path)); if (base.size() + 1 >= sizeof(addr.sun_path)) throw Error("socket path '%s' is too long", base); - strcpy(addr.sun_path, base.c_str()); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError("cannot bind to socket '%s'", path); _exit(0); @@ -1718,7 +1757,7 @@ void bind(int fd, const std::string & path) if (status != 0) throw Error("cannot bind to socket '%s'", path); } else { - strcpy(addr.sun_path, path.c_str()); + memcpy(addr.sun_path, path.c_str(), path.size() + 1); if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError("cannot bind to socket '%s'", path); } @@ -1738,7 +1777,7 @@ void connect(int fd, const std::string & path) std::string base(baseNameOf(path)); if (base.size() + 1 >= sizeof(addr.sun_path)) throw Error("socket path '%s' is too long", base); - strcpy(addr.sun_path, base.c_str()); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError("cannot connect to socket at '%s'", path); _exit(0); @@ -1747,7 +1786,7 @@ void connect(int fd, const std::string & path) if (status != 0) throw Error("cannot connect to socket at '%s'", path); } else { - strcpy(addr.sun_path, path.c_str()); + memcpy(addr.sun_path, path.c_str(), path.size() + 1); if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError("cannot connect to socket at '%s'", path); } @@ -1766,7 +1805,7 @@ void commonChildInit(Pipe & logPipe) logger = makeSimpleLogger(); const static string pathNullDevice = "/dev/null"; - restoreProcessContext(); + restoreProcessContext(false); /* Put the child in a separate session (and thus a separate process group) so that it has no controlling terminal (meaning diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 29232453f..bc96bfed1 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -11,6 +11,7 @@ #include <unistd.h> #include <signal.h> +#include <atomic> #include <functional> #include <map> #include <sstream> @@ -300,7 +301,15 @@ void setStackSize(size_t stackSize); /* Restore the original inherited Unix process context (such as signal masks, stack size, CPU affinity). */ -void restoreProcessContext(); +void restoreProcessContext(bool restoreMounts = true); + +/* Save the current mount namespace. Ignored if called more than + once. */ +void saveMountNamespace(); + +/* Restore the mount namespace saved by saveMountNamespace(). Ignored + if saveMountNamespace() was never called. */ +void restoreMountNamespace(); class ExecError : public Error @@ -329,7 +338,7 @@ void closeOnExec(int fd); /* User interruption. */ -extern bool _isInterrupted; +extern std::atomic<bool> _isInterrupted; extern thread_local std::function<bool()> interruptCheck; @@ -511,6 +520,29 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_ } +/* Remove and return the first item from a container. */ +template <class T> +std::optional<typename T::value_type> remove_begin(T & c) +{ + auto i = c.begin(); + if (i == c.end()) return {}; + auto v = std::move(*i); + c.erase(i); + return v; +} + + +/* Remove and return the first item from a container. */ +template <class T> +std::optional<typename T::value_type> pop(T & c) +{ + if (c.empty()) return {}; + auto v = std::move(c.front()); + c.pop(); + return v; +} + + template<typename T> class Callback; @@ -571,6 +603,9 @@ extern PathFilter defaultPathFilter; /* Common initialisation performed in child processes. */ void commonChildInit(Pipe & logPipe); +/* Create a Unix domain socket. */ +AutoCloseFD createUnixDomainSocket(); + /* Create a Unix domain socket in listen mode. */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 73d93480e..e2325c91f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -105,7 +105,8 @@ static void main_nix_build(int argc, char * * argv) // List of environment variables kept for --pure std::set<string> keepVars{ - "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", + "HOME", "XDG_RUNTIME_DIR", "USER", "LOGNAME", "DISPLAY", + "WAYLAND_DISPLAY", "WAYLAND_SOCKET", "PATH", "TERM", "IN_NIX_SHELL", "NIX_SHELL_PRESERVE_PROMPT", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL", "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy" }; @@ -358,6 +359,7 @@ static void main_nix_build(int argc, char * * argv) is not set, then build bashInteractive from <nixpkgs>. */ auto shell = getEnv("NIX_BUILD_SHELL"); + std::optional<StorePath> shellDrv; if (!shell) { @@ -374,8 +376,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = store->parseStorePath(drv->queryDrvPath()); pathsToBuild.push_back({bashDrv}); pathsToCopy.insert(bashDrv); - - shell = drv->queryOutPath() + "/bin/bash"; + shellDrv = bashDrv; } catch (Error & e) { logError(e.info()); @@ -401,6 +402,11 @@ static void main_nix_build(int argc, char * * argv) if (dryRun) return; + if (shellDrv) { + auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value()); + shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; + } + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { auto resolvedDrv = drv.tryResolve(*store); assert(resolvedDrv && "Successfully resolved the derivation"); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a86f55f84..b9e7be1c6 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -224,6 +224,91 @@ static void checkSelectorUse(DrvNames & selectors) } +namespace { + +std::set<std::string> searchByPrefix(const DrvInfos & allElems, std::string_view prefix) { + constexpr std::size_t maxResults = 3; + std::set<std::string> result; + for (const auto & drvInfo : allElems) { + const auto drvName = DrvName { drvInfo.queryName() }; + if (hasPrefix(drvName.name, prefix)) { + result.emplace(drvName.name); + + if (result.size() >= maxResults) { + break; + } + } + } + return result; +} + +struct Match +{ + DrvInfo drvInfo; + std::size_t index; + + Match(DrvInfo drvInfo_, std::size_t index_) + : drvInfo{std::move(drvInfo_)} + , index{index_} + {} +}; + +/* If a selector matches multiple derivations + with the same name, pick the one matching the current + system. If there are still multiple derivations, pick the + one with the highest priority. If there are still multiple + derivations, pick the one with the highest version. + Finally, if there are still multiple derivations, + arbitrarily pick the first one. */ +std::vector<Match> pickNewestOnly(EvalState & state, std::vector<Match> matches) { + /* Map from package names to derivations. */ + std::map<std::string, Match> newest; + StringSet multiple; + + for (auto & match : matches) { + auto & oneDrv = match.drvInfo; + + const auto drvName = DrvName { oneDrv.queryName() }; + long comparison = 1; + + const auto itOther = newest.find(drvName.name); + + if (itOther != newest.end()) { + auto & newestDrv = itOther->second.drvInfo; + + comparison = + oneDrv.querySystem() == newestDrv.querySystem() ? 0 : + oneDrv.querySystem() == settings.thisSystem ? 1 : + newestDrv.querySystem() == settings.thisSystem ? -1 : 0; + if (comparison == 0) + comparison = comparePriorities(state, oneDrv, newestDrv); + if (comparison == 0) + comparison = compareVersions(drvName.version, DrvName { newestDrv.queryName() }.version); + } + + if (comparison > 0) { + newest.erase(drvName.name); + newest.emplace(drvName.name, match); + multiple.erase(drvName.fullName); + } else if (comparison == 0) { + multiple.insert(drvName.fullName); + } + } + + matches.clear(); + for (auto & [name, match] : newest) { + if (multiple.find(name) != multiple.end()) + warn( + "there are multiple derivations named '%1%'; using the first one", + name); + matches.push_back(match); + } + + return matches; +} + +} // end namespace + static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, const Strings & args, bool newestOnly) { @@ -232,80 +317,43 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, selectors.emplace_back("*"); DrvInfos elems; - set<unsigned int> done; - - for (auto & i : selectors) { - typedef list<std::pair<DrvInfo, unsigned int> > Matches; - Matches matches; - unsigned int n = 0; - for (DrvInfos::const_iterator j = allElems.begin(); - j != allElems.end(); ++j, ++n) - { - DrvName drvName(j->queryName()); - if (i.matches(drvName)) { - i.hits++; - matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n)); + std::set<std::size_t> done; + + for (auto & selector : selectors) { + std::vector<Match> matches; + for (const auto & [index, drvInfo] : enumerate(allElems)) { + const auto drvName = DrvName { drvInfo.queryName() }; + if (selector.matches(drvName)) { + ++selector.hits; + matches.emplace_back(drvInfo, index); } } - /* If `newestOnly', if a selector matches multiple derivations - with the same name, pick the one matching the current - system. If there are still multiple derivations, pick the - one with the highest priority. If there are still multiple - derivations, pick the one with the highest version. - Finally, if there are still multiple derivations, - arbitrarily pick the first one. */ if (newestOnly) { - - /* Map from package names to derivations. */ - typedef map<string, std::pair<DrvInfo, unsigned int> > Newest; - Newest newest; - StringSet multiple; - - for (auto & j : matches) { - DrvName drvName(j.first.queryName()); - long d = 1; - - Newest::iterator k = newest.find(drvName.name); - - if (k != newest.end()) { - d = j.first.querySystem() == k->second.first.querySystem() ? 0 : - j.first.querySystem() == settings.thisSystem ? 1 : - k->second.first.querySystem() == settings.thisSystem ? -1 : 0; - if (d == 0) - d = comparePriorities(state, j.first, k->second.first); - if (d == 0) - d = compareVersions(drvName.version, DrvName(k->second.first.queryName()).version); - } - - if (d > 0) { - newest.erase(drvName.name); - newest.insert(Newest::value_type(drvName.name, j)); - multiple.erase(j.first.queryName()); - } else if (d == 0) { - multiple.insert(j.first.queryName()); - } - } - - matches.clear(); - for (auto & j : newest) { - if (multiple.find(j.second.first.queryName()) != multiple.end()) - printInfo( - "warning: there are multiple derivations named '%1%'; using the first one", - j.second.first.queryName()); - matches.push_back(j.second); - } + matches = pickNewestOnly(state, std::move(matches)); } /* Insert only those elements in the final list that we haven't inserted before. */ - for (auto & j : matches) - if (done.insert(j.second).second) - elems.push_back(j.first); + for (auto & match : matches) + if (done.insert(match.index).second) + elems.push_back(match.drvInfo); + + if (selector.hits == 0 && selector.fullName != "*") { + const auto prefixHits = searchByPrefix(allElems, selector.name); + + if (prefixHits.empty()) { + throw Error("selector '%1%' matches no derivations", selector.fullName); + } else { + std::string suggestionMessage = ", maybe you meant:"; + for (const auto & drvName : prefixHits) { + suggestionMessage += fmt("\n%s", drvName); + } + throw Error("selector '%1%' matches no derivations" + suggestionMessage, selector.fullName); + } + } } - checkSelectorUse(selectors); - return elems; } @@ -879,7 +927,7 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems) placeholder.write(nullptr); } else { PathSet context; - printValueAsJSON(*globals.state, true, *v, placeholder, context); + printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } } } @@ -1149,10 +1197,10 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } else if (v->type() == nList) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); - for (unsigned int j = 0; j < v->listSize(); ++j) { - if (v->listElems()[j]->type() != nString) continue; + for (auto elem : v->listItems()) { + if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = v->listElems()[j]->string.s; + attrs3["value"] = elem->string.s; xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 25d0fa3ba..19a954ddd 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -50,9 +50,9 @@ void processExpr(EvalState & state, const Strings & attrPaths, else state.autoCallFunction(autoArgs, v, vRes); if (output == okXML) - printValueAsXML(state, strict, location, vRes, std::cout, context); + printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) - printValueAsJSON(state, strict, vRes, std::cout, context); + printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); else { if (strict) state.forceValueDeep(vRes); std::cout << vRes << std::endl; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 65d61e005..c7517cf79 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -112,7 +112,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (json) { JSONPlaceholder jsonOut(std::cout); - printValueAsJSON(*state, true, *v, jsonOut, context); + printValueAsJSON(*state, true, *v, pos, jsonOut, context); } else { diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index d995d6274..07031c909 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -31,38 +31,38 @@ at the first error. The following flake output attributes must be derivations: * `checks.`*system*`.`*name* -* `defaultPackage.`*system*` -* `devShell.`*system*` -* `devShells.`*system*`.`*name*` -* `nixosConfigurations.`*name*`.config.system.build.toplevel +* `defaultPackage.`*system* +* `devShell.`*system* +* `devShells.`*system*`.`*name* +* `nixosConfigurations.`*name*`.config.system.build.toplevel` * `packages.`*system*`.`*name* The following flake output attributes must be [app definitions](./nix3-run.md): * `apps.`*system*`.`*name* -* `defaultApp.`*system*` +* `defaultApp.`*system* The following flake output attributes must be [template definitions](./nix3-flake-init.md): * `defaultTemplate` -* `templates`.`*name* +* `templates.`*name* The following flake output attributes must be *Nixpkgs overlays*: * `overlay` -* `overlays`.`*name* +* `overlays.`*name* The following flake output attributes must be *NixOS modules*: * `nixosModule` -* `nixosModules`.`*name* +* `nixosModules.`*name* The following flake output attributes must be [bundlers](./nix3-bundle.md): -* `bundlers`.`*name* +* `bundlers.`*name* * `defaultBundler` In addition, the `hydraJobs` output is evaluated in the same way as diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 68bb76742..97f4d911c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -252,6 +252,14 @@ struct CmdFlakeInfo : CmdFlakeMetadata } }; +static bool argHasName(std::string_view arg, std::string_view expected) +{ + return + arg == expected + || arg == "_" + || (hasPrefix(arg, "_") && arg.substr(1) == expected); +} + struct CmdFlakeCheck : FlakeCommand { bool build = true; @@ -346,10 +354,14 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (!v.isLambda() || v.lambda.fun->hasFormals() || std::string(v.lambda.fun->arg) != "final") + if (!v.isLambda() + || v.lambda.fun->hasFormals() + || !argHasName(v.lambda.fun->arg, "final")) throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body); - if (!body || body->hasFormals() || std::string(body->arg) != "prev") + if (!body + || body->hasFormals() + || !argHasName(body->arg, "prev")) throw Error("overlay does not take an argument named 'prev'"); // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. @@ -1040,7 +1052,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON (attrPath.size() == 1 && attrPath[0] == "overlay") || (attrPath.size() == 2 && attrPath[0] == "overlays") ? std::make_pair("nixpkgs-overlay", "Nixpkgs overlay") : attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? std::make_pair("nixos-configuration", "NixOS configuration") : - attrPath.size() == 2 && attrPath[0] == "nixosModules" ? std::make_pair("nixos-module", "NixOS module") : + (attrPath.size() == 1 && attrPath[0] == "nixosModule") + || (attrPath.size() == 2 && attrPath[0] == "nixosModules") ? std::make_pair("nixos-module", "NixOS module") : std::make_pair("unknown", "unknown"); if (json) { j.emplace("type", type); diff --git a/src/nix/main.cc b/src/nix/main.cc index 1e033f4f2..60b0aa410 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -255,6 +255,16 @@ void mainWrapped(int argc, char * * argv) initNix(); initGC(); + #if __linux__ + if (getuid() == 0) { + try { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + } catch (Error & e) { } + } + #endif + programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 6a92576c7..c496f94f8 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -226,6 +226,7 @@ struct CmdRegistry : virtual NixMultiCommand void run() override { + settings.requireExperimentalFeature(Xp::Flakes); if (!command) throw UsageError("'nix registry' requires a sub-command."); command->second->prepare(); diff --git a/src/nix/registry.md b/src/nix/registry.md index a1674bd2e..d5c9ef442 100644 --- a/src/nix/registry.md +++ b/src/nix/registry.md @@ -2,7 +2,7 @@ R""( # Description -`nix flake` provides subcommands for managing *flake +`nix registry` provides subcommands for managing *flake registries*. Flake registries are a convenience feature that allows you to refer to flakes using symbolic identifiers such as `nixpkgs`, rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9c0d22438..f453343f3 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -279,6 +279,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt) }; setupSignals(); + Finally resetTerminal([&]() { rl_deprep_terminal(); }); char * s = readline(prompt.c_str()); Finally doFree([&]() { free(s); }); restoreSignals(); @@ -356,6 +357,8 @@ StringSet NixRepl::completePrefix(string prefix) // Quietly ignore evaluation errors. } catch (UndefinedVarError & e) { // Quietly ignore undefined variable errors. + } catch (BadURL & e) { + // Quietly ignore BadURL flake-related errors. } } @@ -427,7 +430,8 @@ bool NixRepl::processLine(string line) << " :s <expr> Build dependencies of derivation, then start nix-shell\n" << " :t <expr> Describe result of evaluation\n" << " :u <expr> Build derivation, then start nix-shell\n" - << " :doc <expr> Show documentation of a builtin function\n"; + << " :doc <expr> Show documentation of a builtin function\n" + << " :log <expr> Show logs for a derivation\n"; } else if (command == ":a" || command == ":add") { @@ -471,7 +475,10 @@ bool NixRepl::processLine(string line) auto args = editorFor(pos); auto editor = args.front(); args.pop_front(); - runProgram(editor, true, args); + + // runProgram redirects stdout to a StringSink, + // using runProgram2 to allow editors to display their UI + runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); // Reload right after exiting the editor state->resetFileCache(); @@ -494,7 +501,7 @@ bool NixRepl::processLine(string line) runNix("nix-shell", {state->store->printStorePath(drvPath)}); } - else if (command == ":b" || command == ":i" || command == ":s") { + else if (command == ":b" || command == ":i" || command == ":s" || command == ":log") { Value v; evalString(arg, v); StorePath drvPath = getDerivationPath(v); @@ -504,10 +511,31 @@ bool NixRepl::processLine(string line) state->store->buildPaths({DerivedPath::Built{drvPath}}); auto drv = state->store->readDerivation(drvPath); logger->cout("\nThis derivation produced the following outputs:"); - for (auto & i : drv.outputsAndOptPaths(*state->store)) - logger->cout(" %s -> %s", i.first, state->store->printStorePath(*i.second.second)); + for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) + logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath)); } else if (command == ":i") { runNix("nix-env", {"-i", drvPathRaw}); + } else if (command == ":log") { + settings.readOnlyMode = true; + Finally roModeReset([&]() { + settings.readOnlyMode = false; + }); + auto subs = getDefaultSubstituters(); + + subs.push_front(state->store); + + bool foundLog = false; + RunPager pager; + for (auto & sub : subs) { + auto log = sub->getBuildLog(drvPath); + if (log) { + printInfo("got build log for '%s' from '%s'", drvPathRaw, sub->getUri()); + logger->writeToStdout(*log); + foundLog = true; + break; + } + } + if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw); } else { runNix("nix-shell", {drvPathRaw}); } @@ -644,7 +672,10 @@ void NixRepl::addVarToScope(const Symbol & name, Value & v) { if (displ >= envSize) throw Error("environment full; cannot add more variables"); - staticEnv.vars[name] = displ; + if (auto oldVar = staticEnv.find(name); oldVar != staticEnv.vars.end()) + staticEnv.vars.erase(oldVar); + staticEnv.vars.emplace_back(name, displ); + staticEnv.sort(); env->values[displ++] = &v; varNames.insert((string) name); } @@ -767,12 +798,12 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << "[ "; if (maxDepth > 0) - for (unsigned int n = 0; n < v.listSize(); ++n) { - if (seen.find(v.listElems()[n]) != seen.end()) + for (auto elem : v.listItems()) { + if (seen.count(elem)) str << "«repeated»"; else try { - printValue(str, *v.listElems()[n], maxDepth - 1, seen); + printValue(str, *elem, maxDepth - 1, seen); } catch (AssertionError & e) { str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; } diff --git a/src/nix/repl.md b/src/nix/repl.md index bba60f871..9b6f2bee3 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -35,14 +35,17 @@ R""( nix-repl> emacs.drvPath "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv" - nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out" + nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello; hello > $out" - nix-repl> :b x + nix-repl> :b drv this derivation produced the following outputs: out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello nix-repl> builtins.readFile drv "Hello, world!\n" + + nix-repl> :log drv + Hello, world! ``` # Description diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 6a238efbe..3d659d6d2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -218,8 +218,7 @@ struct CmdKey : NixMultiCommand void run() override { if (!command) - throw UsageError("'nix flake' requires a sub-command."); - settings.requireExperimentalFeature(Xp::Flakes); + throw UsageError("'nix key' requires a sub-command."); command->second->prepare(); command->second->run(); } diff --git a/tests/ca-shell.nix b/tests/ca-shell.nix new file mode 100644 index 000000000..ad2ab6aff --- /dev/null +++ b/tests/ca-shell.nix @@ -0,0 +1 @@ +{ ... }@args: import ./shell.nix (args // { contentAddressed = true; }) diff --git a/tests/ca/build-with-garbage-path.sh b/tests/ca/build-with-garbage-path.sh index 9aa08a899..884cd2802 100755 --- a/tests/ca/build-with-garbage-path.sh +++ b/tests/ca/build-with-garbage-path.sh @@ -8,7 +8,7 @@ requireDaemonNewerThan "2.4pre20210621" # Get the output path of `rootCA`, and put some garbage instead outPath="$(nix-build ./content-addressed.nix -A rootCA --no-out-link)" -nix-store --delete "$outPath" +nix-store --delete $(nix-store -q --referrers-closure "$outPath") touch "$outPath" # The build should correctly remove the garbage and put the expected path instead diff --git a/tests/ca/repl.sh b/tests/ca/repl.sh new file mode 100644 index 000000000..3808c7cb2 --- /dev/null +++ b/tests/ca/repl.sh @@ -0,0 +1,5 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. && source repl.sh diff --git a/tests/common.sh.in b/tests/common.sh.in index 08f5e0a77..61abab1d7 100644 --- a/tests/common.sh.in +++ b/tests/common.sh.in @@ -36,8 +36,9 @@ export PATH=@bindir@:$PATH if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH fi +DAEMON_PATH="$PATH" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - export NIX_DAEMON_COMMAND="$NIX_DAEMON_PACKAGE/bin/nix-daemon" + DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" fi coreutils=@coreutils@ @@ -89,7 +90,7 @@ startDaemon() { # Start the daemon, wait for the socket to appear. !!! # ‘nix-daemon’ should have an option to fork into the background. rm -f $NIX_DAEMON_SOCKET_PATH - ${NIX_DAEMON_COMMAND:-nix daemon} & + PATH=$DAEMON_PATH nix daemon & for ((i = 0; i < 30; i++)); do if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then break; fi sleep 1 diff --git a/tests/flake-local-settings.sh b/tests/flake-local-settings.sh new file mode 100644 index 000000000..09f6b4ca8 --- /dev/null +++ b/tests/flake-local-settings.sh @@ -0,0 +1,34 @@ +source common.sh + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local + +cp ./simple.nix ./simple.builder.sh ./config.nix $TEST_HOME + +cd $TEST_HOME + +rm -f post-hook-ran +cat <<EOF > echoing-post-hook.sh +#!/bin/sh + +echo "ThePostHookRan" > $PWD/post-hook-ran +EOF +chmod +x echoing-post-hook.sh + +cat <<EOF > flake.nix +{ + nixConfig.post-build-hook = "$PWD/echoing-post-hook.sh"; + + outputs = a: { + defaultPackage.$system = import ./simple.nix; + }; +} +EOF + +# Without --accept-flake-config, the post hook should not run. +nix build < /dev/null +(! [[ -f post-hook-ran ]]) +clearStore + +nix build --accept-flake-config +test -f post-hook-ran || fail "The post hook should have ran" diff --git a/tests/flakes.sh b/tests/flakes.sh index 57d1b9aad..20966ab2a 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -707,10 +707,10 @@ cat > $flakeFollowsA/flake.nix <<EOF B = { url = "path:./flakeB"; inputs.foobar.follows = "D"; + inputs.nonFlake.follows = "D"; }; D.url = "path:./flakeD"; - foobar.url = "path:./flakeE"; }; outputs = { ... }: {}; } @@ -720,7 +720,9 @@ cat > $flakeFollowsB/flake.nix <<EOF { description = "Flake B"; inputs = { - foobar.url = "path:./../flakeE"; + foobar.url = "path:$flakeFollowsA/flakeE"; + nonFlake.url = "path:$nonFlakeDir"; + goodoo.follows = "C/goodoo"; C = { url = "path:./flakeC"; inputs.foobar.follows = "foobar"; @@ -734,7 +736,8 @@ cat > $flakeFollowsC/flake.nix <<EOF { description = "Flake C"; inputs = { - foobar.url = "path:./../../flakeE"; + foobar.url = "path:$flakeFollowsA/flakeE"; + goodoo.follows = "foobar"; }; outputs = { ... }: {}; } @@ -759,12 +762,43 @@ EOF git -C $flakeFollowsA add flake.nix flakeB/flake.nix \ flakeB/flakeC/flake.nix flakeD/flake.nix flakeE/flake.nix +nix flake update $flakeFollowsA + +oldLock="$(cat "$flakeFollowsA/flake.lock")" + +# Ensure that locking twice doesn't change anything + nix flake lock $flakeFollowsA +newLock="$(cat "$flakeFollowsA/flake.lock")" + +diff <(echo "$newLock") <(echo "$oldLock") + [[ $(jq -c .nodes.B.inputs.C $flakeFollowsA/flake.lock) = '"C"' ]] [[ $(jq -c .nodes.B.inputs.foobar $flakeFollowsA/flake.lock) = '["D"]' ]] [[ $(jq -c .nodes.C.inputs.foobar $flakeFollowsA/flake.lock) = '["B","foobar"]' ]] +# Ensure removing follows from flake.nix removes them from the lockfile + +cat > $flakeFollowsA/flake.nix <<EOF +{ + description = "Flake A"; + inputs = { + B = { + url = "path:./flakeB"; + inputs.nonFlake.follows = "D"; + }; + D.url = "path:./flakeD"; + }; + outputs = { ... }: {}; +} +EOF + +nix flake lock $flakeFollowsA + +[[ $(jq -c .nodes.B.inputs.foobar $flakeFollowsA/flake.lock) = '"foobar"' ]] +jq -r -c '.nodes | keys | .[]' $flakeFollowsA/flake.lock | grep "^foobar$" + # Ensure a relative path is not allowed to go outside the store path cat > $flakeFollowsA/flake.nix <<EOF { diff --git a/tests/function-trace.sh b/tests/function-trace.sh index 3b7f364e3..0b7f49d82 100755 --- a/tests/function-trace.sh +++ b/tests/function-trace.sh @@ -60,8 +60,6 @@ function-trace exited (string):1:1 at expect_trace '(x: x) 1 2' " function-trace entered (string):1:1 at function-trace exited (string):1:1 at -function-trace entered (string):1:1 at -function-trace exited (string):1:1 at " # Not a function diff --git a/tests/gc-non-blocking.sh b/tests/gc-non-blocking.sh new file mode 100644 index 000000000..8b21c6f1c --- /dev/null +++ b/tests/gc-non-blocking.sh @@ -0,0 +1,33 @@ +# Test whether the collector is non-blocking, i.e. a build can run in +# parallel with it. +source common.sh + +needLocalStore "the GC test needs a synchronisation point" + +clearStore + +fifo=$TEST_ROOT/test.fifo +mkfifo "$fifo" + +dummy=$(nix store add-path ./simple.nix) + +running=$TEST_ROOT/running +touch $running + +(_NIX_TEST_GC_SYNC=$fifo nix-store --gc -vvvvv; rm $running) & +pid=$! + +sleep 2 + +outPath=$(nix-build -o "$TEST_ROOT/result" -E " + with import ./config.nix; + mkDerivation { + name = \"non-blocking\"; + buildCommand = \"set -x; test -e $running; mkdir \$out; echo > $fifo\"; + }") + +wait $pid + +(! test -e $running) +(! test -e $dummy) +test -e $outPath diff --git a/tests/gc.sh b/tests/gc.sh index cf0e2c32d..a736b63db 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -1,5 +1,7 @@ source common.sh +clearStore + drvPath=$(nix-instantiate dependencies.nix) outPath=$(nix-store -rvv "$drvPath") @@ -23,6 +25,12 @@ test -e $inUse if nix-store --delete $outPath; then false; fi test -e $outPath +for i in $NIX_STORE_DIR/*; do + if [[ $i =~ /trash ]]; then continue; fi # compat with old daemon + touch $i.lock + touch $i.chroot +done + nix-collect-garbage # Check that the root and its dependencies haven't been deleted. @@ -38,3 +46,7 @@ nix-collect-garbage # Check that the output has been GC'd. if test -e $outPath/foobar; then false; fi + +# Check that the store is empty. +rmdir $NIX_STORE_DIR/.links +rmdir $NIX_STORE_DIR diff --git a/tests/lang/eval-okay-sort.exp b/tests/lang/eval-okay-sort.exp index 148b93516..899119e20 100644 --- a/tests/lang/eval-okay-sort.exp +++ b/tests/lang/eval-okay-sort.exp @@ -1 +1 @@ -[ [ 42 77 147 249 483 526 ] [ 526 483 249 147 77 42 ] [ "bar" "fnord" "foo" "xyzzy" ] [ { key = 1; value = "foo"; } { key = 1; value = "fnord"; } { key = 2; value = "bar"; } ] ] +[ [ 42 77 147 249 483 526 ] [ 526 483 249 147 77 42 ] [ "bar" "fnord" "foo" "xyzzy" ] [ { key = 1; value = "foo"; } { key = 1; value = "fnord"; } { key = 2; value = "bar"; } ] [ [ ] [ ] [ 1 ] [ 1 4 ] [ 1 5 ] [ 1 6 ] [ 2 ] [ 2 3 ] [ 3 ] [ 3 ] ] ] diff --git a/tests/lang/eval-okay-sort.nix b/tests/lang/eval-okay-sort.nix index 8299c3a4a..50aa78e40 100644 --- a/tests/lang/eval-okay-sort.nix +++ b/tests/lang/eval-okay-sort.nix @@ -4,5 +4,17 @@ with builtins; (sort (x: y: y < x) [ 483 249 526 147 42 77 ]) (sort lessThan [ "foo" "bar" "xyzzy" "fnord" ]) (sort (x: y: x.key < y.key) - [ { key = 1; value = "foo"; } { key = 2; value = "bar"; } { key = 1; value = "fnord"; } ]) + [ { key = 1; value = "foo"; } { key = 2; value = "bar"; } { key = 1; value = "fnord"; } ]) + (sort lessThan [ + [ 1 6 ] + [ ] + [ 2 3 ] + [ 3 ] + [ 1 5 ] + [ 2 ] + [ 1 ] + [ ] + [ 1 4 ] + [ 3 ] + ]) ] diff --git a/tests/local.mk b/tests/local.mk index d88c3a875..936b72c2a 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -4,6 +4,7 @@ nix_tests = \ gc.sh \ ca/gc.sh \ gc-concurrent.sh \ + gc-non-blocking.sh \ gc-auto.sh \ referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \ gc-runtime.sh check-refs.sh filter-source.sh \ @@ -45,9 +46,9 @@ nix_tests = \ recursive.sh \ describe-stores.sh \ flakes.sh \ + flake-local-settings.sh \ build.sh \ - compute-levels.sh \ - repl.sh \ + repl.sh ca/repl.sh \ ca/build.sh \ ca/build-with-garbage-path.sh \ ca/duplicate-realisation-in-closure.sh \ @@ -61,6 +62,10 @@ nix_tests = \ eval-store.sh # parallel.sh +ifeq ($(HAVE_LIBCPUID), 1) + nix_tests += compute-levels.sh +endif + install-tests += $(foreach x, $(nix_tests), tests/$(x)) tests-environment = NIX_REMOTE= $(bash) -e diff --git a/tests/multiple-outputs.sh b/tests/multiple-outputs.sh index 0bca12b42..0d45ad35b 100644 --- a/tests/multiple-outputs.sh +++ b/tests/multiple-outputs.sh @@ -76,7 +76,10 @@ if nix-build multiple-outputs.nix -A cyclic --no-out-link; then exit 1 fi +# Do a GC. This should leave an empty store. echo "collecting garbage..." rm $TEST_ROOT/result* nix-store --gc --keep-derivations --keep-outputs nix-store --gc --print-roots +rm -rf $NIX_STORE_DIR/.links +rmdir $NIX_STORE_DIR diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index a31d35887..3241d7a0f 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -3,59 +3,53 @@ source common.sh clearStore if [[ -n ${CONTENT_ADDRESSED:-} ]]; then - nix-shell () { - command nix-shell --arg contentAddressed true "$@" - } - - nix_develop() { - nix develop --arg contentAddressed true "$@" - } + shellDotNix="$PWD/ca-shell.nix" else - nix_develop() { - nix develop "$@" - } + shellDotNix="$PWD/shell.nix" fi +export NIX_PATH=nixpkgs="$shellDotNix" + # Test nix-shell -A export IMPURE_VAR=foo export SELECTED_IMPURE_VAR=baz -export NIX_BUILD_SHELL=$SHELL -output=$(nix-shell --pure shell.nix -A shellDrv --run \ + +output=$(nix-shell --pure "$shellDotNix" -A shellDrv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') [ "$output" = " - foo - bar - true" ] # Test --keep -output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \ +output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR "$shellDotNix" -A shellDrv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"') [ "$output" = " - foo - bar - baz" ] # Test nix-shell on a .drv -[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ +[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] -[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ +[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] # Test nix-shell on a .drv symlink # Legacy: absolute path and .drv extension required -nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell.drv +nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell.drv [[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # New behaviour: just needs to resolve to a derivation in the store -nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell +nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell [[ $(nix-shell --pure $TEST_ROOT/shell --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # Test nix-shell -p -output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') +output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') [ "$output" = "foo bar" ] # Test nix-shell -p --arg x y -output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo --argstr fooContents baz --run 'echo "$(foo)"') +output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo --argstr fooContents baz --run 'echo "$(foo)"') [ "$output" = "baz" ] # Test nix-shell shebang mode @@ -91,18 +85,18 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] # Test 'nix develop'. -nix_develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' +nix develop -f "$shellDotNix" shellDrv -c bash -c '[[ -n $stdenv ]]' # Ensure `nix develop -c` preserves stdin -echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo +echo foo | nix develop -f "$shellDotNix" shellDrv -c cat | grep -q foo # Ensure `nix develop -c` actually executes the command if stdout isn't a terminal -nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo +nix develop -f "$shellDotNix" shellDrv -c echo foo |& grep -q foo # Test 'nix print-dev-env'. -[[ $(nix print-dev-env -f shell.nix shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]] +[[ $(nix print-dev-env -f "$shellDotNix" shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]] -source <(nix print-dev-env -f shell.nix shellDrv) +source <(nix print-dev-env -f "$shellDotNix" shellDrv) [[ -n $stdenv ]] [[ ${arr1[2]} = "3 4" ]] [[ ${arr2[1]} = $'\n' ]] diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index c994fbb98..cb4b5c5fc 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -6,7 +6,10 @@ nix eval --expr 'assert 1 + 2 == 3; true' [[ $(nix eval --impure --expr 'builtins.readFile ./pure-eval.sh') =~ clearStore ]] -(! nix eval --expr 'builtins.readFile ./pure-eval.sh') +missingImpureErrorMsg=$(! nix eval --expr 'builtins.readFile ./pure-eval.sh' 2>&1) + +echo "$missingImpureErrorMsg" | grep -q -- --impure || \ + fail "The error message should mention the “--impure” flag to unblock users" (! nix eval --expr builtins.currentTime) (! nix eval --expr builtins.currentSystem) diff --git a/tests/repair.sh b/tests/repair.sh index 12dcde8ea..1105b446b 100644 --- a/tests/repair.sh +++ b/tests/repair.sh @@ -30,7 +30,7 @@ nix-store --verify-path $path2 chmod u+w $path2 touch $path2/bad -nix-store --delete $(nix-store -qd $path2) +nix-store --delete $(nix-store -q --referrers-closure $(nix-store -qd $path2)) (! nix-store --verify --check-contents --repair) diff --git a/tests/repl.sh b/tests/repl.sh index 4e3059517..995db869c 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -1,15 +1,31 @@ source common.sh replCmds=" +simple = 1 simple = import ./simple.nix :b simple +:log simple +" + +replFailingCmds=" +failing = import ./simple-failing.nix +:b failing +:log failing " testRepl () { local nixArgs=("$@") - local outPath=$(nix repl "${nixArgs[@]}" <<< "$replCmds" |& + local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replCmds")" + echo "$replOutput" + local outPath=$(echo "$replOutput" |& grep -o -E "$NIX_STORE_DIR/\w*-simple") nix path-info "${nixArgs[@]}" "$outPath" + # simple.nix prints a PATH during build + echo "$replOutput" | grep -qs 'PATH=' || fail "nix repl :log doesn't output logs" + local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replFailingCmds")" + echo "$replOutput" + echo "$replOutput" | grep -qs 'This should fail' \ + || fail "nix repl :log doesn't output logs for a failed derivation" } # Simple test, try building a drv diff --git a/tests/shell.nix b/tests/shell.nix index 4912d295a..92d94fbc2 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -74,6 +74,10 @@ let pkgs = rec { ''; bash = shell; + bashInteractive = runCommand "bash" {} '' + mkdir -p $out/bin + ln -s ${shell} $out/bin/bash + ''; # ruby "interpreter" that outputs "$@" ruby = runCommand "ruby" {} '' diff --git a/tests/simple-failing.nix b/tests/simple-failing.nix new file mode 100644 index 000000000..d176c9c51 --- /dev/null +++ b/tests/simple-failing.nix @@ -0,0 +1,12 @@ +with import ./config.nix; + +mkDerivation { + name = "simple-failing"; + builder = builtins.toFile "builder.sh" + '' + echo "This should fail" + exit 1 + ''; + PATH = ""; + goodPath = path; +} |