diff options
236 files changed, 5438 insertions, 2022 deletions
diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 942d7a971..be3f6af97 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -11,6 +11,10 @@ assignees: '' <!-- describe your problem --> +## Proposal + +<!-- propose a solution --> + ## Checklist <!-- make sure this issue is not redundant or obsolete --> @@ -22,10 +26,6 @@ assignees: '' [source]: https://github.com/NixOS/nix/tree/master/doc/manual/src [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation -## Proposal - -<!-- propose a solution --> - ## Priorities Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/labeler.yml b/.github/labeler.yml index fce0d3aeb..12120bdb3 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -16,7 +16,7 @@ "new-cli": - src/nix/**/* -"tests": +"with-tests": # Unit tests - src/*/tests/**/* # Functional and integration tests diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index b04723b95..816474ed5 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.2.0 + uses: zeebe-io/backport-action@v1.3.1 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c06c77043..c3a17d106 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,7 @@ jobs: tests: needs: [check_secrets] strategy: + fail-fast: false matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} @@ -19,7 +20,10 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v20 + - uses: cachix/install-nix-action@v22 + with: + # The sandbox would otherwise be disabled by default on Darwin + extra_nix_config: "sandbox = true" - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v12 if: needs.check_secrets.outputs.cachix == 'true' @@ -58,7 +62,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v20 + - uses: cachix/install-nix-action@v22 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - uses: cachix/cachix-action@v12 @@ -73,13 +77,14 @@ jobs: needs: [installer, check_secrets] if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' strategy: + fail-fast: false matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v20 + - uses: cachix/install-nix-action@v22 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -106,7 +111,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v20 + - uses: cachix/install-nix-action@v22 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV diff --git a/.gitignore b/.gitignore index 7ae1071d0..969194650 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,7 @@ perl/Makefile.config /doc/manual/generated/* /doc/manual/nix.json /doc/manual/conf-file.json -/doc/manual/builtins.json +/doc/manual/language.json /doc/manual/xp-features.json /doc/manual/src/SUMMARY.md /doc/manual/src/command-ref/new-cli @@ -26,6 +26,7 @@ perl/Makefile.config /doc/manual/src/command-ref/experimental-features-shortlist.md /doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md +/doc/manual/src/language/builtin-constants.md # /scripts/ /scripts/nix-profile.sh @@ -89,6 +90,7 @@ perl/Makefile.config /tests/ca/config.nix /tests/dyn-drv/config.nix /tests/repl-result-out +/tests/test-libstoreconsumer/test-libstoreconsumer # /tests/lang/ /tests/lang/*.out @@ -1 +1 @@ -2.16.0 +2.17.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 57a949906..4a72a8eac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,6 @@ We appreciate your support. Reading and following these guidelines will help us make the contribution process easy and effective for everyone involved. - ## Report a bug 1. Check on the [GitHub issue tracker](https://github.com/NixOS/nix/issues) if your bug was already reported. @@ -30,6 +29,8 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. 2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. + + Issues labeled ["good first issue"](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. Pull requests addressing issues labeled ["idea approved"](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. 3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. @@ -27,6 +27,7 @@ makefiles += \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ tests/local.mk \ + tests/test-libstoreconsumer/local.mk \ tests/plugins/local.mk else makefiles += \ diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix new file mode 100644 index 000000000..3fc1fae42 --- /dev/null +++ b/doc/manual/generate-builtin-constants.nix @@ -0,0 +1,29 @@ +let + inherit (builtins) concatStringsSep attrValues mapAttrs; + inherit (import ./utils.nix) optionalString squash; +in + +builtinsInfo: +let + showBuiltin = name: { doc, type, impure-only }: + let + type' = optionalString (type != null) " (${type})"; + + impureNotice = optionalString impure-only '' + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + ''; + in + squash '' + <dt id="builtin-constants-${name}"> + <a href="#builtin-constants-${name}"><code>${name}</code>${type'}</a> + </dt> + <dd> + + ${doc} + + ${impureNotice} + + </dd> + ''; +in +concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 71f96153f..813a287f5 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,24 +1,28 @@ let - inherit (builtins) concatStringsSep attrNames; + inherit (builtins) concatStringsSep attrValues mapAttrs; + inherit (import ./utils.nix) optionalString squash; in builtinsInfo: let - showBuiltin = name: + showBuiltin = name: { doc, args, arity, experimental-feature }: let - inherit (builtinsInfo.${name}) doc args; + experimentalNotice = optionalString (experimental-feature != null) '' + This function is only available if the [${experimental-feature}](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) experimental feature is enabled. + ''; in - '' + squash '' <dt id="builtins-${name}"> <a href="#builtins-${name}"><code>${name} ${listArgs args}</code></a> </dt> <dd> - ${doc} + ${doc} + + ${experimentalNotice} </dd> ''; listArgs = args: concatStringsSep " " (map (s: "<var>${s}</var>") args); in -concatStringsSep "\n" (map showBuiltin (attrNames builtinsInfo)) - +concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index b4b7283ef..abdfd6a62 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -128,14 +128,20 @@ $(d)/xp-features.json: $(bindir)/nix $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtins.md: $(d)/builtins.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix +$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix @cat doc/manual/src/language/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/builtins.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-builtins > $@.tmp +$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix + @cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp; + @cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp + @mv $@.tmp $@ + +$(d)/language.json: $(bindir)/nix + $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp @mv $@.tmp $@ # Generate the HTML manual. @@ -167,7 +173,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 5cd6fdea2..dcdb5d6e9 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -330,17 +330,31 @@ const redirects = { "ssec-relnotes-2.0": "release-notes/rl-2.0.html", "ssec-relnotes-2.1": "release-notes/rl-2.1.html", "ssec-relnotes-2.2": "release-notes/rl-2.2.html", - "ssec-relnotes-2.3": "release-notes/rl-2.3.html" + "ssec-relnotes-2.3": "release-notes/rl-2.3.html", }, "language/values.html": { "simple-values": "#primitives", "lists": "#list", "strings": "#string", "lists": "#list", - "attribute-sets": "#attribute-set" + "attribute-sets": "#attribute-set", }, "installation/installing-binary.html": { - "uninstalling": "uninstall.html" + "linux": "uninstall.html#linux", + "macos": "uninstall.html#macos", + "uninstalling": "uninstall.html", + } + "contributing/hacking.html": { + "nix-with-flakes": "#building-nix-with-flakes", + "classic-nix": "#building-nix", + "running-tests": "testing.html#running-tests", + "unit-tests": "testing.html#unit-tests", + "functional-tests": "testing.html#functional-tests", + "debugging-failing-functional-tests": "testing.html#debugging-failing-functional-tests", + "integration-tests": "testing.html#integration-tests", + "installer-tests": "testing.html#installer-tests", + "one-time-setup": "testing.html#one-time-setup", + "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", } }; diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 606aecd8f..1bd8fa774 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -97,14 +97,20 @@ - [manifest.json](command-ref/files/manifest.json.md) - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) -- [Architecture](architecture/architecture.md) +- [Architecture and Design](architecture/architecture.md) + - [File System Object](architecture/file-system-object.md) +- [Protocols](protocols/protocols.md) + - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) + - [Testing](contributing/testing.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) + - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) diff --git a/doc/manual/src/advanced-topics/diff-hook.md b/doc/manual/src/advanced-topics/diff-hook.md index 4a742c160..207aad3b8 100644 --- a/doc/manual/src/advanced-topics/diff-hook.md +++ b/doc/manual/src/advanced-topics/diff-hook.md @@ -48,13 +48,13 @@ If the build passes and is deterministic, Nix will exit with a status code of 0: ```console -$ nix-build ./deterministic.nix -A stable +$ nix-build ./deterministic.nix --attr stable this derivation will be built: /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable -$ nix-build ./deterministic.nix -A stable --check +$ nix-build ./deterministic.nix --attr stable --check checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... /nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable ``` @@ -63,13 +63,13 @@ If the build is not deterministic, Nix will exit with a status code of 1: ```console -$ nix-build ./deterministic.nix -A unstable +$ nix-build ./deterministic.nix --attr unstable this derivation will be built: /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable -$ nix-build ./deterministic.nix -A unstable --check +$ nix-build ./deterministic.nix --attr unstable --check checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs @@ -89,7 +89,7 @@ Using `--check` with `--keep-failed` will cause Nix to keep the second build's output in a special, `.check` path: ```console -$ nix-build ./deterministic.nix -A unstable --check --keep-failed +$ nix-build ./deterministic.nix --attr unstable --check --keep-failed checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... note: keeping build directory '/tmp/nix-build-unstable.drv-0' error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index fefd10100..73a113d35 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -38,11 +38,9 @@ contains Nix. > **Warning** > -> If you are building via the Nix daemon, it is the Nix daemon user -> account (that is, `root`) that should have SSH access to the remote -> machine. If you can’t or don’t want to configure `root` to be able to -> access to remote machine, you can use a private Nix store instead by -> passing e.g. `--store ~/my-nix`. +> If you are building via the Nix daemon, it is the Nix daemon user account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote machine. +> +> If you can’t or don’t want to configure `root` to be able to access the remote machine, you can use a private Nix store instead by passing e.g. `--store ~/my-nix` when running a Nix command from the local machine. The list of remote machines can be specified on the command line or in the Nix configuration file. The former is convenient for testing. For diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index 1479cc3a4..a251dec48 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -90,7 +90,7 @@ Then, restart the `nix-daemon`. Build any derivation, for example: ```console -$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' +$ nix-build --expr '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' this derivation will be built: /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index e51958052..9e969972e 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -7,11 +7,11 @@ It should help users understand why Nix behaves as it does, and it should help d Nix consists of [hierarchical layers]. -[hierarchical layers]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers +[hierarchical layers]: https://en.wikipedia.org/wiki/Multitier_architecture#Layers The following [concept map] shows its main components (rectangles), the objects they operate on (rounded rectangles), and their interactions (connecting phrases): -[concept map]: https://en.m.wikipedia.org/wiki/Concept_map +[concept map]: https://en.wikipedia.org/wiki/Concept_map ``` @@ -76,7 +76,7 @@ The result of a build task can be input to another build task. The following [data flow diagram] shows a build plan for illustration. Build inputs used as instructions to a build task are marked accordingly: -[data flow diagram]: https://en.m.wikipedia.org/wiki/Data-flow_diagram +[data flow diagram]: https://en.wikipedia.org/wiki/Data-flow_diagram ``` +--------------------------------------------------------------------+ diff --git a/doc/manual/src/architecture/file-system-object.md b/doc/manual/src/architecture/file-system-object.md new file mode 100644 index 000000000..42f047260 --- /dev/null +++ b/doc/manual/src/architecture/file-system-object.md @@ -0,0 +1,64 @@ +# File System Object + +Nix uses a simplified model of the file system, which consists of file system objects. +Every file system object is one of the following: + + - File + + - A possibly empty sequence of bytes for contents + - A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission + + - Directory + + Mapping of names to child file system objects + + - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link) + + An arbitrary string. + Nix does not assign any semantics to symbolic links. + +File system objects and their children form a tree. +A bare file or symlink can be a root file system object. + +Nix does not encode any other file system notions such as [hard links](https://en.m.wikipedia.org/wiki/Hard_link), [permissions](https://en.m.wikipedia.org/wiki/File-system_permissions), timestamps, or other metadata. + +## Examples of file system objects + +A plain file: + +``` +50 B, executable: false +``` + +An executable file: + +``` +122 KB, executable: true +``` + +A symlink: + +``` +-> /usr/bin/sh +``` + +A directory with contents: + +``` +├── bin +│ └── hello: 35 KB, executable: true +└── share + ├── info + │ └── hello.info: 36 KB, executable: false + └── man + └── man1 + └── hello.1.gz: 790 B, executable: false +``` + +A directory that contains a symlink and other directories: + +``` +├── bin -> share/go/bin +├── nix-support/ +└── share/ +``` diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index 44b7ba86d..1e4085977 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -4,49 +4,67 @@ # Description -By default Nix reads settings from the following places: +Nix supports a variety of configuration settings, which are read from configuration files or taken as command line flags. - - The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. - `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if - `NIX_CONF_DIR` is set. Values loaded in this file are not forwarded - to the Nix daemon. The client assumes that the daemon has already - loaded them. +## Configuration file - - If `NIX_USER_CONF_FILES` is set, then each path separated by `:` - will be loaded in reverse order. +By default Nix reads settings from the following places, in that order: - Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` - and `XDG_CONFIG_HOME`. If unset, `XDG_CONFIG_DIRS` defaults to - `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` - as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). +1. The system-wide configuration file `sysconfdir/nix/nix.conf` (i.e. `/etc/nix/nix.conf` on most systems), or `$NIX_CONF_DIR/nix.conf` if [`NIX_CONF_DIR`](./env-common.md#env-NIX_CONF_DIR) is set. - - If `NIX_CONFIG` is set, its contents is treated as the contents of - a configuration file. + Values loaded in this file are not forwarded to the Nix daemon. + The client assumes that the daemon has already loaded them. -The configuration files consist of `name = value` pairs, one per -line. Other files can be included with a line like `include path`, -where *path* is interpreted relative to the current conf file and a -missing file is an error unless `!include` is used instead. Comments -start with a `#` character. Here is an example configuration file: +1. If [`NIX_USER_CONF_FILES`](./env-common.md#env-NIX_USER_CONF_FILES) is set, then each path separated by `:` will be loaded in reverse order. - keep-outputs = true # Nice for developers - keep-derivations = true # Idem + Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` and [`XDG_CONFIG_HOME`](./env-common.md#env-XDG_CONFIG_HOME). + If unset, `XDG_CONFIG_DIRS` defaults to `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). -You can override settings on the command line using the `--option` -flag, e.g. `--option keep-outputs false`. Every configuration setting -also has a corresponding command line flag, e.g. `--max-jobs 16`; for -Boolean settings, there are two flags to enable or disable the setting -(e.g. `--keep-failed` and `--no-keep-failed`). +1. If [`NIX_CONFIG`](./env-common.md#env-NIX_CONFIG) is set, its contents are treated as the contents of a configuration file. -A configuration setting usually overrides any previous value. However, -you can prefix the name of the setting by `extra-` to *append* to the -previous value. For instance, +### File format - substituters = a b - extra-substituters = c d +Configuration files consist of `name = value` pairs, one per line. +Comments start with a `#` character. -defines the `substituters` setting to be `a b c d`. This is also -available as a command line flag (e.g. `--extra-substituters`). +Example: -The following settings are currently available: +``` +keep-outputs = true # Nice for developers +keep-derivations = true # Idem +``` + +Other files can be included with a line like `include <path>`, where `<path>` is interpreted relative to the current configuration file. +A missing file is an error unless `!include` is used instead. + +A configuration setting usually overrides any previous value. +However, for settings that take a list of items, you can prefix the name of the setting by `extra-` to *append* to the previous value. + +For instance, + +``` +substituters = a b +extra-substituters = c d +``` + +defines the `substituters` setting to be `a b c d`. + +Unknown option names are not an error, and are simply ignored with a warning. + +## Command line flags + +Configuration options can be set on the command line, overriding the values set in the [configuration file](#configuration-file): + +- Every configuration setting has corresponding command line flag (e.g. `--max-jobs 16`). + Boolean settings do not need an argument, and can be explicitly disabled with the `no-` prefix (e.g. `--keep-failed` and `--no-keep-failed`). + + Unknown option names are invalid flags (unless there is already a flag with that name), and are rejected with an error. + +- The flag `--option <name> <value>` is interpreted exactly like a `<name> = <value>` in a setting file. + + Unknown option names are ignored with a warning. + +The `extra-` prefix is supported for settings that take a list of items (e.g. `--extra-trusted users alice` or `--option extra-trusted-users alice`). + +# Available settings diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index bf00be84f..b4a9bb2a9 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -71,9 +71,12 @@ Most Nix commands interpret the following environment variables: Settings are separated by the newline character. - <span id="env-NIX_USER_CONF_FILES">[`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES)</span>\ - Overrides the location of the user Nix configuration files to load - from (defaults to the XDG spec locations). The variable is treated - as a list separated by the `:` token. + Overrides the location of the Nix user configuration files to load from. + + The default are the locations according to the [XDG Base Directory Specification]. + See the [XDG Base Directories](#xdg-base-directories) sub-section for details. + + The variable is treated as a list separated by the `:` token. - <span id="env-TMPDIR">[`TMPDIR`](#env-TMPDIR)</span>\ Use the specified directory to store temporary files. In particular, @@ -103,15 +106,19 @@ Most Nix commands interpret the following environment variables: 384 MiB. Setting it to a low value reduces memory consumption, but will increase runtime due to the overhead of garbage collection. -## XDG Base Directory +## XDG Base Directories + +Nix follows the [XDG Base Directory Specification]. + +For backwards compatibility, Nix commands will follow the standard only when [`use-xdg-base-directories`] is enabled. +[New Nix commands](@docroot@/command-ref/new-cli/nix.md) (experimental) conform to the standard by default. -New Nix commands conform to the [XDG Base Directory Specification], and use the following environment variables to determine locations of various state and configuration files: +The following environment variables are used to determine locations of various state and configuration files: - [`XDG_CONFIG_HOME`]{#env-XDG_CONFIG_HOME} (default `~/.config`) - [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`) - [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`) -Classic Nix commands can also be made to follow this standard using the [`use-xdg-base-directories`] configuration option. [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html -[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories
\ No newline at end of file +[`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index 44de4cf53..b548edf82 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -51,8 +51,9 @@ derivation). # Options -All options not listed here are passed to `nix-store --realise`, -except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. +All options not listed here are passed to +[`nix-store --realise`](nix-store/realise.md), +except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md). - <span id="opt-no-out-link">[`--no-out-link`](#opt-no-out-link)<span> @@ -69,6 +70,8 @@ except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. Change the name of the symlink to the output path created from `result` to *outlink*. +{{#include ./status-build-failure.md}} + {{#include ./opt-common.md}} {{#include ./env-common.md}} @@ -76,7 +79,7 @@ except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. # Examples ```console -$ nix-build '<nixpkgs>' -A firefox +$ nix-build '<nixpkgs>' --attr firefox store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv /nix/store/d18hyl92g30l...-firefox-1.5.0.7 @@ -91,7 +94,7 @@ If a derivation has multiple outputs, `nix-build` will build the default (first) output. You can also build all outputs: ```console -$ nix-build '<nixpkgs>' -A openssl.all +$ nix-build '<nixpkgs>' --attr openssl.all ``` This will create a symlink for each output named `result-outputname`. @@ -101,7 +104,7 @@ outputs `out`, `bin` and `man`, `nix-build` will create symlinks specific output: ```console -$ nix-build '<nixpkgs>' -A openssl.man +$ nix-build '<nixpkgs>' --attr openssl.man ``` This will create a symlink `result-man`. @@ -109,7 +112,7 @@ This will create a symlink `result-man`. Build a Nix expression given on the command line: ```console -$ nix-build -E 'with import <nixpkgs> { }; runCommand "foo" { } "echo bar > $out"' +$ nix-build --expr 'with import <nixpkgs> { }; runCommand "foo" { } "echo bar > $out"' $ cat ./result bar ``` @@ -118,5 +121,5 @@ Build the GNU Hello package from the latest revision of the master branch of Nixpkgs: ```console -$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello +$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz --attr hello ``` diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index 72d3e422b..025f758e7 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -4,7 +4,7 @@ # Synopsis -`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--rollback` [*generation*] } +`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--list-generations` | `--rollback` [*generation*] } # Description @@ -39,6 +39,15 @@ This command has the following operations: for `nix-env` operations (by symlinking them from the directory `~/.nix-defexpr`). + - `--list-generations`\ + Prints a list of all the current existing generations for the + channel profile. + + Works the same way as + ``` + nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations + ``` + - `--rollback` \[*generation*\]\ Reverts the previous call to `nix-channel --update`. Optionally, you can specify a specific channel generation @@ -52,6 +61,12 @@ The list of subscribed channels is stored in `~/.nix-channels`. {{#include ./env-common.md}} +# Files + +`nix-channel` operates on the following files. + +{{#include ./files/channels.md}} + # Examples To subscribe to the Nixpkgs channel and install the GNU Hello package: @@ -59,18 +74,18 @@ To subscribe to the Nixpkgs channel and install the GNU Hello package: ```console $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable $ nix-channel --update -$ nix-env -iA nixpkgs.hello +$ nix-env --install --attr nixpkgs.hello ``` You can revert channel updates using `--rollback`: ```console -$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' +$ nix-instantiate --eval --expr '(import <nixpkgs> {}).lib.version' "14.04.527.0e935f1" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version' +$ nix-instantiate --eval --expr '(import <nixpkgs> {}).lib.version' "14.04.526.dbadfad" ``` diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 51db5fc67..3cab79f0e 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -1,6 +1,6 @@ # Name -`nix-collect-garbage` - delete unreachable store paths +`nix-collect-garbage` - delete unreachable [store objects] # Synopsis @@ -8,17 +8,57 @@ # Description -The command `nix-collect-garbage` is mostly an alias of [`nix-store ---gc`](@docroot@/command-ref/nix-store/gc.md), that is, it deletes all -unreachable paths in the Nix store to clean up your system. However, -it provides two additional options: `-d` (`--delete-old`), which -deletes all old generations of all profiles in `/nix/var/nix/profiles` -by invoking `nix-env --delete-generations old` on all profiles (of -course, this makes rollbacks to previous configurations impossible); -and `--delete-older-than` *period*, where period is a value such as -`30d`, which deletes all generations older than the specified number -of days in all profiles in `/nix/var/nix/profiles` (except for the -generations that were active at that point in time). +The command `nix-collect-garbage` is mostly an alias of [`nix-store --gc`](@docroot@/command-ref/nix-store/gc.md). +That is, it deletes all unreachable [store objects] in the Nix store to clean up your system. + +However, it provides two additional options, +[`--delete-old`](#opt-delete-old) and [`--delete-older-than`](#opt-delete-older-than), +which also delete old [profiles], allowing potentially more [store objects] to be deleted because profiles are also garbage collection roots. +These options are the equivalent of running +[`nix-env --delete-generations`](@docroot@/command-ref/nix-env/delete-generations.md) +with various augments on multiple profiles, +prior to running `nix-collect-garbage` (or just `nix-store --gc`) without any flags. + +> **Note** +> +> Deleting previous configurations makes rollbacks to them impossible. + +These flags should be used with care, because they potentially delete generations of profiles used by other users on the system. + +## Locations searched for profiles + +`nix-collect-garbage` cannot know about all profiles; that information doesn't exist. +Instead, it looks in a few locations, and acts on all profiles it finds there: + +1. The default profile locations as specified in the [profiles] section of the manual. + +2. > **NOTE** + > + > Not stable; subject to change + > + > Do not rely on this functionality; it just exists for migration purposes and is may change in the future. + > These deprecated paths remain a private implementation detail of Nix. + + `$NIX_STATE_DIR/profiles` and `$NIX_STATE_DIR/profiles/per-user`. + + With the exception of `$NIX_STATE_DIR/profiles/per-user/root` and `$NIX_STATE_DIR/profiles/default`, these directories are no longer used by other commands. + `nix-collect-garbage` looks there anyways in order to clean up profiles from older versions of Nix. + +# Options + +These options are for deleting old [profiles] prior to deleting unreachable [store objects]. + +- <span id="opt-delete-old">[`--delete-old`](#opt-delete-old)</span> / `-d`\ + Delete all old generations of profiles. + + This is the equivalent of invoking `nix-env --delete-generations old` on each found profile. + +- <span id="opt-delete-older-than">[`--delete-older-than`](#opt-delete-older-than)</span> *period*\ + Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). + *period* is a value such as `30d`, which would mean 30 days. + + This is the equivalent of invoking [`nix-env --delete-generations <period>`](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. + See the documentation of that command for additional information about the *period* argument. {{#include ./opt-common.md}} @@ -32,3 +72,6 @@ generations of each profile, do ```console $ nix-collect-garbage -d ``` + +[profiles]: @docroot@/command-ref/files/profiles.md +[store objects]: @docroot@/glossary.md#gloss-store-object diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index 0801e8246..fbf6828da 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -87,5 +87,5 @@ environment: ```console $ nix-copy-closure --from alice@itchy.labs \ /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 -$ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 +$ nix-env --install /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 ``` diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index b4a3dce49..941723216 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -49,7 +49,7 @@ These pages can be viewed offline: # Selectors -Several commands, such as `nix-env -q` and `nix-env -i`, take a list of +Several commands, such as `nix-env --query ` and `nix-env --install `, take a list of arguments that specify the packages on which to operate. These are extended regular expressions that must match the entire name of the package. (For details on regular expressions, see **regex**(7).) The match is @@ -83,6 +83,8 @@ match. Here are some examples: # Files +`nix-env` operates on the following files. + {{#include ./files/default-nix-expression.md}} {{#include ./files/profiles.md}} diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md index 6f0af5384..adc6fc219 100644 --- a/doc/manual/src/command-ref/nix-env/delete-generations.md +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -9,14 +9,47 @@ # Description This operation deletes the specified generations of the current profile. -The generations can be a list of generation numbers, the special value -`old` to delete all non-current generations, a value such as `30d` to -delete all generations older than the specified number of days (except -for the generation that was active at that point in time), or a value -such as `+5` to keep the last `5` generations ignoring any newer than -current, e.g., if `30` is the current generation `+5` will delete -generation `25` and all older generations. Periodically deleting old -generations is important to make garbage collection effective. + +*generations* can be a one of the following: + +- <span id="generations-list">`<number>...`</span>:\ + A list of generation numbers, each one a separate command-line argument. + + Delete exactly the profile generations given by their generation number. + Deleting the current generation is not allowed. + +- The special value <span id="generations-old">`old`</span> + + Delete all generations except the current one. + + > **WARNING** + > + > Older *and newer* generations will be deleted by this operation. + > + > One might expect this to just delete older generations than the curent one, but that is only true if the current generation is also the latest. + > Because one can roll back to a previous generation, it is possible to have generations newer than the current one. + > They will also be deleted. + +- <span id="generations-time">`<number>d`</span>:\ + The last *number* days + + *Example*: `30d` + + Delete all generations created more than *number* days ago, except the most recent one of them. + This allows rolling back to generations that were available within the specified period. + +- <span id="generations-count">`+<number>`</span>:\ + The last *number* generations up to the present + + *Example*: `+5` + + Keep the last *number* generations, along with any newer than current. + +Periodically deleting old generations is important to make garbage collection +effective. +The is because profiles are also garbage collection roots — any [store object] reachable from a profile is "alive" and ineligible for deletion. + +[store object]: @docroot@/glossary.md#gloss-store-object {{#include ./opt-common.md}} @@ -28,19 +61,35 @@ generations is important to make garbage collection effective. # Examples +## Delete explicit generation numbers + ```console $ nix-env --delete-generations 3 4 8 ``` +Delete the generations numbered 3, 4, and 8, so long as the current active generation is not any of those. + +## Keep most-recent by count (number of generations) + ```console $ nix-env --delete-generations +5 ``` +Suppose `30` is the current generation, and we currently have generations numbered `20` through `32`. + +Then this command will delete generations `20` through `25` (`<= 30 - 5`), +and keep generations `26` through `31` (`> 30 - 5`). + +## Keep most-recent by time (number of days) + ```console $ nix-env --delete-generations 30d ``` +This command will delete all generations older than 30 days, except for the generation that was active 30 days ago (if it currently exists). + +## Delete all older + ```console -$ nix-env -p other_profile --delete-generations old +$ nix-env --profile other_profile --delete-generations old ``` - diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index d754accfe..ad179cbc7 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -36,7 +36,7 @@ a number of possible ways: then the derivation with the highest version will be installed. You can force the installation of multiple derivations with the same - name by being specific about the versions. For instance, `nix-env -i + name by being specific about the versions. For instance, `nix-env --install gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will probably cause a user environment conflict\!). @@ -44,7 +44,7 @@ a number of possible ways: paths* that select attributes from the top-level Nix expression. This is faster than using derivation names and unambiguous. To find out the attribute paths of available - packages, use `nix-env -qaP`. + packages, use `nix-env --query --available --attr-path `. - If `--from-profile` *path* is given, *args* is a set of names denoting installed store paths in the profile *path*. This is an @@ -87,7 +87,7 @@ a number of possible ways: - `--remove-all` / `-r`\ Remove all previously installed packages first. This is equivalent - to running `nix-env -e '.*'` first, except that everything happens + to running `nix-env --uninstall '.*'` first, except that everything happens in a single transaction. {{#include ./opt-common.md}} @@ -103,9 +103,9 @@ a number of possible ways: To install a package using a specific attribute path from the active Nix expression: ```console -$ nix-env -iA gcc40mips +$ nix-env --install --attr gcc40mips installing `gcc-4.0.2' -$ nix-env -iA xorg.xorgserver +$ nix-env --install --attr xorg.xorgserver installing `xorg-server-1.2.0' ``` @@ -133,32 +133,32 @@ installing `gcc-3.3.2' To install all derivations in the Nix expression `foo.nix`: ```console -$ nix-env -f ~/foo.nix -i '.*' +$ nix-env --file ~/foo.nix --install '.*' ``` To copy the store path with symbolic name `gcc` from another profile: ```console -$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc +$ nix-env --install --from-profile /nix/var/nix/profiles/foo gcc ``` To install a specific [store derivation] (typically created by `nix-instantiate`): ```console -$ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv +$ nix-env --install /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv ``` To install a specific output path: ```console -$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3 +$ nix-env --install /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3 ``` To install from a Nix expression specified on the command-line: ```console -$ nix-env -f ./foo.nix -i -E \ +$ nix-env --file ./foo.nix --install --expr \ 'f: (f {system = "i686-linux";}).subversionWithJava' ``` @@ -170,7 +170,7 @@ function defined in `./foo.nix`. A dry-run tells you which paths will be downloaded or built from source: ```console -$ nix-env -f '<nixpkgs>' -iA hello --dry-run +$ nix-env --file '<nixpkgs>' --install --attr hello --dry-run (dry run; not doing anything) installing ‘hello-2.10’ this path will be fetched (0.04 MiB download, 0.19 MiB unpacked): @@ -182,6 +182,6 @@ To install Firefox from the latest revision in the Nixpkgs/NixOS 14.12 channel: ```console -$ nix-env -f https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz -iA firefox +$ nix-env --file https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz --install --attr firefox ``` diff --git a/doc/manual/src/command-ref/nix-env/query.md b/doc/manual/src/command-ref/nix-env/query.md index 18f0ee210..c9b4d8513 100644 --- a/doc/manual/src/command-ref/nix-env/query.md +++ b/doc/manual/src/command-ref/nix-env/query.md @@ -137,7 +137,7 @@ derivation is shown unless `--no-name` is specified. To show installed packages: ```console -$ nix-env -q +$ nix-env --query bison-1.875c docbook-xml-4.2 firefox-1.0.4 @@ -149,7 +149,7 @@ ORBit2-2.8.3 To show available packages: ```console -$ nix-env -qa +$ nix-env --query --available firefox-1.0.7 GConf-2.4.0.1 MPlayer-1.0pre7 @@ -160,7 +160,7 @@ ORBit2-2.8.3 To show the status of available packages: ```console -$ nix-env -qas +$ nix-env --query --available --status -P- firefox-1.0.7 (not installed but present) --S GConf-2.4.0.1 (not present, but there is a substitute for fast installation) --S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!) @@ -171,14 +171,14 @@ IP- ORBit2-2.8.3 (installed and by definition present) To show available packages in the Nix expression `foo.nix`: ```console -$ nix-env -f ./foo.nix -qa +$ nix-env --file ./foo.nix --query --available foo-1.2.3 ``` To compare installed versions to what’s available: ```console -$ nix-env -qc +$ nix-env --query --compare-versions ... acrobat-reader-7.0 - ? (package is not available at all) autoconf-2.59 = 2.59 (same version) @@ -189,7 +189,7 @@ firefox-1.0.4 < 1.0.7 (a more recent version is available) To show all packages with “`zip`” in the name: ```console -$ nix-env -qa '.*zip.*' +$ nix-env --query --available '.*zip.*' bzip2-1.0.6 gzip-1.6 zip-3.0 @@ -199,7 +199,7 @@ zip-3.0 To show all packages with “`firefox`” or “`chromium`” in the name: ```console -$ nix-env -qa '.*(firefox|chromium).*' +$ nix-env --query --available '.*(firefox|chromium).*' chromium-37.0.2062.94 chromium-beta-38.0.2125.24 firefox-32.0.3 @@ -210,6 +210,6 @@ firefox-with-plugins-13.0.1 To show all packages in the latest revision of the Nixpkgs repository: ```console -$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa +$ nix-env --file https://github.com/NixOS/nixpkgs/archive/master.tar.gz --query --available ``` diff --git a/doc/manual/src/command-ref/nix-env/set-flag.md b/doc/manual/src/command-ref/nix-env/set-flag.md index 63f0a0ff9..e04b22a91 100644 --- a/doc/manual/src/command-ref/nix-env/set-flag.md +++ b/doc/manual/src/command-ref/nix-env/set-flag.md @@ -46,16 +46,16 @@ To prevent the currently installed Firefox from being upgraded: $ nix-env --set-flag keep true firefox ``` -After this, `nix-env -u` will ignore Firefox. +After this, `nix-env --upgrade ` will ignore Firefox. To disable the currently installed Firefox, then install a new Firefox while the old remains part of the profile: ```console -$ nix-env -q +$ nix-env --query firefox-2.0.0.9 (the current one) -$ nix-env --preserve-installed -i firefox-2.0.0.11 +$ nix-env --preserve-installed --install firefox-2.0.0.11 installing `firefox-2.0.0.11' building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment' collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox' @@ -65,10 +65,10 @@ collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox' $ nix-env --set-flag active false firefox setting flag on `firefox-2.0.0.9' -$ nix-env --preserve-installed -i firefox-2.0.0.11 +$ nix-env --preserve-installed --install firefox-2.0.0.11 installing `firefox-2.0.0.11' -$ nix-env -q +$ nix-env --query firefox-2.0.0.11 (the enabled one) firefox-2.0.0.9 (the disabled one) ``` diff --git a/doc/manual/src/command-ref/nix-env/set.md b/doc/manual/src/command-ref/nix-env/set.md index c1cf75739..b9950eeab 100644 --- a/doc/manual/src/command-ref/nix-env/set.md +++ b/doc/manual/src/command-ref/nix-env/set.md @@ -25,6 +25,6 @@ The following updates a profile such that its current generation will contain just Firefox: ```console -$ nix-env -p /nix/var/nix/profiles/browser --set firefox +$ nix-env --profile /nix/var/nix/profiles/browser --set firefox ``` diff --git a/doc/manual/src/command-ref/nix-env/switch-generation.md b/doc/manual/src/command-ref/nix-env/switch-generation.md index e550325c4..38cf0534d 100644 --- a/doc/manual/src/command-ref/nix-env/switch-generation.md +++ b/doc/manual/src/command-ref/nix-env/switch-generation.md @@ -27,7 +27,7 @@ Switching will fail if the specified generation does not exist. # Examples ```console -$ nix-env -G 42 +$ nix-env --switch-generation 42 switching from generation 50 to 42 ``` diff --git a/doc/manual/src/command-ref/nix-env/switch-profile.md b/doc/manual/src/command-ref/nix-env/switch-profile.md index b389e4140..5ae2fdced 100644 --- a/doc/manual/src/command-ref/nix-env/switch-profile.md +++ b/doc/manual/src/command-ref/nix-env/switch-profile.md @@ -22,5 +22,5 @@ the symlink `~/.nix-profile` is made to point to *path*. # Examples ```console -$ nix-env -S ~/my-profile +$ nix-env --switch-profile ~/my-profile ``` diff --git a/doc/manual/src/command-ref/nix-env/uninstall.md b/doc/manual/src/command-ref/nix-env/uninstall.md index e9ec8a15e..734cc7675 100644 --- a/doc/manual/src/command-ref/nix-env/uninstall.md +++ b/doc/manual/src/command-ref/nix-env/uninstall.md @@ -24,5 +24,5 @@ designated by the symbolic names *drvnames* are removed. ```console $ nix-env --uninstall gcc -$ nix-env -e '.*' (remove everything) +$ nix-env --uninstall '.*' (remove everything) ``` diff --git a/doc/manual/src/command-ref/nix-env/upgrade.md b/doc/manual/src/command-ref/nix-env/upgrade.md index f88ffcbee..322dfbda2 100644 --- a/doc/manual/src/command-ref/nix-env/upgrade.md +++ b/doc/manual/src/command-ref/nix-env/upgrade.md @@ -76,21 +76,21 @@ version is installed. # Examples ```console -$ nix-env --upgrade -A nixpkgs.gcc +$ nix-env --upgrade --attr nixpkgs.gcc upgrading `gcc-3.3.1' to `gcc-3.4' ``` When there are no updates available, nothing will happen: ```console -$ nix-env --upgrade -A nixpkgs.pan +$ nix-env --upgrade --attr nixpkgs.pan ``` Using `-A` is preferred when possible, as it is faster and unambiguous but it is also possible to upgrade to a specific version by matching the derivation name: ```console -$ nix-env -u gcc-3.3.2 --always +$ nix-env --upgrade gcc-3.3.2 --always upgrading `gcc-3.4' to `gcc-3.3.2' ``` @@ -98,7 +98,7 @@ To try to upgrade everything (matching packages based on the part of the derivation name without version): ```console -$ nix-env -u +$ nix-env --upgrade upgrading `hello-2.1.2' to `hello-2.1.3' upgrading `mozilla-1.2' to `mozilla-1.4' ``` diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index e55fb2afd..e1b4a3e80 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -88,7 +88,7 @@ Instantiate [store derivation]s from a Nix expression, and build them using `nix $ nix-instantiate test.nix (instantiate) /nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv -$ nix-store -r $(nix-instantiate test.nix) (build) +$ nix-store --realise $(nix-instantiate test.nix) (build) ... /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 (output path) @@ -100,30 +100,30 @@ dr-xr-xr-x 2 eelco users 4096 1970-01-01 01:00 lib You can also give a Nix expression on the command line: ```console -$ nix-instantiate -E 'with import <nixpkgs> { }; hello' +$ nix-instantiate --expr 'with import <nixpkgs> { }; hello' /nix/store/j8s4zyv75a724q38cb0r87rlczaiag4y-hello-2.8.drv ``` This is equivalent to: ```console -$ nix-instantiate '<nixpkgs>' -A hello +$ nix-instantiate '<nixpkgs>' --attr hello ``` Parsing and evaluating Nix expressions: ```console -$ nix-instantiate --parse -E '1 + 2' +$ nix-instantiate --parse --expr '1 + 2' 1 + 2 ``` ```console -$ nix-instantiate --eval -E '1 + 2' +$ nix-instantiate --eval --expr '1 + 2' 3 ``` ```console -$ nix-instantiate --eval --xml -E '1 + 2' +$ nix-instantiate --eval --xml --expr '1 + 2' <?xml version='1.0' encoding='utf-8'?> <expr> <int value="3" /> @@ -133,7 +133,7 @@ $ nix-instantiate --eval --xml -E '1 + 2' The difference between non-strict and strict evaluation: ```console -$ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }' +$ nix-instantiate --eval --xml --expr 'rec { x = "foo"; y = x; }' ... <attr name="x"> <string value="foo" /> @@ -148,7 +148,7 @@ Note that `y` is left unevaluated (the XML representation doesn’t attempt to show non-normal forms). ```console -$ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }' +$ nix-instantiate --eval --xml --strict --expr 'rec { x = "foo"; y = x; }' ... <attr name="x"> <string value="foo" /> diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 576e5ba0b..195b72be5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -89,7 +89,7 @@ All options not listed here are passed to `nix-store - `--packages` / `-p` *packages*…\ Set up an environment in which the specified packages are present. The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell -p libjpeg openjdk` + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` will start a shell in which the packages denoted by the attribute names `libjpeg` and `openjdk` are present. @@ -118,7 +118,7 @@ To build the dependencies of the package Pan, and start an interactive shell in which to build it: ```console -$ nix-shell '<nixpkgs>' -A pan +$ nix-shell '<nixpkgs>' --attr pan [nix-shell]$ eval ${unpackPhase:-unpackPhase} [nix-shell]$ cd $sourceRoot [nix-shell]$ eval ${patchPhase:-patchPhase} @@ -137,7 +137,7 @@ To clear the environment first, and do some additional automatic initialisation of the interactive shell: ```console -$ nix-shell '<nixpkgs>' -A pan --pure \ +$ nix-shell '<nixpkgs>' --attr pan --pure \ --command 'export NIX_DEBUG=1; export NIX_CORES=8; return' ``` @@ -146,13 +146,13 @@ Nix expressions can also be given on the command line using the `-E` and packages `sqlite` and `libX11`: ```console -$ nix-shell -E 'with import <nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""' +$ nix-shell --expr 'with import <nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""' ``` A shorter way to do the same is: ```console -$ nix-shell -p sqlite xorg.libX11 +$ nix-shell --packages sqlite xorg.libX11 [nix-shell]$ echo $NIX_LDFLAGS … -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib … ``` @@ -162,7 +162,7 @@ the `buildInputs = [ ... ]` shown above, not only package names. So the following is also legal: ```console -$ nix-shell -p sqlite 'git.override { withManual = false; }' +$ nix-shell --packages sqlite 'git.override { withManual = false; }' ``` The `-p` flag looks up Nixpkgs in the Nix search path. You can override @@ -171,7 +171,7 @@ gives you a shell containing the Pan package from a specific revision of Nixpkgs: ```console -$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz +$ nix-shell --packages pan -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz [nix-shell:~]$ pan --version Pan 0.139 @@ -185,7 +185,7 @@ done by starting the script with the following lines: ```bash #! /usr/bin/env nix-shell -#! nix-shell -i real-interpreter -p packages +#! nix-shell -i real-interpreter --packages packages ``` where *real-interpreter* is the “real” script interpreter that will be @@ -202,7 +202,7 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python -p python pythonPackages.prettytable +#! nix-shell -i python --packages python pythonPackages.prettytable import prettytable @@ -217,7 +217,7 @@ requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: ```perl #! /usr/bin/env nix-shell -#! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP +#! nix-shell -i perl --packages perl perlPackages.HTMLTokeParserSimple perlPackages.LWP use HTML::TokeParser::Simple; @@ -235,7 +235,7 @@ package like Terraform: ```bash #! /usr/bin/env nix-shell -#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])" terraform apply ``` @@ -251,7 +251,7 @@ branch): ```haskell #! /usr/bin/env nix-shell -#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" +#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz import Network.Curl.Download diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md index 62656d599..c2f3c42ef 100644 --- a/doc/manual/src/command-ref/nix-store/dump.md +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -23,7 +23,7 @@ produce the same NAR archive. For instance, directory entries are always sorted so that the actual on-disk order doesn’t influence the result. This means that the cryptographic hash of a NAR dump of a path is usable as a fingerprint of the contents of the path. Indeed, -the hashes of store paths stored in Nix’s database (see `nix-store -q +the hashes of store paths stored in Nix’s database (see `nix-store --query --hash`) are SHA-256 hashes of the NAR dump of each store path. NAR archives support filenames of unlimited length and 64-bit file diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md index aeea38636..1bc46f53b 100644 --- a/doc/manual/src/command-ref/nix-store/export.md +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -31,7 +31,7 @@ To copy a whole closure, do something like: ```console -$ nix-store --export $(nix-store -qR paths) > out +$ nix-store --export $(nix-store --query --requisites paths) > out ``` To import the whole closure again, run: diff --git a/doc/manual/src/command-ref/nix-store/opt-common.md b/doc/manual/src/command-ref/nix-store/opt-common.md index bf6566555..dd9a6bf21 100644 --- a/doc/manual/src/command-ref/nix-store/opt-common.md +++ b/doc/manual/src/command-ref/nix-store/opt-common.md @@ -11,7 +11,7 @@ The following options are allowed for all `nix-store` operations, but may not al be created in `/nix/var/nix/gcroots/auto/`. For instance, ```console - $ nix-store --add-root /home/eelco/bla/result -r ... + $ nix-store --add-root /home/eelco/bla/result --realise ... $ ls -l /nix/var/nix/gcroots/auto lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index 9f7dbd3e8..cd45a4932 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -145,7 +145,7 @@ Print the closure (runtime dependencies) of the `svn` program in the current user environment: ```console -$ nix-store -qR $(which svn) +$ nix-store --query --requisites $(which svn) /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 /nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4 ... @@ -154,7 +154,7 @@ $ nix-store -qR $(which svn) Print the build-time dependencies of `svn`: ```console -$ nix-store -qR $(nix-store -qd $(which svn)) +$ nix-store --query --requisites $(nix-store --query --deriver $(which svn)) /nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv /nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh /nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv @@ -168,7 +168,7 @@ the derivation (`-qd`), not the closure of the output path that contains Show the build-time dependencies as a tree: ```console -$ nix-store -q --tree $(nix-store -qd $(which svn)) +$ nix-store --query --tree $(nix-store --query --deriver $(which svn)) /nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv +---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh +---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv @@ -180,7 +180,7 @@ $ nix-store -q --tree $(nix-store -qd $(which svn)) Show all paths that depend on the same OpenSSL library as `svn`: ```console -$ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn))) +$ nix-store --query --referrers $(nix-store --query --binding openssl $(nix-store --query --deriver $(which svn))) /nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0 /nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 /nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3 @@ -191,7 +191,7 @@ Show all paths that directly or indirectly depend on the Glibc (C library) used by `svn`: ```console -$ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}') +$ nix-store --query --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}') /nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2 /nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4 ... @@ -204,7 +204,7 @@ Make a picture of the runtime dependency graph of the current user environment: ```console -$ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps +$ nix-store --query --graph ~/.nix-profile | dot -Tps > graph.ps $ gv graph.ps ``` @@ -212,7 +212,7 @@ Show every garbage collector root that points to a store path that depends on `svn`: ```console -$ nix-store -q --roots $(which svn) +$ nix-store --query --roots $(which svn) /nix/var/nix/profiles/default-81-link /nix/var/nix/profiles/default-82-link /home/eelco/.local/state/nix/profiles/profile-97-link diff --git a/doc/manual/src/command-ref/nix-store/read-log.md b/doc/manual/src/command-ref/nix-store/read-log.md index 4a88e9382..d1ff17891 100644 --- a/doc/manual/src/command-ref/nix-store/read-log.md +++ b/doc/manual/src/command-ref/nix-store/read-log.md @@ -27,7 +27,7 @@ substitute, then the log is unavailable. # Example ```console -$ nix-store -l $(which ktorrent) +$ nix-store --read-log $(which ktorrent) building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1 unpacking sources unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index f61a20100..c19aea75e 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -54,36 +54,7 @@ The following flags are available: previous build, the new output path is left in `/nix/store/name.check.` -Special exit codes: - - - `100`\ - Generic build failure, the builder process returned with a non-zero - exit code. - - - `101`\ - Build timeout, the build was aborted because it did not complete - within the specified `timeout`. - - - `102`\ - Hash mismatch, the build output was rejected because it does not - match the [`outputHash` attribute of the - derivation](@docroot@/language/advanced-attributes.md). - - - `104`\ - Not deterministic, the build succeeded in check mode but the - resulting output is not binary reproducible. - -With the `--keep-going` flag it's possible for multiple failures to -occur, in this case the 1xx status codes are or combined using binary -or. - - 1100100 - ^^^^ - |||`- timeout - ||`-- output hash mismatch - |`--- build failure - `---- not deterministic - +{{#include ../status-build-failure.md}} {{#include ./opt-common.md}} @@ -99,7 +70,7 @@ This operation is typically used to build [store derivation]s produced by [store derivation]: @docroot@/glossary.md#gloss-store-derivation ```console -$ nix-store -r $(nix-instantiate ./test.nix) +$ nix-store --realise $(nix-instantiate ./test.nix) /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 ``` @@ -108,7 +79,7 @@ This is essentially what [`nix-build`](@docroot@/command-ref/nix-build.md) does. To test whether a previously-built derivation is deterministic: ```console -$ nix-build '<nixpkgs>' -A hello --check -K +$ nix-build '<nixpkgs>' --attr hello --check -K ``` Use [`nix-store --read-log`](./read-log.md) to show the stderr and stdout of a build: diff --git a/doc/manual/src/command-ref/nix-store/verify-path.md b/doc/manual/src/command-ref/nix-store/verify-path.md index 59ffe92a3..927201599 100644 --- a/doc/manual/src/command-ref/nix-store/verify-path.md +++ b/doc/manual/src/command-ref/nix-store/verify-path.md @@ -24,6 +24,6 @@ path has changed, and 1 otherwise. To verify the integrity of the `svn` command and all its dependencies: ```console -$ nix-store --verify-path $(nix-store -qR $(which svn)) +$ nix-store --verify-path $(nix-store --query --requisites $(which svn)) ``` diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 7a012250d..54c0a1d0d 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -162,11 +162,11 @@ Most Nix commands accept the following command-line options: }: ... ``` - So if you call this Nix expression (e.g., when you do `nix-env -iA + So if you call this Nix expression (e.g., when you do `nix-env --install --attr pkgname`), the function will be called automatically using the value [`builtins.currentSystem`](@docroot@/language/builtins.md) for the `system` argument. You can override this using `--arg`, e.g., - `nix-env -iA pkgname --arg system \"i686-freebsd\"`. (Note that + `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the quotes.) @@ -199,7 +199,7 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for - use, give your expression to the `nix-shell -p` convenience flag + use, give your expression to the `nix-shell --packages ` convenience flag instead. - <span id="opt-I">[`-I`](#opt-I)</span> *path*\ diff --git a/doc/manual/src/command-ref/status-build-failure.md b/doc/manual/src/command-ref/status-build-failure.md new file mode 100644 index 000000000..06114eb29 --- /dev/null +++ b/doc/manual/src/command-ref/status-build-failure.md @@ -0,0 +1,34 @@ +# Special exit codes for build failure + +1xx status codes are used when requested builds failed. +The following codes are in use: + +- `100` Generic build failure + + The builder process returned with a non-zero exit code. + +- `101` Build timeout + + The build was aborted because it did not complete within the specified `timeout`. + +- `102` Hash mismatch + + The build output was rejected because it does not match the + [`outputHash` attribute of the derivation](@docroot@/language/advanced-attributes.md). + +- `104` Not deterministic + + The build succeeded in check mode but the resulting output is not binary reproducible. + +With the `--keep-going` flag it's possible for multiple failures to occur. +In this case the 1xx status codes are or combined using +[bitwise OR](https://en.wikipedia.org/wiki/Bitwise_operation#OR). + +``` +0b1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic +``` diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/contributing/cxx.md new file mode 100644 index 000000000..ff9297158 --- /dev/null +++ b/doc/manual/src/contributing/cxx.md @@ -0,0 +1,28 @@ +# C++ style guide + +Some miscellaneous notes on how we write C++. +Formatting we hope to eventually normalize automatically, so this section is free to just discuss higher-level concerns. + +## The `*-impl.hh` pattern + +Let's start with some background info first. +Headers, are supposed to contain declarations, not definitions. +This allows us to change a definition without changing the declaration, and have a very small rebuild during development. +Templates, however, need to be specialized to use-sites. +Absent fancier techniques, templates require that the definition, not just mere declaration, must be available at use-sites in order to make that specialization on the fly as part of compiling those use-sites. +Making definitions available like that means putting them in headers, but that is unfortunately means we get all the extra rebuilds we want to avoid by just putting declarations there as described above. + +The `*-impl.hh` pattern is a ham-fisted partial solution to this problem. +It constitutes: + +- Declaring items only in the main `foo.hh`, including templates. + +- Putting template definitions in a companion `foo-impl.hh` header. + +Most C++ developers would accompany this by having `foo.hh` include `foo-impl.hh`, to ensure any file getting the template declarations also got the template definitions. +But we've found not doing this has some benefits and fewer than imagined downsides. +The fact remains that headers are rarely as minimal as they could be; +there is often code that needs declarations from the headers but not the templates within them. +With our pattern where `foo.hh` doesn't include `foo-impl.hh`, that means they can just include `foo.hh` +Code that needs both just includes `foo.hh` and `foo-impl.hh`. +This does make linking error possible where something forgets to include `foo-impl.hh` that needs it, but those are build-time only as easy to fix. diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index ca69f076a..7b2440971 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -12,14 +12,15 @@ The following instructions assume you already have some version of Nix installed [installation instructions]: ../installation/installation.md -## Nix with flakes +## Building Nix with flakes -This section assumes you are using Nix with [flakes] enabled. See the [next section](#classic-nix) for equivalent instructions which don't require flakes. +This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. +See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[flakes]: ../command-ref/new-cli/nix3-flake.md#description +[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command -To build all dependencies and start a shell in which all environment -variables are set up so that those dependencies can be found: +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console $ nix develop @@ -55,20 +56,17 @@ To install it in `$(pwd)/outputs` and test it: nix (Nix) 2.12 ``` -To build a release version of Nix: +To build a release version of Nix for the current operating system and CPU architecture: ```console $ nix build ``` -You can also build Nix for one of the [supported target platforms](#target-platforms). +You can also build Nix for one of the [supported platforms](#platforms). -## Classic Nix +## Building Nix -This section is for Nix without [flakes]. - -To build all dependencies and start a shell in which all environment -variables are set up so that those dependencies can be found: +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console $ nix-shell @@ -77,7 +75,7 @@ $ nix-shell To get a shell with one of the other [supported compilation environments](#compilation-environments): ```console -$ nix-shell -A devShells.x86_64-linux.native-clang11StdenvPackages +$ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages ``` > **Note** @@ -102,13 +100,13 @@ To install it in `$(pwd)/outputs` and test it: nix (Nix) 2.12 ``` -To build Nix for the current operating system and CPU architecture use +To build a release version of Nix for the current operating system and CPU architecture: ```console $ nix-build ``` -You can also build Nix for one of the [supported target platforms](#target-platforms). +You can also build Nix for one of the [supported platforms](#platforms). ## Platforms @@ -139,7 +137,7 @@ $ nix build .#packages.aarch64-linux.default for flake-enabled Nix, or ```console -$ nix-build -A packages.aarch64-linux.default +$ nix-build --attr packages.aarch64-linux.default ``` for classic Nix. @@ -166,7 +164,7 @@ $ nix build .#nix-ccacheStdenv for flake-enabled Nix, or ```console -$ nix-build -A nix-ccacheStdenv +$ nix-build --attr nix-ccacheStdenv ``` for classic Nix. @@ -192,171 +190,6 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -## Running tests - -### Unit-tests - -The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined -under `src/{library_name}/tests` using the -[googletest](https://google.github.io/googletest/) and -[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. - -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. - -### Functional tests - -The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. -Each test is a bash script. - -The whole test suite can be run with: - -```shell-session -$ make install && make installcheck -ran test tests/foo.sh... [PASS] -ran test tests/bar.sh... [PASS] -... -``` - -Individual tests can be run with `make`: - -```shell-session -$ make tests/${testName}.sh.test -ran test tests/${testName}.sh... [PASS] -``` - -or without `make`: - -```shell-session -$ ./mk/run-test.sh tests/${testName}.sh -ran test tests/${testName}.sh... [PASS] -``` - -To see the complete output, one can also run: - -```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh -+ foo -output from foo -+ bar -output from bar -... -``` - -The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. - -#### Debugging failing functional tests - -When a functional test fails, it usually does so somewhere in the middle of the script. - -To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. - -For example, if the script looks like: - -```bash -foo -nix blah blub -bar -``` -edit it like so: - -```diff - foo --nix blah blub -+gdb --args nix blah blub - bar -``` - -Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: - -```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh -... -+ gdb blash blub -GNU gdb (GDB) 12.1 -... -(gdb) -``` - -One can debug the Nix invocation in all the usual ways. -For example, enter `run` to start the Nix invocation. - -### Integration tests - -The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. -These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. -Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>). - -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` - -### Installer tests - -After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. - -Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): - -- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: - - `x86_64-linux` - - `armv6l-linux` - - `armv7l-linux` - - `x86_64-darwin` - -- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. - -#### One-time setup - -1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). -2. At cachix.org: - - Create or log in to an account. - - Create a Cachix cache using the format `<github-username>-nix-install-tests`. - - Navigate to the new cache > Settings > Auth Tokens. - - Generate a new Cachix auth token and copy the generated value. -3. At github.com: - - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. - - Name the secret `CACHIX_AUTH_TOKEN`. - - Paste the copied value of the Cachix cache auth token. - -#### Using the CI-generated installer for manual testing - -After the CI run completes, you can check the output to extract the installer URL: -1. Click into the detailed view of the CI run. -2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). -3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) -4. Copy the value of `install_url` -5. To generate an install command, plug this `install_url` and your GitHub username into this template: - - ```console - curl -L <install_url> | sh -s -- --tarball-url-prefix https://<github-username>-nix-install-tests.cachix.org/serve - ``` - -<!-- #### Manually generating test installers - -There's obviously a manual way to do this, and it's still the only way for -platforms that lack GA runners. - -I did do this back in Fall 2020 (before the GA approach encouraged here). I'll -sketch what I recall in case it encourages someone to fill in detail, but: I -didn't know what I was doing at the time and had to fumble/ask around a lot-- -so I don't want to uphold any of it as "right". It may have been dumb or -the _hard_ way from the getgo. Fundamentals may have changed since. - -Here's the build command I used to do this on and for x86_64-darwin: -nix build --out-link /tmp/foo ".#checks.x86_64-darwin.binaryTarball" - -I used the stable out-link to make it easier to script the next steps: -link=$(readlink /tmp/foo) -cp $link/*-darwin.tar.xz ~/somewheres - -I've lost the last steps and am just going from memory: - -From here, I think I had to extract and modify the `install` script to point -it at this tarball (which I scped to my own site, but it might make more sense -to just share them locally). I extracted this script once and then just -search/replaced in it for each new build. - -The installer now supports a `--tarball-url-prefix` flag which _may_ have -solved this need? ---> - ### Checking links in the manual The build checks for broken internal links. @@ -378,7 +211,7 @@ rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/comman [`mdbook-linkcheck`] does not implement checking [URI fragments] yet. [`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck -[URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment +[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment #### `@docroot@` variable diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md new file mode 100644 index 000000000..e5f80a928 --- /dev/null +++ b/doc/manual/src/contributing/testing.md @@ -0,0 +1,167 @@ +# Running tests + +## Unit-tests + +The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined +under `src/{library_name}/tests` using the +[googletest](https://google.github.io/googletest/) and +[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. + +You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. + +## Functional tests + +The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +Each test is a bash script. + +The whole test suite can be run with: + +```shell-session +$ make install && make installcheck +ran test tests/foo.sh... [PASS] +ran test tests/bar.sh... [PASS] +... +``` + +Individual tests can be run with `make`: + +```shell-session +$ make tests/${testName}.sh.test +ran test tests/${testName}.sh... [PASS] +``` + +or without `make`: + +```shell-session +$ ./mk/run-test.sh tests/${testName}.sh +ran test tests/${testName}.sh... [PASS] +``` + +To see the complete output, one can also run: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh ++ foo +output from foo ++ bar +output from bar +... +``` + +The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. + +### Debugging failing functional tests + +When a functional test fails, it usually does so somewhere in the middle of the script. + +To figure out what's wrong, it is convenient to run the test regularly up to the failing `nix` command, and then run that command with a debugger like GDB. + +For example, if the script looks like: + +```bash +foo +nix blah blub +bar +``` +edit it like so: + +```diff + foo +-nix blah blub ++gdb --args nix blah blub + bar +``` + +Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: + +```shell-session +$ ./mk/debug-test.sh tests/${testName}.sh +... ++ gdb blash blub +GNU gdb (GDB) 12.1 +... +(gdb) +``` + +One can debug the Nix invocation in all the usual ways. +For example, enter `run` to start the Nix invocation. + +## Integration tests + +The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. +These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. +Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>). + +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` + +## Installer tests + +After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. + +Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): + +- The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: + - `x86_64-linux` + - `armv6l-linux` + - `armv7l-linux` + - `x86_64-darwin` + +- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. + +### One-time setup + +1. Have a GitHub account with a fork of the [Nix repository](https://github.com/NixOS/nix). +2. At cachix.org: + - Create or log in to an account. + - Create a Cachix cache using the format `<github-username>-nix-install-tests`. + - Navigate to the new cache > Settings > Auth Tokens. + - Generate a new Cachix auth token and copy the generated value. +3. At github.com: + - Navigate to your Nix fork > Settings > Secrets > Actions > New repository secret. + - Name the secret `CACHIX_AUTH_TOKEN`. + - Paste the copied value of the Cachix cache auth token. + +## Working on documentation + +### Using the CI-generated installer for manual testing + +After the CI run completes, you can check the output to extract the installer URL: +1. Click into the detailed view of the CI run. +2. Click into any `installer_test` run (the URL you're here to extract will be the same in all of them). +3. Click into the `Run cachix/install-nix-action@v...` step and click the detail triangle next to the first log line (it will also be `Run cachix/install-nix-action@v...`) +4. Copy the value of `install_url` +5. To generate an install command, plug this `install_url` and your GitHub username into this template: + + ```console + curl -L <install_url> | sh -s -- --tarball-url-prefix https://<github-username>-nix-install-tests.cachix.org/serve + ``` + +<!-- #### Manually generating test installers + +There's obviously a manual way to do this, and it's still the only way for +platforms that lack GA runners. + +I did do this back in Fall 2020 (before the GA approach encouraged here). I'll +sketch what I recall in case it encourages someone to fill in detail, but: I +didn't know what I was doing at the time and had to fumble/ask around a lot-- +so I don't want to uphold any of it as "right". It may have been dumb or +the _hard_ way from the getgo. Fundamentals may have changed since. + +Here's the build command I used to do this on and for x86_64-darwin: +nix build --out-link /tmp/foo ".#checks.x86_64-darwin.binaryTarball" + +I used the stable out-link to make it easier to script the next steps: +link=$(readlink /tmp/foo) +cp $link/*-darwin.tar.xz ~/somewheres + +I've lost the last steps and am just going from memory: + +From here, I think I had to extract and modify the `install` script to point +it at this tarball (which I scped to my own site, but it might make more sense +to just share them locally). I extracted this script once and then just +search/replaced in it for each new build. + +The installer now supports a `--tarball-url-prefix` flag which _may_ have +solved this need? +--> + diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index eeb19ad50..ac0bb3c2f 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -85,12 +85,17 @@ [store path]: #gloss-store-path + - [file system object]{#gloss-store-object}\ + The Nix data model for representing simplified file system data. + + See [File System Object](@docroot@/architecture/file-system-object.md) for details. + + [file system object]: #gloss-file-system-object + - [store object]{#gloss-store-object}\ - A file that is an immediate child of the Nix store directory. These - can be regular files, but also entire directory trees. Store objects - can be sources (objects copied from outside of the store), - derivation outputs (objects produced by running a build task), or - derivations (files describing a build task). + + A store object consists of a [file system object], [reference]s to other store objects, and other metadata. + It can be referred to by a [store path]. [store object]: #gloss-store-object @@ -101,11 +106,8 @@ derivation. - [output-addressed store object]{#gloss-output-addressed-store-object}\ - A store object whose store path hashes its content. This - includes derivations, the outputs of - [content-addressed derivations](#gloss-content-addressed-derivation), - and the outputs of - [fixed-output derivations](#gloss-fixed-output-derivation). + A [store object] whose [store path] is determined by its contents. + This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). - [substitute]{#gloss-substitute}\ A substitute is a command invocation stored in the [Nix database] that @@ -115,9 +117,10 @@ from some server. - [substituter]{#gloss-substituter}\ - A *substituter* is an additional store from which Nix will - copy store objects it doesn't have. For details, see the - [`substituters` option](./command-ref/conf-file.md#conf-substituters). + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. + + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. [substituter]: #gloss-substituter @@ -163,7 +166,7 @@ build-time dependencies, while the closure of its output path is equivalent to its runtime dependencies. For correct deployment it is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command `nix-store -qR` prints out + files could be missing. The command `nix-store --query --requisites ` prints out closures of store paths. As an example, if the [store object] at path `P` contains a [reference] diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 5a708f11b..d4babf1ea 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -10,7 +10,7 @@ - Bash Shell. The `./configure` script relies on bashisms, so Bash is required. - - A version of GCC or Clang that supports C++17. + - A version of GCC or Clang that supports C++20. - `pkg-config` to locate dependencies. If your distribution does not provide it, you can get it from diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 24efc4681..6d09f54d8 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -2,13 +2,13 @@ Multi-user Nix users on macOS can upgrade Nix by running: `sudo -i sh -c 'nix-channel --update && -nix-env -iA nixpkgs.nix && +nix-env --install --attr nixpkgs.nix && launchctl remove org.nixos.nix-daemon && launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'` Single-user installations of Nix should run this: `nix-channel --update; -nix-env -iA nixpkgs.nix nixpkgs.cacert` +nix-env --install --attr nixpkgs.nix nixpkgs.cacert` Multi-user Nix users on Linux should run this with sudo: `nix-channel ---update; nix-env -iA nixpkgs.nix nixpkgs.cacert; systemctl +--update; nix-env --install --attr nixpkgs.nix nixpkgs.cacert; systemctl daemon-reload; systemctl restart nix-daemon` diff --git a/doc/manual/src/introduction.md b/doc/manual/src/introduction.md index b54346db8..76489bc1b 100644 --- a/doc/manual/src/introduction.md +++ b/doc/manual/src/introduction.md @@ -76,7 +76,7 @@ there after an upgrade. This means that you can _roll back_ to the old version: ```console -$ nix-env --upgrade -A nixpkgs.some-package +$ nix-env --upgrade --attr nixpkgs.some-package $ nix-env --rollback ``` @@ -122,7 +122,7 @@ Nix expressions generally describe how to build a package from source, so an installation action like ```console -$ nix-env --install -A nixpkgs.firefox +$ nix-env --install --attr nixpkgs.firefox ``` _could_ cause quite a bit of build activity, as not only Firefox but @@ -158,7 +158,7 @@ Pan newsreader, as described by [its Nix expression](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/newsreaders/pan/default.nix): ```console -$ nix-shell '<nixpkgs>' -A pan +$ nix-shell '<nixpkgs>' --attr pan ``` You’re then dropped into a shell where you can edit, build and test diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md new file mode 100644 index 000000000..50f43006d --- /dev/null +++ b/doc/manual/src/language/builtin-constants-prefix.md @@ -0,0 +1,5 @@ +# Built-in Constants + +These constants are built into the Nix language evaluator: + +<dl> diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md new file mode 100644 index 000000000..a74db2857 --- /dev/null +++ b/doc/manual/src/language/builtin-constants-suffix.md @@ -0,0 +1 @@ +</dl> diff --git a/doc/manual/src/language/builtin-constants.md b/doc/manual/src/language/builtin-constants.md deleted file mode 100644 index c6bc9b74c..000000000 --- a/doc/manual/src/language/builtin-constants.md +++ /dev/null @@ -1,19 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - -- [`builtins`]{#builtins-builtins} (attribute set) - - Contains all the [built-in functions](./builtins.md) and values, in order to avoid polluting the global scope. - - Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: - - ```nix - if builtins ? getEnv then builtins.getEnv "PATH" else "" - ``` - -- [`builtins.currentSystem`]{#builtins-currentSystem} (string) - - The built-in value `currentSystem` evaluates to the Nix platform - identifier for the Nix installation on which the expression is being - evaluated, such as `"i686-linux"` or `"x86_64-darwin"`. diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 35e3dccc3..7b2321466 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -3,7 +3,7 @@ This section lists the functions built into the Nix language evaluator. All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. -For convenience, some built-ins are can be accessed directly: +For convenience, some built-ins can be accessed directly: - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index 1c01f2cc7..c53eb8889 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -2,8 +2,11 @@ ## Recursive sets -Recursive sets are just normal sets, but the attributes can refer to -each other. For example, +Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: ```nix rec { @@ -12,7 +15,9 @@ rec { }.x ``` -evaluates to `123`. Note that without `rec` the binding `x = y;` would +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would refer to the variable `y` in the surrounding scope, if one exists, and would be invalid if no such variable exists. That is, in a normal (non-recursive) set, attributes are not added to the lexical scope; in a @@ -33,7 +38,10 @@ will crash with an `infinite recursion encountered` error message. ## Let-expressions A let-expression allows you to define local variables for an expression. -For instance, + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: ```nix let @@ -42,18 +50,19 @@ let in x + y ``` -evaluates to `"foobar"`. +This evaluates to `"foobar"`. ## Inheriting attributes -When defining a set or in a let-expression it is often convenient to -copy variables from the surrounding lexical scope (e.g., when you want -to propagate attributes). This can be shortened using the `inherit` -keyword. For instance, +When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: ```nix let x = 123; in -{ inherit x; +{ + inherit x; y = 456; } ``` @@ -62,15 +71,23 @@ is equivalent to ```nix let x = 123; in -{ x = x; +{ + x = x; y = 456; } ``` -and both evaluate to `{ x = 123; y = 456; }`. (Note that this works -because `x` is added to the lexical scope by the `let` construct.) It is -also possible to inherit attributes from another set. For instance, in -this fragment from `all-packages.nix`, +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, ```nix graphviz = (import ../tools/graphics/graphviz) { diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index 3eabe1a02..29950a52d 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,12 +1,11 @@ # Nix Language -The Nix language is +The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. +It is: - *domain-specific* - It only exists for the Nix package manager: - to describe packages and configurations as well as their variants and compositions. - It is not intended for general purpose use. + It comes with [built-in functions](@docroot@/language/builtins.md) to integrate with the Nix store, which manages files and performs the derivations declared in the Nix language. - *declarative* @@ -25,7 +24,7 @@ The Nix language is - *lazy* - Expressions are only evaluated when their value is needed. + Values are only computed when they are needed. - *dynamically typed* diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 3e929724d..f8382ae19 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -35,17 +35,14 @@ ## Attribute selection +> *attrset* `.` *attrpath* \[ `or` *expr* \] + Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -<!-- FIXME: the following should to into its own language syntax section, but that needs more work to fit in well --> - -An attribute path is a dot-separated list of attribute names. -An attribute name can be an identifier or a string. +An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). -> *attrpath* = *name* [ `.` *name* ]... \ -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` +> *attrpath* = *name* [ `.` *name* ]... [Attribute selection]: #attribute-selection diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 9d0301753..2ae3e143a 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -164,9 +164,17 @@ Note that lists are only lazy in values, and they are strict in length. An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). +An attribute name can be an identifier or a [string](#string). +An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). + +> *name* = *identifier* | *string* \ +> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` + Names and values are separated by an equal sign (`=`). Each value is an arbitrary expression terminated by a semicolon (`;`). +> *attrset* = `{` [ *name* `=` *expr* `;` `]`... `}` + Attributes can appear in any order. An attribute name may only occur once. @@ -182,15 +190,19 @@ Example: This defines a set with attributes named `x`, `text`, `y`. -Attributes can be selected from a set using the `.` operator. For -instance, +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: ```nix { a = "Foo"; b = "Bar"; }.a ``` -evaluates to `"Foo"`. It is possible to provide a default value in an -attribute selection using the `or` keyword: +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: ```nix { a = "Foo"; b = "Bar"; }.c or "Xyzzy" diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md index 5f1d7a89c..6b86e763e 100644 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ b/doc/manual/src/package-management/basic-package-mgmt.md @@ -47,7 +47,7 @@ $ nix-channel --update You can view the set of available packages in Nixpkgs: ```console -$ nix-env -qaP +$ nix-env --query --available --attr-path nixpkgs.aterm aterm-2.2 nixpkgs.bash bash-3.0 nixpkgs.binutils binutils-2.15 @@ -65,7 +65,7 @@ If you downloaded Nixpkgs yourself, or if you checked it out from GitHub, then you need to pass the path to your Nixpkgs tree using the `-f` flag: ```console -$ nix-env -qaPf /path/to/nixpkgs +$ nix-env --query --available --attr-path --file /path/to/nixpkgs aterm aterm-2.2 bash bash-3.0 … @@ -77,7 +77,7 @@ Nixpkgs. You can filter the packages by name: ```console -$ nix-env -qaP firefox +$ nix-env --query --available --attr-path firefox nixpkgs.firefox-esr firefox-91.3.0esr nixpkgs.firefox firefox-94.0.1 ``` @@ -85,7 +85,7 @@ nixpkgs.firefox firefox-94.0.1 and using regular expressions: ```console -$ nix-env -qaP 'firefox.*' +$ nix-env --query --available --attr-path 'firefox.*' ``` It is also possible to see the *status* of available packages, i.e., @@ -93,7 +93,7 @@ whether they are installed into the user environment and/or present in the system: ```console -$ nix-env -qaPs +$ nix-env --query --available --attr-path --status … -PS nixpkgs.bash bash-3.0 --S nixpkgs.binutils binutils-2.15 @@ -110,10 +110,10 @@ which is Nix’s mechanism for doing binary deployment. It just means that Nix knows that it can fetch a pre-built package from somewhere (typically a network server) instead of building it locally. -You can install a package using `nix-env -iA`. For instance, +You can install a package using `nix-env --install --attr `. For instance, ```console -$ nix-env -iA nixpkgs.subversion +$ nix-env --install --attr nixpkgs.subversion ``` will install the package called `subversion` from `nixpkgs` channel (which is, of course, the @@ -143,14 +143,14 @@ instead of the attribute path, as `nix-env` does not record which attribute was used for installing: ```console -$ nix-env -e subversion +$ nix-env --uninstall subversion ``` Upgrading to a new version is just as easy. If you have a new release of Nix Packages, you can do: ```console -$ nix-env -uA nixpkgs.subversion +$ nix-env --upgrade --attr nixpkgs.subversion ``` This will *only* upgrade Subversion if there is a “newer” version in the @@ -163,15 +163,15 @@ whatever version is in the Nix expressions, use `-i` instead of `-u`; You can also upgrade all packages for which there are newer versions: ```console -$ nix-env -u +$ nix-env --upgrade ``` Sometimes it’s useful to be able to ask what `nix-env` would do, without actually doing it. For instance, to find out what packages would be -upgraded by `nix-env -u`, you can do +upgraded by `nix-env --upgrade `, you can do ```console -$ nix-env -u --dry-run +$ nix-env --upgrade --dry-run (dry run; not doing anything) upgrading `libxslt-1.1.0' to `libxslt-1.1.10' upgrading `graphviz-1.10' to `graphviz-1.12' diff --git a/doc/manual/src/package-management/binary-cache-substituter.md b/doc/manual/src/package-management/binary-cache-substituter.md index 5befad9f8..855eaf470 100644 --- a/doc/manual/src/package-management/binary-cache-substituter.md +++ b/doc/manual/src/package-management/binary-cache-substituter.md @@ -9,7 +9,7 @@ The daemon that handles binary cache requests via HTTP, `nix-serve`, is not part of the Nix distribution, but you can install it from Nixpkgs: ```console -$ nix-env -iA nixpkgs.nix-serve +$ nix-env --install --attr nixpkgs.nix-serve ``` You can then start the server, listening for HTTP connections on @@ -35,7 +35,7 @@ On the client side, you can tell Nix to use your binary cache using `--substituters`, e.g.: ```console -$ nix-env -iA nixpkgs.firefox --substituters http://avalon:8080/ +$ nix-env --install --attr nixpkgs.firefox --substituters http://avalon:8080/ ``` The option `substituters` tells Nix to use this binary cache in diff --git a/doc/manual/src/package-management/channels.md b/doc/manual/src/package-management/channels.md index 93c8b41a6..8e4da180b 100644 --- a/doc/manual/src/package-management/channels.md +++ b/doc/manual/src/package-management/channels.md @@ -43,7 +43,7 @@ operations (via the symlink `~/.nix-defexpr/channels`). Consequently, you can then say ```console -$ nix-env -u +$ nix-env --upgrade ``` to upgrade all packages in your profile to the latest versions available diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md index d3fac4d76..14326298b 100644 --- a/doc/manual/src/package-management/copy-closure.md +++ b/doc/manual/src/package-management/copy-closure.md @@ -15,7 +15,7 @@ With `nix-store path (that is, the path and all its dependencies) to a file, and then unpack that file into another Nix store. For example, - $ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure + $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure writes the closure of Firefox to a file. You can then copy this file to another machine and install the closure: @@ -27,7 +27,7 @@ store are ignored. It is also possible to pipe the export into another command, e.g. to copy and install a closure directly to/on another machine: - $ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \ + $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ ssh alice@itchy.example.org "bunzip2 | nix-store --import" However, `nix-copy-closure` is generally more efficient because it only diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/src/package-management/profiles.md index d1a2580d4..1d9e672a8 100644 --- a/doc/manual/src/package-management/profiles.md +++ b/doc/manual/src/package-management/profiles.md @@ -39,7 +39,7 @@ just Subversion 1.1.2 (arrows in the figure indicate symlinks). This would be what we would obtain if we had done ```console -$ nix-env -iA nixpkgs.subversion +$ nix-env --install --attr nixpkgs.subversion ``` on a set of Nix expressions that contained Subversion 1.1.2. @@ -54,7 +54,7 @@ environment is generated based on the current one. For instance, generation 43 was created from generation 42 when we did ```console -$ nix-env -iA nixpkgs.subversion nixpkgs.firefox +$ nix-env --install --attr nixpkgs.subversion nixpkgs.firefox ``` on a set of Nix expressions that contained Firefox and a new version of @@ -127,7 +127,7 @@ All `nix-env` operations work on the profile pointed to by (abbreviation `-p`): ```console -$ nix-env -p /nix/var/nix/profiles/other-profile -iA nixpkgs.subversion +$ nix-env --profile /nix/var/nix/profiles/other-profile --install --attr nixpkgs.subversion ``` This will *not* change the `~/.nix-profile` symlink. diff --git a/doc/manual/src/package-management/ssh-substituter.md b/doc/manual/src/package-management/ssh-substituter.md index c59933f61..7014c3cc8 100644 --- a/doc/manual/src/package-management/ssh-substituter.md +++ b/doc/manual/src/package-management/ssh-substituter.md @@ -6,7 +6,7 @@ automatically fetching any store paths in Firefox’s closure if they are available on the server `avalon`: ```console -$ nix-env -iA nixpkgs.firefox --substituters ssh://alice@avalon +$ nix-env --install --attr nixpkgs.firefox --substituters ssh://alice@avalon ``` This works similar to the binary cache substituter that Nix usually @@ -25,7 +25,7 @@ You can also copy the closure of some store path, without installing it into your profile, e.g. ```console -$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters +$ nix-store --realise /nix/store/m85bxg…-firefox-34.0.5 --substituters ssh://alice@avalon ``` diff --git a/doc/manual/src/protocols/protocols.md b/doc/manual/src/protocols/protocols.md new file mode 100644 index 000000000..d6bf1d809 --- /dev/null +++ b/doc/manual/src/protocols/protocols.md @@ -0,0 +1,4 @@ +# Protocols + +This chapter documents various developer-facing interfaces provided by +Nix. diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md new file mode 100644 index 000000000..0d3212303 --- /dev/null +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -0,0 +1,42 @@ +# Lockable HTTP Tarball Protocol + +Tarball flakes can be served as regular tarballs via HTTP or the file +system (for `file://` URLs). Unless the server implements the Lockable +HTTP Tarball protocol, it is the responsibility of the user to make sure that +the URL always produces the same tarball contents. + +An HTTP server can return an "immutable" HTTP URL appropriate for lock +files. This allows users to specify a tarball flake input in +`flake.nix` that requests the latest version of a flake +(e.g. `https://example.org/hello/latest.tar.gz`), while `flake.lock` +will record a URL whose contents will not change +(e.g. `https://example.org/hello/<revision>.tar.gz`). To do so, the +server must return an [HTTP `Link` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link) with the `rel` attribute set to +`immutable`, as follows: + +``` +Link: <flakeref>; rel="immutable" +``` + +(Note the required `<` and `>` characters around *flakeref*.) + +*flakeref* must be a tarball flakeref. It can contain flake attributes +such as `narHash`, `rev` and `revCount`. If `narHash` is included, its +value must be the NAR hash of the unpacked tarball (as computed via +`nix hash path`). Nix checks the contents of the returned tarball +against the `narHash` attribute. The `rev` and `revCount` attributes +are useful when the tarball flake is a mirror of a fetcher type that +has those attributes, such as Git or GitHub. They are not checked by +Nix. + +``` +Link: <https://example.org/hello/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz + ?rev=442793d9ec0584f6a6e82fa253850c8085bb150a + &revCount=835 + &narHash=sha256-GUm8Uh/U74zFCwkvt9Mri4DSM%2BmHj3tYhXUkYpiv31M%3D>; rel="immutable" +``` + +(The linebreaks in this example are for clarity and must not be included in the actual response.) + +For tarball flakes, the value of the `lastModified` flake attribute is +defined as the timestamp of the newest file inside the tarball. diff --git a/doc/manual/src/release-notes/rl-2.16.md b/doc/manual/src/release-notes/rl-2.16.md new file mode 100644 index 000000000..97b40d0b8 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.16.md @@ -0,0 +1,8 @@ +# Release 2.16 (2023-05-31) + +* Speed-up of downloads from binary caches. + The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs). + The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs). + The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs). + +* The function [`builtins.replaceStrings`](@docroot@/language/builtins.md#builtins-replaceStrings) is now lazy in the value of its second argument `to`. That is, `to` is only evaluated when its corresponding pattern in `from` is matched in the string `s`. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index bc0d41bdf..139d07188 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,6 +1,8 @@ # Release X.Y (202?-??-??) -- Speed-up of downloads from binary caches. - The number of parallel downloads (also known as substitutions) has been separated from the [`--max-jobs` setting](../command-ref/conf-file.md#conf-max-jobs). - The new setting is called [`max-substitution-jobs`](../command-ref/conf-file.md#conf-max-substitution-jobs). - The number of parallel downloads is now set to 16 by default (previously, the default was 1 due to the coupling to build jobs). +- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand + +* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. + +- Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. + Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. diff --git a/docker.nix b/docker.nix index 52199af66..bd16b71cd 100644 --- a/docker.nix +++ b/docker.nix @@ -190,6 +190,12 @@ let cp -a ${rootEnv}/* $out/ ln -s ${manifest} $out/manifest.nix ''; + flake-registry-path = if (flake-registry == null) then + null + else if (builtins.readFileType (toString flake-registry)) == "directory" then + "${flake-registry}/flake-registry.json" + else + flake-registry; in pkgs.runCommand "base-system" { @@ -202,7 +208,7 @@ let ]; allowSubstitutes = false; preferLocalBuild = true; - } '' + } ('' env set -x mkdir -p $out/etc @@ -249,15 +255,15 @@ let ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh - '' + (lib.optionalString (flake-registry != null) '' + '' + (lib.optionalString (flake-registry-path != null) '' nixCacheDir="/root/.cache/nix" mkdir -p $out$nixCacheDir globalFlakeRegistryPath="$nixCacheDir/flake-registry.json" - ln -s ${flake-registry}/flake-registry.json $out$globalFlakeRegistryPath + ln -s ${flake-registry-path} $out$globalFlakeRegistryPath mkdir -p $out/nix/var/nix/gcroots/auto rootName=$(${pkgs.nix}/bin/nix --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName - ''); + '')); in pkgs.dockerTools.buildLayeredImageWithNixDb { @@ -590,6 +590,8 @@ tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix; + tests.tarballFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/tarball-flakes.nix; + tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; tests.setuid = lib.genAttrs diff --git a/maintainers/README.md b/maintainers/README.md index d13349438..0d520cb0c 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -117,6 +117,7 @@ Pull requests in this column are reviewed together during work meetings. This is both for spreading implementation knowledge and for establishing common values in code reviews. When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member. +If significant changes are requested or reviewers cannot come to a conclusion in reasonable time, the pull request is [marked as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request#converting-a-pull-request-to-a-draft). ### Assigned diff --git a/maintainers/release-process.md b/maintainers/release-process.md index ec9e96489..d85266b81 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -119,8 +119,7 @@ release: TODO: This script requires the right AWS credentials. Document. TODO: This script currently requires a - `/home/eelco/Dev/nix-pristine` and - `/home/eelco/Dev/nixpkgs-pristine`. + `/home/eelco/Dev/nix-pristine`. TODO: trigger nixos.org netlify: https://docs.netlify.com/configure-builds/build-hooks/ @@ -141,7 +140,7 @@ release: $ git checkout master $ git pull $ NEW_VERSION=2.13.0 - $ echo -n $NEW_VERSION > .version + $ echo $NEW_VERSION > .version $ git checkout -b bump-$NEW_VERSION $ git commit -a -m 'Bump version' $ git push --set-upstream origin bump-$NEW_VERSION diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 77469148a..ebc536f12 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -15,7 +15,6 @@ my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; my $channelsBucketName = "nix-channels"; -my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; my $TMPDIR = $ENV{'TMPDIR'} // "/tmp"; @@ -81,6 +80,38 @@ my $s3_us = Net::Amazon::S3->new( my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; +sub getStorePath { + my ($jobName, $output) = @_; + my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; +} + +sub copyManual { + my $manual = getStorePath("build.x86_64-linux", "doc"); + print "$manual\n"; + + my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; + print "$manualNar\n"; + + unless (-e $manualNar) { + system("NIX_REMOTE=$binaryCache nix store dump-path '$manual' | xz > '$manualNar'.tmp") == 0 + or die "unable to fetch $manual\n"; + rename("$manualNar.tmp", $manualNar) or die; + } + + unless (-e "$tmpDir/manual") { + system("xz -d < '$manualNar' | nix-store --restore $tmpDir/manual.tmp") == 0 + or die "unable to unpack $manualNar\n"; + rename("$tmpDir/manual.tmp/share/doc/nix/manual", "$tmpDir/manual") or die; + system("rm -rf '$tmpDir/manual.tmp'") == 0 or die; + } + + system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0 + or die "syncing manual to S3\n"; +} + +copyManual; + sub downloadFile { my ($jobName, $productNr, $dstName) = @_; @@ -180,9 +211,20 @@ if ($isLatest) { system("docker manifest push nixos/nix:latest") == 0 or die; } +# Upload nix-fallback-paths.nix. +write_file("$tmpDir/fallback-paths.nix", + "{\n" . + " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + "}\n"); + # Upload release files to S3. for my $fn (glob "$tmpDir/*") { my $name = basename($fn); + next if $name eq "manual"; my $dstKey = "$releaseDir/" . $name; unless (defined $releasesBucket->head_key($dstKey)) { print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n"; @@ -190,8 +232,7 @@ for my $fn (glob "$tmpDir/*") { my $configuration = (); $configuration->{content_type} = "application/octet-stream"; - if ($fn =~ /.sha256|install/) { - # Text files + if ($fn =~ /.sha256|install|\.nix$/) { $configuration->{content_type} = "text/plain"; } @@ -200,28 +241,6 @@ for my $fn (glob "$tmpDir/*") { } } -# Update nix-fallback-paths.nix. -if ($isLatest) { - system("cd $nixpkgsDir && git pull") == 0 or die; - - sub getStorePath { - my ($jobName) = @_; - my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'"; - } - - write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", - "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . - "}\n"); - - system("cd $nixpkgsDir && git commit -a -m 'nix-fallback-paths.nix: Update to $version'") == 0 or die; -} - # Update the "latest" symlink. $channelsBucket->add_key( "nix-latest/install", "", diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index f46413630..45fbea02c 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -10,6 +10,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process LimitNOFILE=1048576 +TasksMax=1048576 [Install] WantedBy=multi-user.target diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 5111a5dde..0326d3415 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -100,7 +100,7 @@ poly_extra_try_me_commands() { poly_configure_nix_daemon_service() { task "Setting up the nix-daemon LaunchDaemon" _sudo "to set up the nix-daemon as a LaunchDaemon" \ - /bin/cp -f "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" + /usr/bin/install -m -rw-r--r-- "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" _sudo "to load the LaunchDaemon plist for nix-daemon" \ launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 7c66538b0..656769d84 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -246,8 +246,15 @@ printf -v _OLD_LINE_FMT "%b" $'\033[1;7;31m-'"$ESC ${RED}%L${ESC}" printf -v _NEW_LINE_FMT "%b" $'\033[1;7;32m+'"$ESC ${GREEN}%L${ESC}" _diff() { + # macOS Ventura doesn't ship with GNU diff. Print similar output except + # without +/- markers or dimming + if diff --version | grep -q "Apple diff"; then + printf -v CHANGED_GROUP_FORMAT "%b" "${GREEN}%>${RED}%<${ESC}" + diff --changed-group-format="$CHANGED_GROUP_FORMAT" "$@" + else # simple colorized diff comatible w/ pre `--color` versions - diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" + diff --unchanged-group-format="$_UNCHANGED_GRP_FMT" --old-line-format="$_OLD_LINE_FMT" --new-line-format="$_NEW_LINE_FMT" --unchanged-line-format=" %L" "$@" + fi } confirm_rm() { @@ -693,6 +700,10 @@ EOF } welcome_to_nix() { + local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" + local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}") + local -r GROUP_TEXT=$(echo -ne "${BLUE}(gid ${NIX_BUILD_GROUP_ID})${ESC}") + ok "Welcome to the Multi-User Nix Installation" cat <<EOF @@ -706,8 +717,8 @@ manager. This will happen in a few stages: 2. Show you what I am going to install and where. Then I will ask if you are ready to continue. -3. Create the system users and groups that the Nix daemon uses to run - builds. +3. Create the system users ${RANGE_TEXT} and groups ${GROUP_TEXT} + that the Nix daemon uses to run builds. 4. Perform the basic installation of the Nix files daemon. @@ -873,7 +884,7 @@ configure_shell_profile() { fi done - task "Setting up shell profiles for Fish with with ${PROFILE_FISH_SUFFIX} inside ${PROFILE_FISH_PREFIXES[*]}" + task "Setting up shell profiles for Fish with ${PROFILE_FISH_SUFFIX} inside ${PROFILE_FISH_PREFIXES[*]}" for fish_prefix in "${PROFILE_FISH_PREFIXES[@]}"; do if [ ! -d "$fish_prefix" ]; then # this specific prefix (ie: /etc/fish) is very likely to exist diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 323e04fdb..2fb17d06f 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -299,7 +299,7 @@ connected: !trusted || *trusted; }); - // See the very large comment in `case wopBuildDerivation:` in + // See the very large comment in `case WorkerProto::Op::BuildDerivation:` in // `src/libstore/daemon.cc` that explains the trust model here. // // This condition mirrors that: that code enforces the "rules" outlined there; diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 6c4648b34..4fc197956 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -239,9 +239,7 @@ void MixProfile::updateProfile(const StorePath & storePath) if (!store) throw Error("'--profile' is not supported for this Nix store"); auto profile2 = absPath(*profile); switchLink(profile2, - createGeneration( - ref<LocalFSStore>(store), - profile2, storePath)); + createGeneration(*store, profile2, storePath)); } void MixProfile::updateProfile(const BuiltPaths & buildables) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index ff3abd534..3df2c71a5 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -105,7 +105,9 @@ MixEvalArgs::MixEvalArgs() )", .category = category, .labels = {"path"}, - .handler = {[&](std::string s) { searchPath.push_back(s); }} + .handler = {[&](std::string s) { + searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + }} }); addFlag({ @@ -165,7 +167,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath; + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index b65cb5b20..6359b2579 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -3,6 +3,7 @@ #include "args.hh" #include "common-args.hh" +#include "search-path.hh" namespace nix { @@ -19,7 +20,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - Strings searchPath; + SearchPath searchPath; std::optional<std::string> evalStoreUrl; diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index eb944240b..4da9b131b 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -151,7 +151,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() }, ExtraPathInfoFlake::Flake { .originalRef = flakeRef, - .resolvedRef = getLockedFlake()->flake.lockedRef, + .lockedRef = getLockedFlake()->flake.lockedRef, }), }}; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 7ac4358d2..314918c14 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -19,7 +19,7 @@ struct ExtraPathInfoFlake : ExtraPathInfoValue */ struct Flake { FlakeRef originalRef; - FlakeRef resolvedRef; + FlakeRef lockedRef; }; Flake flake; diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index a2b882355..10b077fb5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -701,7 +701,7 @@ RawInstallablesCommand::RawInstallablesCommand() { addFlag({ .longName = "stdin", - .description = "Read installables from the standard input.", + .description = "Read installables from the standard input. No default installable applied.", .handler = {&readFromStdIn, true} }); @@ -730,9 +730,9 @@ void RawInstallablesCommand::run(ref<Store> store) while (std::cin >> word) { rawInstallables.emplace_back(std::move(word)); } + } else { + applyDefaultInstallables(rawInstallables); } - - applyDefaultInstallables(rawInstallables); run(store, std::move(rawInstallables)); } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 4b160a100..f9e9c2bf8 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -68,7 +68,7 @@ struct NixRepl const Path historyFile; - NixRepl(const Strings & searchPath, nix::ref<Store> store,ref<EvalState> state, + NixRepl(const SearchPath & searchPath, nix::ref<Store> store,ref<EvalState> state, std::function<AnnotatedValues()> getValues); virtual ~NixRepl(); @@ -104,7 +104,7 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, +NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state, std::function<NixRepl::AnnotatedValues()> getValues) : AbstractNixRepl(state) , debugTraceIndex(0) @@ -1024,7 +1024,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create( - const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, + const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state, std::function<AnnotatedValues()> getValues) { return std::make_unique<NixRepl>( @@ -1044,7 +1044,7 @@ void AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - const Strings & searchPath = {}; + SearchPath searchPath = {}; auto repl = std::make_unique<NixRepl>( searchPath, openStore(), diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 731c8e6db..6d88883fe 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -25,7 +25,7 @@ struct AbstractNixRepl typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues; static std::unique_ptr<AbstractNixRepl> create( - const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state, + const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state, std::function<AnnotatedValues()> getValues); static void runSimple( diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 740a5e677..be1bdb806 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -4,6 +4,7 @@ #include "util.hh" #include "store-api.hh" #include "derivations.hh" +#include "downstream-placeholder.hh" #include "globals.hh" #include "eval-inline.hh" #include "filetransfer.hh" @@ -94,11 +95,16 @@ RootValue allocRootValue(Value * v) #endif } -void Value::print(const SymbolTable & symbols, std::ostream & str, - std::set<const void *> * seen) const +void Value::print(const SymbolTable &symbols, std::ostream &str, + std::set<const void *> *seen, int depth) const + { checkInterrupt(); + if (depth <= 0) { + str << "«too deep»"; + return; + } switch (internalType) { case tInt: str << integer; @@ -122,7 +128,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << "{ "; for (auto & i : attrs->lexicographicOrder(symbols)) { str << symbols[i->name] << " = "; - i->value->print(symbols, str, seen); + i->value->print(symbols, str, seen, depth - 1); str << "; "; } str << "}"; @@ -138,7 +144,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, str << "[ "; for (auto v2 : listItems()) { if (v2) - v2->print(symbols, str, seen); + v2->print(symbols, str, seen, depth - 1); else str << "(nullptr)"; str << " "; @@ -180,11 +186,10 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, } } - -void Value::print(const SymbolTable & symbols, std::ostream & str, bool showRepeated) const -{ +void Value::print(const SymbolTable &symbols, std::ostream &str, + bool showRepeated, int depth) const { std::set<const void *> seen; - print(symbols, str, showRepeated ? nullptr : &seen); + print(symbols, str, showRepeated ? nullptr : &seen, depth); } // Pretty print types for assertion errors @@ -210,20 +215,21 @@ const Value * getPrimOp(const Value &v) { return primOp; } -std::string_view showType(ValueType type) +std::string_view showType(ValueType type, bool withArticle) { + #define WA(a, w) withArticle ? a " " w : w switch (type) { - case nInt: return "an integer"; - case nBool: return "a Boolean"; - case nString: return "a string"; - case nPath: return "a path"; + case nInt: return WA("an", "integer"); + case nBool: return WA("a", "Boolean"); + case nString: return WA("a", "string"); + case nPath: return WA("a", "path"); case nNull: return "null"; - case nAttrs: return "a set"; - case nList: return "a list"; - case nFunction: return "a function"; - case nExternal: return "an external value"; - case nFloat: return "a float"; - case nThunk: return "a thunk"; + case nAttrs: return WA("a", "set"); + case nList: return WA("a", "list"); + case nFunction: return WA("a", "function"); + case nExternal: return WA("an", "external value"); + case nFloat: return WA("a", "float"); + case nThunk: return WA("a", "thunk"); } abort(); } @@ -492,7 +498,7 @@ ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr) EvalState::EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref<Store> store, std::shared_ptr<Store> buildStore) : sWith(symbols.create("<with>")) @@ -557,30 +563,32 @@ EvalState::EvalState( /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { - for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); + for (auto & i : _searchPath.elements) + addToSearchPath(SearchPath::Elem {i}); + for (auto & i : evalSettings.nixPath.get()) + addToSearchPath(SearchPath::Elem::parse(i)); } if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); - for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i); - if (!r.first) continue; + for (auto & i : searchPath.elements) { + auto r = resolveSearchPathPath(i.path); + if (!r) continue; - auto path = r.second; + auto path = *std::move(r); - if (store->isInStore(r.second)) { + if (store->isInStore(path)) { try { StorePathSet closure; - store->computeFSClosure(store->toStorePath(r.second).first, closure); + store->computeFSClosure(store->toStorePath(path).first, closure); for (auto & path : closure) allowPath(path); } catch (InvalidPath &) { - allowPath(r.second); + allowPath(path); } } else - allowPath(r.second); + allowPath(path); } } @@ -701,28 +709,34 @@ Path EvalState::toRealPath(const Path & path, const NixStringContext & context) } -Value * EvalState::addConstant(const std::string & name, Value & v) +Value * EvalState::addConstant(const std::string & name, Value & v, Constant info) { Value * v2 = allocValue(); *v2 = v; - addConstant(name, v2); + addConstant(name, v2, info); return v2; } -void EvalState::addConstant(const std::string & name, Value * v) +void EvalState::addConstant(const std::string & name, Value * v, Constant info) { - staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); - baseEnv.values[baseEnvDispl++] = v; auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); -} + constantInfos.push_back({name2, info}); -Value * EvalState::addPrimOp(const std::string & name, - size_t arity, PrimOpFun primOp) -{ - return addPrimOp(PrimOp { .fun = primOp, .arity = arity, .name = name }); + if (!(evalSettings.pureEval && info.impureOnly)) { + /* Check the type, if possible. + + We might know the type of a thunk in advance, so be allowed + to just write it down in that case. */ + if (auto gotType = v->type(true); gotType != nThunk) + assert(info.type == gotType); + + /* Install value the base environment. */ + staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); + } } @@ -736,7 +750,10 @@ Value * EvalState::addPrimOp(PrimOp && primOp) vPrimOp->mkPrimOp(new PrimOp(primOp)); Value v; v.mkApp(vPrimOp, vPrimOp); - return addConstant(primOp.name, v); + return addConstant(primOp.name, v, { + .type = nThunk, // FIXME + .doc = primOp.doc, + }); } auto envName = symbols.create(primOp.name); @@ -762,13 +779,13 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v) { if (v.isPrimOp()) { auto v2 = &v; - if (v2->primOp->doc) + if (auto * doc = v2->primOp->doc) return Doc { .pos = {}, .name = v2->primOp->name, .arity = v2->primOp->arity, .args = v2->primOp->args, - .doc = v2->primOp->doc, + .doc = doc, }; } return {}; @@ -1058,7 +1075,7 @@ void EvalState::mkOutputString( ? store->printStorePath(*std::move(optOutputPath)) /* Downstream we would substitute this for an actual path once we build the floating CA derivation */ - : downstreamPlaceholder(*store, drvPath, outputName), + : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(), NixStringContext { NixStringContextElem::Built { .drvPath = drvPath, @@ -2380,7 +2397,7 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str // This is testing for the case of CA derivations auto sExpected = optOutputPath ? store->printStorePath(*optOutputPath) - : downstreamPlaceholder(*store, b.drvPath, output); + : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render(); if (s != sExpected) error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", @@ -2619,7 +2636,7 @@ Strings EvalSettings::getDefaultNixPath() { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathExists(p)) { + if (pathAccessible(p)) { if (s.empty()) { res.push_back(p); } else { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index a90ff34c0..277e77ad5 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -9,6 +9,7 @@ #include "config.hh" #include "experimental-features.hh" #include "input-accessor.hh" +#include "search-path.hh" #include <map> #include <optional> @@ -25,15 +26,72 @@ struct DerivedPath; enum RepairFlag : bool; +/** + * Function that implements a primop. + */ typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v); +/** + * Info about a primitive operation, and its implementation + */ struct PrimOp { - PrimOpFun fun; - size_t arity; + /** + * Name of the primop. `__` prefix is treated specially. + */ std::string name; + + /** + * Names of the parameters of a primop, for primops that take a + * fixed number of arguments to be substituted for these parameters. + */ std::vector<std::string> args; + + /** + * Aritiy of the primop. + * + * If `args` is not empty, this field will be computed from that + * field instead, so it doesn't need to be manually set. + */ + size_t arity = 0; + + /** + * Optional free-form documentation about the primop. + */ const char * doc = nullptr; + + /** + * Implementation of the primop. + */ + PrimOpFun fun; + + /** + * Optional experimental for this to be gated on. + */ + std::optional<ExperimentalFeature> experimentalFeature; +}; + +/** + * Info about a constant + */ +struct Constant +{ + /** + * Optional type of the constant (known since it is a fixed value). + * + * @todo we should use an enum for this. + */ + ValueType type = nThunk; + + /** + * Optional free-form documentation about the constant. + */ + const char * doc = nullptr; + + /** + * Whether the constant is impure, and not available in pure mode. + */ + bool impureOnly = false; }; #if HAVE_BOEHMGC @@ -65,11 +123,6 @@ std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -// FIXME: maybe change this to an std::variant<SourcePath, URL>. -typedef std::pair<std::string, std::string> SearchPathElem; -typedef std::list<SearchPathElem> SearchPath; - - /** * Initialise the Boehm GC, if applicable. */ @@ -256,7 +309,7 @@ private: SearchPath searchPath; - std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + std::map<std::string, std::optional<std::string>> searchPathResolved; /** * Cache used by checkSourcePath(). @@ -283,12 +336,12 @@ private: public: EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref<Store> store, std::shared_ptr<Store> buildStore = nullptr); ~EvalState(); - void addToSearchPath(const std::string & s); + void addToSearchPath(SearchPath::Elem && elem); SearchPath getSearchPath() { return searchPath; } @@ -370,12 +423,16 @@ public: * Look up a file in the search path. */ SourcePath findFile(const std::string_view path); - SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); + SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** + * Try to resolve a search path value (not the optinal key part) + * * If the specified search path element is a URI, download it. + * + * If it is not found, return `std::nullopt` */ - std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); + std::optional<std::string> resolveSearchPathPath(const SearchPath::Path & path); /** * Evaluate an expression to normal form @@ -483,7 +540,7 @@ public: * Coerce to `DerivedPath`. * * Must be a string which is either a literal store path or a - * "placeholder (see `downstreamPlaceholder()`). + * "placeholder (see `DownstreamPlaceholder`). * * Even more importantly, the string context must be exactly one * element, which is either a `NixStringContextElem::Opaque` or @@ -509,18 +566,23 @@ public: */ std::shared_ptr<StaticEnv> staticBaseEnv; // !!! should be private + /** + * Name and documentation about every constant. + * + * Constants from primops are hard to crawl, and their docs will go + * here too. + */ + std::vector<std::pair<std::string, Constant>> constantInfos; + private: unsigned int baseEnvDispl = 0; void createBaseEnv(); - Value * addConstant(const std::string & name, Value & v); + Value * addConstant(const std::string & name, Value & v, Constant info); - void addConstant(const std::string & name, Value * v); - - Value * addPrimOp(const std::string & name, - size_t arity, PrimOpFun primOp); + void addConstant(const std::string & name, Value * v, Constant info); Value * addPrimOp(PrimOp && primOp); @@ -534,6 +596,10 @@ public: std::optional<std::string> name; size_t arity; std::vector<std::string> args; + /** + * Unlike the other `doc` fields in this file, this one should never be + * `null`. + */ const char * doc; }; @@ -622,7 +688,7 @@ public: * @param optOutputPath Optional output path for that string. Must * be passed if and only if output store object is input-addressed. * Will be printed to form string if passed, otherwise a placeholder - * will be used (see `downstreamPlaceholder()`). + * will be used (see `DownstreamPlaceholder`). */ void mkOutputString( Value & value, @@ -700,8 +766,11 @@ struct DebugTraceStacker { /** * @return A string representing the type of the value `v`. + * + * @param withArticle Whether to begin with an english article, e.g. "an + * integer" vs "integer". */ -std::string_view showType(ValueType type); +std::string_view showType(ValueType type, bool withArticle = true); std::string showType(const Value & v); /** @@ -733,7 +802,12 @@ struct EvalSettings : Config Setting<Strings> nixPath{ this, getDefaultNixPath(), "nix-path", - "List of directories to be searched for `<...>` file references."}; + R"( + List of directories to be searched for `<...>` file references + + In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtin-constants-nixPath). + )"}; Setting<bool> restrictEval{ this, false, "restrict-eval", @@ -741,11 +815,18 @@ struct EvalSettings : Config If set to `true`, the Nix evaluator will not allow access to any files outside of the Nix search path (as set via the `NIX_PATH` environment variable or the `-I` option), or to URIs outside of - `allowed-uri`. The default is `false`. + [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). + The default is `false`. )"}; Setting<bool> pureEval{this, false, "pure-eval", - "Whether to restrict file system and network access to files specified by cryptographic hash."}; + R"( + Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: + + - Restrict file system and network access to files specified by cryptographic hash + - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + )" + }; Setting<bool> enableImportFromDerivation{ this, true, "allow-import-from-derivation", diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 60bb6a71e..5aa44d6a1 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -788,9 +788,6 @@ static RegisterPrimOp r2({ ```nix (builtins.getFlake "github:edolstra/dwarffs").rev ``` - - This function is only available if you enable the experimental feature - `flakes`. )", .fun = prim_getFlake, .experimentalFeature = Xp::Flakes, diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 462b3b602..a3a8608d9 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -36,7 +36,7 @@ static inline PosIdx makeCurPos(const YYLTYPE & loc, ParseData * data) #define CUR_POS makeCurPos(*yylloc, data) // backup to recover from yyless(0) -YYLTYPE prev_yylloc; +thread_local YYLTYPE prev_yylloc; static void initLoc(YYLTYPE * loc) { diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 4d981712a..0a1ad9967 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -275,7 +275,12 @@ static Expr * stripIndentation(const PosIdx pos, SymbolTable & symbols, } /* If this is a single string, then don't do a concatenation. */ - return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2); + if (es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second)) { + auto *const result = (*es2)[0].second; + delete es2; + return result; + } + return new ExprConcatStrings(pos, true, es2); } @@ -330,7 +335,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %type <ind_string_parts> ind_string_parts %type <e> path_start string_parts string_attr %type <id> attr -%token <id> ID ATTRPATH +%token <id> ID %token <str> STR IND_STR %token <n> INT %token <nf> FLOAT @@ -658,7 +663,7 @@ Expr * EvalState::parse( ParseData data { .state = *this, .symbols = symbols, - .basePath = std::move(basePath), + .basePath = basePath, .origin = {origin}, }; @@ -729,19 +734,9 @@ Expr * EvalState::parseStdin() } -void EvalState::addToSearchPath(const std::string & s) +void EvalState::addToSearchPath(SearchPath::Elem && elem) { - size_t pos = s.find('='); - std::string prefix; - Path path; - if (pos == std::string::npos) { - path = s; - } else { - prefix = std::string(s, 0, pos); - path = std::string(s, pos + 1); - } - - searchPath.emplace_back(prefix, path); + searchPath.elements.emplace_back(std::move(elem)); } @@ -751,22 +746,19 @@ SourcePath EvalState::findFile(const std::string_view path) } -SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) +SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) { - for (auto & i : searchPath) { - std::string suffix; - if (i.first.empty()) - suffix = concatStrings("/", path); - else { - auto s = i.first.size(); - if (path.compare(0, s, i.first) != 0 || - (path.size() > s && path[s] != '/')) - continue; - suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); - } - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - Path res = r.second + suffix; + for (auto & i : searchPath.elements) { + auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); + + if (!suffixOpt) continue; + auto suffix = *suffixOpt; + + auto rOpt = resolveSearchPathPath(i.path); + if (!rOpt) continue; + auto r = *rOpt; + + Path res = suffix == "" ? r : concatStrings(r, "/", suffix); if (pathExists(res)) return CanonPath(canonPath(res)); } @@ -783,49 +775,53 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p } -std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathElem & elem) +std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0) { - auto i = searchPathResolved.find(elem.second); + auto & value = value0.s; + auto i = searchPathResolved.find(value); if (i != searchPathResolved.end()) return i->second; - std::pair<bool, std::string> res; + std::optional<std::string> res; - if (EvalSettings::isPseudoUrl(elem.second)) { + if (EvalSettings::isPseudoUrl(value)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).first.storePath; - res = { true, store->toRealPath(storePath) }; + store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - else if (hasPrefix(elem.second, "flake:")) { + else if (hasPrefix(value, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(elem.second.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", elem.second); + auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", value); auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; - res = { true, store->toRealPath(storePath) }; + res = { store->toRealPath(storePath) }; } else { - auto path = absPath(elem.second); + auto path = absPath(value); if (pathExists(path)) - res = { true, path }; + res = { path }; else { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - debug("resolved search path element '%s' to '%s'", elem.second, res.second); + if (res) + debug("resolved search path element '%s' to '%s'", value, *res); + else + debug("failed to resolve search path element '%s'", value); - searchPathResolved[elem.second] = res; + searchPathResolved[value] = res; return res; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 6fbd66389..8a61e57cc 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,11 +1,12 @@ #include "archive.hh" #include "derivations.hh" +#include "downstream-placeholder.hh" #include "eval-inline.hh" #include "eval.hh" #include "globals.hh" #include "json-to-value.hh" #include "names.hh" -#include "references.hh" +#include "path-references.hh" #include "store-api.hh" #include "util.hh" #include "value-to-json.hh" @@ -87,7 +88,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) auto outputs = resolveDerivedPath(*store, drv); for (auto & [outputName, outputPath] : outputs) { res.insert_or_assign( - downstreamPlaceholder(*store, drv.drvPath, outputName), + DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), store->printStorePath(outputPath) ); } @@ -237,7 +238,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v } } -static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { +static RegisterPrimOp primop_scopedImport(PrimOp { .name = "scopedImport", .arity = 2, .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -691,7 +692,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value * * a v.listElems()[n++] = i; } -static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info { +static RegisterPrimOp primop_genericClosure(PrimOp { .name = "__genericClosure", .args = {"attrset"}, .arity = 1, @@ -808,7 +809,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * } } -static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { +static RegisterPrimOp primop_addErrorContext(PrimOp { .name = "__addErrorContext", .arity = 2, .fun = prim_addErrorContext, @@ -1151,16 +1152,14 @@ drvName, Bindings * attrs, Value & v) if (i->value->type() == nNull) continue; } - if (i->name == state.sContentAddressed) { - contentAddressed = state.forceBool(*i->value, noPos, context_below); - if (contentAddressed) - experimentalFeatureSettings.require(Xp::CaDerivations); + if (i->name == state.sContentAddressed && state.forceBool(*i->value, noPos, context_below)) { + contentAddressed = true; + experimentalFeatureSettings.require(Xp::CaDerivations); } - else if (i->name == state.sImpure) { - isImpure = state.forceBool(*i->value, noPos, context_below); - if (isImpure) - experimentalFeatureSettings.require(Xp::ImpureDerivations); + else if (i->name == state.sImpure && state.forceBool(*i->value, noPos, context_below)) { + isImpure = true; + experimentalFeatureSettings.require(Xp::ImpureDerivations); } /* The `args' attribute is special: it supplies the @@ -1401,7 +1400,7 @@ drvName, Bindings * attrs, Value & v) v.mkAttrs(result); } -static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { +static RegisterPrimOp primop_derivationStrict(PrimOp { .name = "derivationStrict", .arity = 1, .fun = prim_derivationStrict, @@ -1502,7 +1501,9 @@ static RegisterPrimOp primop_storePath({ causes the path to be *copied* again to the Nix store, resulting in a new path (e.g. `/nix/store/ld01dnzc…-source-source`). - This function is not available in pure evaluation mode. + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + + See also [`builtins.fetchClosure`](#builtins-fetchClosure). )", .fun = prim_storePath, }); @@ -1657,7 +1658,10 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V })); } - searchPath.emplace_back(prefix, path); + searchPath.elements.emplace_back(SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = prefix }, + .path = SearchPath::Path { .s = path }, + }); } auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); @@ -1665,9 +1669,52 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); } -static RegisterPrimOp primop_findFile(RegisterPrimOp::Info { +static RegisterPrimOp primop_findFile(PrimOp { .name = "__findFile", - .arity = 2, + .args = {"search path", "lookup path"}, + .doc = R"( + Look up the given path with the given search path. + + A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes, `prefix`, and `path`. + `prefix` is a relative path. + `path` denotes a file system location; the exact syntax depends on the command line interface. + + Examples of search path attribute sets: + + - ``` + { + prefix = "nixos-config"; + path = "/etc/nixos/configuration.nix"; + } + ``` + + - ``` + { + prefix = ""; + path = "/nix/var/nix/profiles/per-user/root/channels"; + } + ``` + + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match. + + This is the process for each entry: + If the lookup path matches `prefix`, then the remainder of the lookup path (the "suffix") is searched for within the directory denoted by `patch`. + Note that the `path` may need to be downloaded at this point to look inside. + If the suffix is found inside that directory, then the entry is a match; + the combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + + The syntax + + ```nix + <nixpkgs> + ``` + + is equivalent to: + + ```nix + builtins.findFile builtins.nixPath "nixpkgs" + ``` + )", .fun = prim_findFile, }); @@ -2386,7 +2433,7 @@ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value * * state.mkPos(v, i->pos); } -static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info { +static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp { .name = "__unsafeGetAttrPos", .arity = 2, .fun = prim_unsafeGetAttrPos, @@ -3909,13 +3956,8 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a for (auto elem : args[0]->listItems()) from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); - std::vector<std::pair<std::string, NixStringContext>> to; - to.reserve(args[1]->listSize()); - for (auto elem : args[1]->listItems()) { - NixStringContext ctx; - auto s = state.forceString(*elem, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); - to.emplace_back(s, std::move(ctx)); - } + std::unordered_map<size_t, std::string> cache; + auto to = args[1]->listItems(); NixStringContext context; auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings"); @@ -3926,10 +3968,19 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a bool found = false; auto i = from.begin(); auto j = to.begin(); - for (; i != from.end(); ++i, ++j) + size_t j_index = 0; + for (; i != from.end(); ++i, ++j, ++j_index) if (s.compare(p, i->size(), *i) == 0) { found = true; - res += j->first; + auto v = cache.find(j_index); + if (v == cache.end()) { + NixStringContext ctx; + auto ts = state.forceString(**j, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); + v = (cache.emplace(j_index, ts)).first; + for (auto& path : ctx) + context.insert(path); + } + res += v->second; if (i->empty()) { if (p < s.size()) res += s[p]; @@ -3937,9 +3988,6 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a } else { p += i->size(); } - for (auto& path : j->second) - context.insert(path); - j->second.clear(); break; } if (!found) { @@ -3957,7 +4005,11 @@ static RegisterPrimOp primop_replaceStrings({ .args = {"from", "to", "s"}, .doc = R"( Given string *s*, replace every occurrence of the strings in *from* - with the corresponding string in *to*. For example, + with the corresponding string in *to*. + + The argument *to* is lazy, that is, it is only evaluated when its corresponding pattern in *from* is matched in the string *s* + + Example: ```nix builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" @@ -4054,22 +4106,10 @@ static RegisterPrimOp primop_splitVersion({ RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; -RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) -{ - if (!primOps) primOps = new PrimOps; - primOps->push_back({ - .name = name, - .args = {}, - .arity = arity, - .fun = fun, - }); -} - - -RegisterPrimOp::RegisterPrimOp(Info && info) +RegisterPrimOp::RegisterPrimOp(PrimOp && primOp) { if (!primOps) primOps = new PrimOps; - primOps->push_back(std::move(info)); + primOps->push_back(std::move(primOp)); } @@ -4082,85 +4122,253 @@ void EvalState::createBaseEnv() /* `builtins' must be first! */ v.mkAttrs(buildBindings(128).finish()); - addConstant("builtins", v); + addConstant("builtins", v, { + .type = nAttrs, + .doc = R"( + Contains all the [built-in functions](@docroot@/language/builtins.md) and values. + + Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: + + ```nix + # if hasContext is not available, we assume `s` has a context + if builtins ? hasContext then builtins.hasContext s else true + ``` + )", + }); v.mkBool(true); - addConstant("true", v); + addConstant("true", v, { + .type = nBool, + .doc = R"( + Primitive value. + + It can be returned by + [comparison operators](@docroot@/language/operators.md#Comparison) + and used in + [conditional expressions](@docroot@/language/constructs.md#Conditionals). + + The name `true` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let true = 1; in true + 1 + ``` + )", + }); v.mkBool(false); - addConstant("false", v); + addConstant("false", v, { + .type = nBool, + .doc = R"( + Primitive value. + + It can be returned by + [comparison operators](@docroot@/language/operators.md#Comparison) + and used in + [conditional expressions](@docroot@/language/constructs.md#Conditionals). + + The name `false` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let false = 1; in false + 1 + ``` + )", + }); v.mkNull(); - addConstant("null", v); + addConstant("null", v, { + .type = nNull, + .doc = R"( + Primitive value. + + The name `null` is not special, and can be shadowed: + + ```nix-repl + nix-repl> let null = 1; in null + 1 + ``` + )", + }); if (!evalSettings.pureEval) { v.mkInt(time(0)); - addConstant("__currentTime", v); + } + addConstant("__currentTime", v, { + .type = nInt, + .doc = R"( + Return the [Unix time](https://en.wikipedia.org/wiki/Unix_time) at first evaluation. + Repeated references to that name will re-use the initially obtained value. + + Example: + + ```console + $ nix repl + Welcome to Nix 2.15.1 Type :? for help. + nix-repl> builtins.currentTime + 1683705525 + + nix-repl> builtins.currentTime + 1683705525 + ``` + + The [store path](@docroot@/glossary.md#gloss-store-path) of a derivation depending on `currentTime` will differ for each evaluation, unless both evaluate `builtins.currentTime` in the same second. + )", + .impureOnly = true, + }); + + if (!evalSettings.pureEval) { v.mkString(settings.thisSystem.get()); - addConstant("__currentSystem", v); } + addConstant("__currentSystem", v, { + .type = nString, + .doc = R"( + The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval). + + It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression: + + ```nix + builtins.derivation { + # ... + system = builtins.currentSystem; + } + ``` + + It can be overridden in order to create derivations for different system than the current one: + + ```console + $ nix-instantiate --system "mips64-linux" --eval --expr 'builtins.currentSystem' + "mips64-linux" + ``` + )", + .impureOnly = true, + }); v.mkString(nixVersion); - addConstant("__nixVersion", v); + addConstant("__nixVersion", v, { + .type = nString, + .doc = R"( + The version of Nix. + + For example, where the command line returns the current Nix version, + + ```shell-session + $ nix --version + nix (Nix) 2.16.0 + ``` + + the Nix language evaluator returns the same value: + + ```nix-repl + nix-repl> builtins.nixVersion + "2.16.0" + ``` + )", + }); v.mkString(store->storeDir); - addConstant("__storeDir", v); + addConstant("__storeDir", v, { + .type = nString, + .doc = R"( + Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use. + + This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md): + + ```shell-session + $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir + "/blah" + ``` + )", + }); /* Language version. This should be increased every time a new language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ v.mkInt(6); - addConstant("__langVersion", v); + addConstant("__langVersion", v, { + .type = nInt, + .doc = R"( + The current version of the Nix language. + )", + }); // Miscellaneous if (evalSettings.enableNativeCode) { - addPrimOp("__importNative", 2, prim_importNative); - addPrimOp("__exec", 1, prim_exec); + addPrimOp({ + .name = "__importNative", + .arity = 2, + .fun = prim_importNative, + }); + addPrimOp({ + .name = "__exec", + .arity = 1, + .fun = prim_exec, + }); } addPrimOp({ - .fun = evalSettings.traceVerbose ? prim_trace : prim_second, - .arity = 2, .name = "__traceVerbose", .args = { "e1", "e2" }, + .arity = 2, .doc = R"( Evaluate *e1* and print its abstract syntax representation on standard error if `--trace-verbose` is enabled. Then return *e2*. This function is useful for debugging. )", + .fun = evalSettings.traceVerbose ? prim_trace : prim_second, }); /* Add a value containing the current Nix expression search path. */ - mkList(v, searchPath.size()); + mkList(v, searchPath.elements.size()); int n = 0; - for (auto & i : searchPath) { + for (auto & i : searchPath.elements) { auto attrs = buildBindings(2); - attrs.alloc("path").mkString(i.second); - attrs.alloc("prefix").mkString(i.first); + attrs.alloc("path").mkString(i.path.s); + attrs.alloc("prefix").mkString(i.prefix.s); (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } - addConstant("__nixPath", v); + addConstant("__nixPath", v, { + .type = nList, + .doc = R"( + The search path used to resolve angle bracket path lookups. + + Angle bracket expressions can be + [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) + using this and + [`builtins.findFile`](./builtins.html#builtins-findFile): + + ```nix + <nixpkgs> + ``` + + is equivalent to: + + ```nix + builtins.findFile builtins.nixPath "nixpkgs" + ``` + )", + }); if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) - if (!primOp.experimentalFeature - || experimentalFeatureSettings.isEnabled(*primOp.experimentalFeature)) + if (experimentalFeatureSettings.isEnabled(primOp.experimentalFeature)) { - addPrimOp({ - .fun = primOp.fun, - .arity = std::max(primOp.args.size(), primOp.arity), - .name = primOp.name, - .args = primOp.args, - .doc = primOp.doc, - }); + auto primOpAdjusted = primOp; + primOpAdjusted.arity = std::max(primOp.args.size(), primOp.arity); + addPrimOp(std::move(primOpAdjusted)); } /* Add a wrapper around the derivation primop that computes the - `drvPath' and `outPath' attributes lazily. */ + `drvPath' and `outPath' attributes lazily. + + Null docs because it is documented separately. + */ auto vDerivation = allocValue(); - addConstant("derivation", vDerivation); + addConstant("derivation", vDerivation, { + .type = nFunction, + }); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 4ae73fe1f..930e7f32a 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -10,17 +10,7 @@ namespace nix { struct RegisterPrimOp { - struct Info - { - std::string name; - std::vector<std::string> args; - size_t arity = 0; - const char * doc; - PrimOpFun fun; - std::optional<ExperimentalFeature> experimentalFeature; - }; - - typedef std::vector<Info> PrimOps; + typedef std::vector<PrimOp> PrimOps; static PrimOps * primOps; /** @@ -28,12 +18,7 @@ struct RegisterPrimOp * will get called during EvalState initialization, so there * may be primops not yet added and builtins is not yet sorted. */ - RegisterPrimOp( - std::string name, - size_t arity, - PrimOpFun fun); - - RegisterPrimOp(Info && info); + RegisterPrimOp(PrimOp && primOp); }; /* These primops are disabled without enableNativeCode, but plugins diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 07bf400cf..8b3468009 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -12,7 +12,11 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, v.mkString(*s); } -static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); +static RegisterPrimOp primop_unsafeDiscardStringContext({ + .name = "__unsafeDiscardStringContext", + .arity = 1, + .fun = prim_unsafeDiscardStringContext +}); static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -22,7 +26,16 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, v.mkBool(!context.empty()); } -static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); +static RegisterPrimOp primop_hasContext({ + .name = "__hasContext", + .args = {"s"}, + .doc = R"( + Return `true` if string *s* has a non-empty context. The + context can be obtained with + [`getContext`](#builtins-getContext). + )", + .fun = prim_hasContext +}); /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a @@ -51,7 +64,11 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p v.mkString(*s, context2); } -static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); +static RegisterPrimOp primop_unsafeDiscardOutputDependency({ + .name = "__unsafeDiscardOutputDependency", + .arity = 1, + .fun = prim_unsafeDiscardOutputDependency +}); /* Extract the context of a string as a structured Nix value. @@ -119,7 +136,30 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, v.mkAttrs(attrs); } -static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); +static RegisterPrimOp primop_getContext({ + .name = "__getContext", + .args = {"s"}, + .doc = R"( + Return the string context of *s*. + + The string context tracks references to derivations within a string. + It is represented as an attribute set of [store derivation](@docroot@/glossary.md#gloss-store-derivation) paths mapping to output names. + + Using [string interpolation](@docroot@/language/string-interpolation.md) on a derivation will add that derivation to the string context. + For example, + + ```nix + builtins.getContext "${derivation { name = "a"; builder = "b"; system = "c"; }}" + ``` + + evaluates to + + ``` + { "/nix/store/arhvjaf6zmlyn8vh8fgn55rpwnxq0n7l-a.drv" = { outputs = [ "out" ]; }; } + ``` + )", + .fun = prim_getContext +}); /* Append the given context to a given string. @@ -192,6 +232,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar v.mkString(orig, context); } -static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); +static RegisterPrimOp primop_appendContext({ + .name = "__appendContext", + .arity = 2, + .fun = prim_appendContext +}); } diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 4cf1f1e0b..7fe8203f4 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -5,37 +5,150 @@ namespace nix { +/** + * Handler for the content addressed case. + * + * @param state Evaluator state and store to write to. + * @param fromStore Store containing the path to rewrite. + * @param fromPath Source path to be rewritten. + * @param toPathMaybe Path to write the rewritten path to. If empty, the error shows the actual path. + * @param v Return `Value` + */ +static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, const std::optional<StorePath> & toPathMaybe, Value &v) { + + // establish toPath or throw + + if (!toPathMaybe || !state.store->isValidPath(*toPathMaybe)) { + auto rewrittenPath = makeContentAddressed(fromStore, *state.store, fromPath); + if (toPathMaybe && *toPathMaybe != rewrittenPath) + throw Error({ + .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(fromPath), + state.store->printStorePath(rewrittenPath), + state.store->printStorePath(*toPathMaybe)), + .errPos = state.positions[pos] + }); + if (!toPathMaybe) + throw Error({ + .msg = hintfmt( + "rewriting '%s' to content-addressed form yielded '%s'\n" + "Use this value for the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(fromPath), + state.store->printStorePath(rewrittenPath)), + .errPos = state.positions[pos] + }); + } + + auto toPath = *toPathMaybe; + + // check and return + + auto resultInfo = state.store->queryPathInfo(toPath); + + if (!resultInfo->isContentAddressed(*state.store)) { + // We don't perform the rewriting when outPath already exists, as an optimisation. + // However, we can quickly detect a mistake if the toPath is input addressed. + throw Error({ + .msg = hintfmt( + "The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n" + "Set 'toPath' to an empty string to make Nix report the correct content-addressed path.", + state.store->printStorePath(toPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(toPath, v); +} + +/** + * Fetch the closure and make sure it's content addressed. + */ +static void runFetchClosureWithContentAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + + if (!state.store->isValidPath(fromPath)) + copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath }); + + auto info = state.store->queryPathInfo(fromPath); + + if (!info->isContentAddressed(*state.store)) { + throw Error({ + .msg = hintfmt( + "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n" + "If you do intend to fetch an input-addressed store path, add\n\n" + " inputAddressed = true;\n\n" + "to the 'fetchClosure' arguments.\n\n" + "Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.", + state.store->printStorePath(fromPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(fromPath, v); +} + +/** + * Fetch the closure and make sure it's input addressed. + */ +static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + + if (!state.store->isValidPath(fromPath)) + copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath }); + + auto info = state.store->queryPathInfo(fromPath); + + if (info->isContentAddressed(*state.store)) { + throw Error({ + .msg = hintfmt( + "The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n" + "Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed", + state.store->printStorePath(fromPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(fromPath, v); +} + +typedef std::optional<StorePath> StorePathOrGap; + static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure"); std::optional<std::string> fromStoreUrl; std::optional<StorePath> fromPath; - bool toCA = false; - std::optional<StorePath> toPath; + std::optional<StorePathOrGap> toPath; + std::optional<bool> inputAddressedMaybe; for (auto & attr : *args[0]->attrs) { const auto & attrName = state.symbols[attr.name]; + auto attrHint = [&]() -> std::string { + return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure"; + }; if (attrName == "fromPath") { NixStringContext context; - fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, - "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure"); + fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } else if (attrName == "toPath") { state.forceValue(*attr.value, attr.pos); - toCA = true; - if (attr.value->type() != nString || attr.value->string.s != std::string("")) { + bool isEmptyString = attr.value->type() == nString && attr.value->string.s == std::string(""); + if (isEmptyString) { + toPath = StorePathOrGap {}; + } + else { NixStringContext context; - toPath = state.coerceToStorePath(attr.pos, *attr.value, context, - "while evaluating the 'toPath' attribute passed to builtins.fetchClosure"); + toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } } else if (attrName == "fromStore") fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos, - "while evaluating the 'fromStore' attribute passed to builtins.fetchClosure"); + attrHint()); + + else if (attrName == "inputAddressed") + inputAddressedMaybe = state.forceBool(*attr.value, attr.pos, attrHint()); else throw Error({ @@ -50,6 +163,18 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg .errPos = state.positions[pos] }); + bool inputAddressed = inputAddressedMaybe.value_or(false); + + if (inputAddressed) { + if (toPath) + throw Error({ + .msg = hintfmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them", + "inputAddressed", + "toPath"), + .errPos = state.positions[pos] + }); + } + if (!fromStoreUrl) throw Error({ .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), @@ -74,55 +199,40 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg auto fromStore = openStore(parsedURL.to_string()); - if (toCA) { - if (!toPath || !state.store->isValidPath(*toPath)) { - auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); - auto i = remappings.find(*fromPath); - assert(i != remappings.end()); - if (toPath && *toPath != i->second) - throw Error({ - .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", - state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second), - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - if (!toPath) - throw Error({ - .msg = hintfmt( - "rewriting '%s' to content-addressed form yielded '%s'; " - "please set this in the 'toPath' attribute passed to 'fetchClosure'", - state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second)), - .errPos = state.positions[pos] - }); - } - } else { - if (!state.store->isValidPath(*fromPath)) - copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); - toPath = fromPath; - } - - /* In pure mode, require a CA path. */ - if (evalSettings.pureEval) { - auto info = state.store->queryPathInfo(*toPath); - if (!info->isContentAddressed(*state.store)) - throw Error({ - .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - } - - state.mkStorePathString(*toPath, v); + if (toPath) + runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v); + else if (inputAddressed) + runFetchClosureWithInputAddressedPath(state, pos, *fromStore, *fromPath, v); + else + runFetchClosureWithContentAddressedPath(state, pos, *fromStore, *fromPath, v); } static RegisterPrimOp primop_fetchClosure({ .name = "__fetchClosure", .args = {"args"}, .doc = R"( - Fetch a Nix store closure from a binary cache, rewriting it into - content-addressed form. For example, + Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache, and return the store path as a string with context. + + This function can be invoked in three ways, that we will discuss in order of preference. + + **Fetch a content-addressed store path** + + Example: + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` + + This is the simplest invocation, and it does not require the user of the expression to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. + + If your store path is [input addressed](@docroot@/glossary.md#gloss-input-addressed-store-object) instead of content addressed, consider the other two invocations. + + **Fetch any store path and rewrite it to a fully content-addressed store path** + + Example: ```nix builtins.fetchClosure { @@ -132,31 +242,42 @@ static RegisterPrimOp primop_fetchClosure({ } ``` - fetches `/nix/store/r2jd...` from the specified binary cache, + This example fetches `/nix/store/r2jd...` from the specified binary cache, and rewrites it into the content-addressed store path `/nix/store/ldbh...`. - If `fromPath` is already content-addressed, or if you are - allowing impure evaluation (`--impure`), then `toPath` may be - omitted. + Like the previous example, no extra configuration or privileges are required. To find out the correct value for `toPath` given a `fromPath`, - you can use `nix store make-content-addressed`: + use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md): ```console # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' ``` - This function is similar to `builtins.storePath` in that it - allows you to use a previously built store path in a Nix - expression. However, it is more reproducible because it requires - specifying a binary cache from which the path can be fetched. - Also, requiring a content-addressed final store path avoids the - need for users to configure binary cache public keys. + Alternatively, set `toPath = ""` and find the correct `toPath` in the error message. + + **Fetch an input-addressed store path as is** + + Example: + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + inputAddressed = true; + } + ``` + + It is possible to fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) and return it as is. + However, this is the least preferred way of invoking `fetchClosure`, because it requires that the input-addressed paths are trusted by the Nix configuration. + + **`builtins.storePath`** - This function is only available if you enable the experimental - feature `fetch-closure`. + `fetchClosure` is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression. + However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched. + Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. )", .fun = prim_fetchClosure, .experimentalFeature = Xp::FetchClosure, diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 2c0d98e74..322692b52 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -88,6 +88,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a state.allowPath(tree.storePath); } -static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); +static RegisterPrimOp r_fetchMercurial({ + .name = "fetchMercurial", + .arity = 1, + .fun = prim_fetchMercurial +}); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index cd7039025..579a45f92 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -22,7 +22,7 @@ void emitTreeAttrs( { assert(input.isLocked()); - auto attrs = state.buildBindings(8); + auto attrs = state.buildBindings(10); state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); @@ -56,6 +56,11 @@ void emitTreeAttrs( } + if (auto dirtyRev = fetchers::maybeGetStrAttr(input.attrs, "dirtyRev")) { + attrs.alloc("dirtyRev").mkString(*dirtyRev); + attrs.alloc("dirtyShortRev").mkString(*fetchers::maybeGetStrAttr(input.attrs, "dirtyShortRev")); + } + if (auto lastModified = input.getLastModified()) { attrs.alloc("lastModified").mkInt(*lastModified); attrs.alloc("lastModifiedDate").mkString( @@ -194,7 +199,11 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, } // FIXME: document -static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); +static RegisterPrimOp primop_fetchTree({ + .name = "fetchTree", + .arity = 1, + .fun = prim_fetchTree +}); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, const std::string & who, bool unpack, std::string name) @@ -262,7 +271,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; if (expectedHash) { @@ -286,9 +295,9 @@ static RegisterPrimOp primop_fetchurl({ .name = "__fetchurl", .args = {"url"}, .doc = R"( - Download the specified URL and return the path of the downloaded - file. This function is not available if [restricted evaluation - mode](../command-ref/conf-file.md) is enabled. + Download the specified URL and return the path of the downloaded file. + + Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", .fun = prim_fetchurl, }); @@ -338,8 +347,7 @@ static RegisterPrimOp primop_fetchTarball({ stdenv.mkDerivation { … } ``` - This function is not available if [restricted evaluation - mode](../command-ref/conf-file.md) is enabled. + Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", .fun = prim_fetchTarball, }); @@ -470,14 +478,9 @@ static RegisterPrimOp primop_fetchGit({ } ``` - > **Note** - > - > Nix will refetch the branch in accordance with - > the option `tarball-ttl`. + Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting. - > **Note** - > - > This behavior is disabled in *Pure evaluation mode*. + This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). - To fetch the content of a checked-out work directory: diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 8a5231781..2f4d4022e 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -3,6 +3,8 @@ #include "../../toml11/toml.hpp" +#include <sstream> + namespace nix { static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val) @@ -58,8 +60,18 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V case toml::value_t::offset_datetime: case toml::value_t::local_date: case toml::value_t::local_time: - // We fail since Nix doesn't have date and time types - throw std::runtime_error("Dates and times are not supported"); + { + if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) { + auto attrs = state.buildBindings(2); + attrs.alloc("_type").mkString("timestamp"); + std::ostringstream s; + s << t; + attrs.alloc("value").mkString(s.str()); + v.mkAttrs(attrs); + } else { + throw std::runtime_error("Dates and times are not supported"); + } + } break;; case toml::value_t::empty: v.mkNull(); @@ -78,6 +90,24 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V } } -static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML); +static RegisterPrimOp primop_fromTOML({ + .name = "fromTOML", + .args = {"e"}, + .doc = R"( + Convert a TOML string to a Nix value. For example, + + ```nix + builtins.fromTOML '' + x=1 + s="a" + [table] + y=2 + '' + ``` + + returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`. + )", + .fun = prim_fromTOML +}); } diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc new file mode 100644 index 000000000..36bb4c3a5 --- /dev/null +++ b/src/libexpr/search-path.cc @@ -0,0 +1,56 @@ +#include "search-path.hh" +#include "util.hh" + +namespace nix { + +std::optional<std::string_view> SearchPath::Prefix::suffixIfPotentialMatch( + std::string_view path) const +{ + auto n = s.size(); + + /* Non-empty prefix and suffix must be separated by a /, or the + prefix is not a valid path prefix. */ + bool needSeparator = n > 0 && (path.size() - n) > 0; + + if (needSeparator && path[n] != '/') { + return std::nullopt; + } + + /* Prefix must be prefix of this path. */ + if (path.compare(0, n, s) != 0) { + return std::nullopt; + } + + /* Skip next path separator. */ + return { + path.substr(needSeparator ? n + 1 : n) + }; +} + + +SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem) +{ + size_t pos = rawElem.find('='); + + return SearchPath::Elem { + .prefix = Prefix { + .s = pos == std::string::npos + ? std::string { "" } + : std::string { rawElem.substr(0, pos) }, + }, + .path = Path { + .s = std::string { rawElem.substr(pos + 1) }, + }, + }; +} + + +SearchPath parseSearchPath(const Strings & rawElems) +{ + SearchPath res; + for (auto & rawElem : rawElems) + res.elements.emplace_back(SearchPath::Elem::parse(rawElem)); + return res; +} + +} diff --git a/src/libexpr/search-path.hh b/src/libexpr/search-path.hh new file mode 100644 index 000000000..ce78135b5 --- /dev/null +++ b/src/libexpr/search-path.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include <optional> + +#include "types.hh" +#include "comparator.hh" + +namespace nix { + +/** + * A "search path" is a list of ways look for something, used with + * `builtins.findFile` and `< >` lookup expressions. + */ +struct SearchPath +{ + /** + * A single element of a `SearchPath`. + * + * Each element is tried in succession when looking up a path. The first + * element to completely match wins. + */ + struct Elem; + + /** + * The first part of a `SearchPath::Elem` pair. + * + * Called a "prefix" because it takes the form of a prefix of a file + * path (first `n` path components). When looking up a path, to use + * a `SearchPath::Elem`, its `Prefix` must match the path. + */ + struct Prefix; + + /** + * The second part of a `SearchPath::Elem` pair. + * + * It is either a path or a URL (with certain restrictions / extra + * structure). + * + * If the prefix of the path we are looking up matches, we then + * check if the rest of the path points to something that exists + * within the directory denoted by this. If so, the + * `SearchPath::Elem` as a whole matches, and that *something* being + * pointed to by the rest of the path we are looking up is the + * result. + */ + struct Path; + + /** + * The list of search path elements. Each one is checked for a path + * when looking up. (The actual lookup entry point is in `EvalState` + * not in this class.) + */ + std::list<SearchPath::Elem> elements; + + /** + * Parse a string into a `SearchPath` + */ + static SearchPath parse(const Strings & rawElems); +}; + +struct SearchPath::Prefix +{ + /** + * Underlying string + * + * @todo Should we normalize this when constructing a `SearchPath::Prefix`? + */ + std::string s; + + GENERATE_CMP(SearchPath::Prefix, me->s); + + /** + * If the path possibly matches this search path element, return the + * suffix that we should look for inside the resolved value of the + * element + * Note the double optionality in the name. While we might have a matching prefix, the suffix may not exist. + */ + std::optional<std::string_view> suffixIfPotentialMatch(std::string_view path) const; +}; + +struct SearchPath::Path +{ + /** + * The location of a search path item, as a path or URL. + * + * @todo Maybe change this to `std::variant<SourcePath, URL>`. + */ + std::string s; + + GENERATE_CMP(SearchPath::Path, me->s); +}; + +struct SearchPath::Elem +{ + + Prefix prefix; + Path path; + + GENERATE_CMP(SearchPath::Elem, me->prefix, me->path); + + /** + * Parse a string into a `SearchPath::Elem` + */ + static SearchPath::Elem parse(std::string_view rawElem); +}; + +} diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 24e95ac39..285651256 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -171,7 +171,7 @@ namespace nix { hintfmt("value is %s while a string was expected", "an integer"), hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); - ASSERT_TRACE2("replaceStrings [ \"old\" ] [ true ] {}", + ASSERT_TRACE2("replaceStrings [ \"oo\" ] [ true ] \"foo\"", TypeError, hintfmt("value is %s while a string was expected", "a Boolean"), hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); diff --git a/src/libexpr/tests/search-path.cc b/src/libexpr/tests/search-path.cc new file mode 100644 index 000000000..dbe7ab95f --- /dev/null +++ b/src/libexpr/tests/search-path.cc @@ -0,0 +1,90 @@ +#include <gtest/gtest.h> +#include <gmock/gmock.h> + +#include "search-path.hh" + +namespace nix { + +TEST(SearchPathElem, parse_justPath) { + ASSERT_EQ( + SearchPath::Elem::parse("foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_emptyPrefix) { + ASSERT_EQ( + SearchPath::Elem::parse("=foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_oneEq) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar" }, + })); +} + +TEST(SearchPathElem, parse_twoEqs) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar=baz"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar=baz" }, + })); +} + + +TEST(SearchPathElem, suffixIfPotentialMatch_justPath) { + SearchPath::Prefix prefix { .s = "" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix1) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix2) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_partialPrefix) { + SearchPath::Prefix prefix { .s = "fooX" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_exactPrefix) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_multiKey) { + SearchPath::Prefix prefix { .s = "foo/bar" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingDoubleSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingPath) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" }); +} + +} diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc new file mode 100644 index 000000000..5e96e12ec --- /dev/null +++ b/src/libexpr/tests/value/print.cc @@ -0,0 +1,236 @@ +#include "tests/libexpr.hh" + +#include "value.hh" + +namespace nix { + +using namespace testing; + +struct ValuePrintingTests : LibExprTest +{ + template<class... A> + void test(Value v, std::string_view expected, A... args) + { + std::stringstream out; + v.print(state.symbols, out, args...); + ASSERT_EQ(out.str(), expected); + } +}; + +TEST_F(ValuePrintingTests, tInt) +{ + Value vInt; + vInt.mkInt(10); + test(vInt, "10"); +} + +TEST_F(ValuePrintingTests, tBool) +{ + Value vBool; + vBool.mkBool(true); + test(vBool, "true"); +} + +TEST_F(ValuePrintingTests, tString) +{ + Value vString; + vString.mkString("some-string"); + test(vString, "\"some-string\""); +} + +TEST_F(ValuePrintingTests, tPath) +{ + Value vPath; + vPath.mkString("/foo"); + test(vPath, "\"/foo\""); +} + +TEST_F(ValuePrintingTests, tNull) +{ + Value vNull; + vNull.mkNull(); + test(vNull, "null"); +} + +TEST_F(ValuePrintingTests, tAttrs) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + test(vAttrs, "{ one = 1; two = 2; }"); +} + +TEST_F(ValuePrintingTests, tList) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + Value vList; + state.mkList(vList, 5); + vList.bigList.elems[0] = &vOne; + vList.bigList.elems[1] = &vTwo; + vList.bigList.size = 3; + + test(vList, "[ 1 2 (nullptr) ]"); +} + +TEST_F(ValuePrintingTests, vThunk) +{ + Value vThunk; + vThunk.mkThunk(nullptr, nullptr); + + test(vThunk, "<CODE>"); +} + +TEST_F(ValuePrintingTests, vApp) +{ + Value vApp; + vApp.mkApp(nullptr, nullptr); + + test(vApp, "<CODE>"); +} + +TEST_F(ValuePrintingTests, vLambda) +{ + Value vLambda; + vLambda.mkLambda(nullptr, nullptr); + + test(vLambda, "<LAMBDA>"); +} + +TEST_F(ValuePrintingTests, vPrimOp) +{ + Value vPrimOp; + vPrimOp.mkPrimOp(nullptr); + + test(vPrimOp, "<PRIMOP>"); +} + +TEST_F(ValuePrintingTests, vPrimOpApp) +{ + Value vPrimOpApp; + vPrimOpApp.mkPrimOpApp(nullptr, nullptr); + + test(vPrimOpApp, "<PRIMOP-APP>"); +} + +TEST_F(ValuePrintingTests, vExternal) +{ + struct MyExternal : ExternalValueBase + { + public: + std::string showType() const override + { + return ""; + } + std::string typeOf() const override + { + return ""; + } + virtual std::ostream & print(std::ostream & str) const override + { + str << "testing-external!"; + return str; + } + } myExternal; + Value vExternal; + vExternal.mkExternal(&myExternal); + + test(vExternal, "testing-external!"); +} + +TEST_F(ValuePrintingTests, vFloat) +{ + Value vFloat; + vFloat.mkFloat(2.0); + + test(vFloat, "2"); +} + +TEST_F(ValuePrintingTests, vBlackhole) +{ + Value vBlackhole; + vBlackhole.mkBlackhole(); + test(vBlackhole, "«potential infinite recursion»"); +} + +TEST_F(ValuePrintingTests, depthAttrs) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + BindingsBuilder builder2(state, state.allocBindings(10)); + builder2.insert(state.symbols.create("one"), &vOne); + builder2.insert(state.symbols.create("two"), &vTwo); + builder2.insert(state.symbols.create("nested"), &vAttrs); + + Value vNested; + vNested.mkAttrs(builder2.finish()); + + test(vNested, "{ nested = «too deep»; one = «too deep»; two = «too deep»; }", false, 1); + test(vNested, "{ nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; }", false, 2); + test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 3); + test(vNested, "{ nested = { one = 1; two = 2; }; one = 1; two = 2; }", false, 4); +} + +TEST_F(ValuePrintingTests, depthList) +{ + Value vOne; + vOne.mkInt(1); + + Value vTwo; + vTwo.mkInt(2); + + BindingsBuilder builder(state, state.allocBindings(10)); + builder.insert(state.symbols.create("one"), &vOne); + builder.insert(state.symbols.create("two"), &vTwo); + + Value vAttrs; + vAttrs.mkAttrs(builder.finish()); + + BindingsBuilder builder2(state, state.allocBindings(10)); + builder2.insert(state.symbols.create("one"), &vOne); + builder2.insert(state.symbols.create("two"), &vTwo); + builder2.insert(state.symbols.create("nested"), &vAttrs); + + Value vNested; + vNested.mkAttrs(builder2.finish()); + + Value vList; + state.mkList(vList, 5); + vList.bigList.elems[0] = &vOne; + vList.bigList.elems[1] = &vTwo; + vList.bigList.elems[2] = &vNested; + vList.bigList.size = 3; + + test(vList, "[ «too deep» «too deep» «too deep» ]", false, 1); + test(vList, "[ 1 2 { nested = «too deep»; one = «too deep»; two = «too deep»; } ]", false, 2); + test(vList, "[ 1 2 { nested = { one = «too deep»; two = «too deep»; }; one = 1; two = 2; } ]", false, 3); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 4); + test(vList, "[ 1 2 { nested = { one = 1; two = 2; }; one = 1; two = 2; } ]", false, 5); +} + +} // namespace nix diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 89c0c36fd..c44683e50 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -2,6 +2,7 @@ ///@file #include <cassert> +#include <climits> #include "symbol-table.hh" #include "value/context.hh" @@ -137,11 +138,11 @@ private: friend std::string showType(const Value & v); - void print(const SymbolTable & symbols, std::ostream & str, std::set<const void *> * seen) const; + void print(const SymbolTable &symbols, std::ostream &str, std::set<const void *> *seen, int depth) const; public: - void print(const SymbolTable & symbols, std::ostream & str, bool showRepeated = false) const; + void print(const SymbolTable &symbols, std::ostream &str, bool showRepeated = false, int depth = INT_MAX) const; // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's @@ -218,8 +219,11 @@ public: /** * Returns the normal type of a Value. This only returns nThunk if * the Value hasn't been forceValue'd + * + * @param invalidIsThunk Instead of aborting an an invalid (probably + * 0, so uninitialized) internal type, return `nThunk`. */ - inline ValueType type() const + inline ValueType type(bool invalidIsThunk = false) const { switch (internalType) { case tInt: return nInt; @@ -234,7 +238,10 @@ public: case tFloat: return nFloat; case tThunk: case tApp: case tBlackhole: return nThunk; } - abort(); + if (invalidIsThunk) + return nThunk; + else + abort(); } /** diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 1a14bb023..9f885a793 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -2,6 +2,7 @@ ///@file #include "types.hh" +#include "hash.hh" #include <variant> diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 91db3a9eb..2860c1ceb 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -159,6 +159,12 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const input.to_string(), *prevLastModified); } + if (auto prevRev = getRev()) { + if (input.getRev() != prevRev) + throw Error("'rev' attribute mismatch in input '%s', expected %s", + input.to_string(), prevRev->gitRev()); + } + if (auto prevRevCount = getRevCount()) { if (input.getRevCount() != prevRevCount) throw Error("'revCount' attribute mismatch in input '%s', expected %d", diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 498ad7e4d..d0738f619 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -158,6 +158,7 @@ struct DownloadFileResult StorePath storePath; std::string etag; std::string effectiveUrl; + std::optional<std::string> immutableUrl; }; DownloadFileResult downloadFile( @@ -167,7 +168,14 @@ DownloadFileResult downloadFile( bool locked, const Headers & headers = {}); -std::pair<Tree, time_t> downloadTarball( +struct DownloadTarballResult +{ + Tree tree; + time_t lastModified; + std::optional<std::string> immutableUrl; +}; + +DownloadTarballResult downloadTarball( ref<Store> store, const std::string & url, const std::string & name, diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 1da8c9609..be5842d53 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -62,6 +62,7 @@ std::optional<std::string> readHead(const Path & path) .program = "git", // FIXME: use 'HEAD' to avoid returning all refs .args = {"ls-remote", "--symref", path}, + .isInteractive = true, }); if (status != 0) return std::nullopt; @@ -242,6 +243,13 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co "lastModified", workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + if (workdirInfo.hasHead) { + input.attrs.insert_or_assign("dirtyRev", chomp( + runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty"); + input.attrs.insert_or_assign("dirtyShortRev", chomp( + runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty"); + } + return {std::move(storePath), input}; } } // end namespace @@ -282,7 +290,7 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") throw Error("unsupported Git input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); @@ -350,7 +358,7 @@ struct GitInputScheme : InputScheme args.push_back(destDir); - runProgram("git", true, args); + runProgram("git", true, args, {}, true); } std::optional<Path> getSourcePath(const Input & input) override @@ -555,7 +563,7 @@ struct GitInputScheme : InputScheme : ref == "HEAD" ? *ref : "refs/heads/" + *ref; - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); + runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); @@ -622,7 +630,7 @@ struct GitInputScheme : InputScheme // everything to ensure we get the rev. Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir)); runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--update-head-ok", "--", repoDir, "refs/*:refs/*" }); + "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true); } runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); @@ -649,7 +657,7 @@ struct GitInputScheme : InputScheme { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl)); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }); + runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true); } filter = isNotDotGitDirectory; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 6c1d573ce..80598e7f8 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -207,21 +207,21 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers); + auto result = downloadTarball(store, url.url, input.getName(), true, url.headers); - input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); + input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); getCache()->add( store, lockedAttrs, { {"rev", rev->gitRev()}, - {"lastModified", uint64_t(lastModified)} + {"lastModified", uint64_t(result.lastModified)} }, - tree.storePath, + result.tree.storePath, true); - return {std::move(tree.storePath), input}; + return {result.tree.storePath, input}; } }; diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index f9909c218..f37a8058b 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -75,22 +75,28 @@ SourcePath SourcePath::resolveSymlinks() const int linksAllowed = 1024; - for (auto & component : path) { - res.path.push(component); - while (true) { - if (auto st = res.maybeLstat()) { + std::list<std::string> todo; + for (auto & c : path) + todo.push_back(std::string(c)); + + while (!todo.empty()) { + auto c = *todo.begin(); + todo.pop_front(); + if (c == "" || c == ".") + ; + else if (c == "..") + res.path.pop(); + else { + res.path.push(c); + if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { if (!linksAllowed--) throw Error("infinite symlink recursion in path '%s'", path); - if (st->type != InputAccessor::tSymlink) break; auto target = res.readLink(); + res.path.pop(); if (hasPrefix(target, "/")) - res = CanonPath(target); - else { - res.path.pop(); - res.path.extend(CanonPath(target)); - } - } else - break; + res.path = CanonPath::root; + todo.splice(todo.begin(), tokenizeString<std::list<std::string>>(target, "/")); + } } } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 96fe5faca..e42aca6db 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -32,7 +32,8 @@ DownloadFileResult downloadFile( return { .storePath = std::move(cached->storePath), .etag = getStrAttr(cached->infoAttrs, "etag"), - .effectiveUrl = getStrAttr(cached->infoAttrs, "url") + .effectiveUrl = getStrAttr(cached->infoAttrs, "url"), + .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; }; @@ -55,12 +56,14 @@ DownloadFileResult downloadFile( } // FIXME: write to temporary file. - Attrs infoAttrs({ {"etag", res.etag}, {"url", res.effectiveUri}, }); + if (res.immutableUrl) + infoAttrs.emplace("immutableUrl", *res.immutableUrl); + std::optional<StorePath> storePath; if (res.cached) { @@ -111,10 +114,11 @@ DownloadFileResult downloadFile( .storePath = std::move(*storePath), .etag = res.etag, .effectiveUrl = res.effectiveUri, + .immutableUrl = res.immutableUrl, }; } -std::pair<Tree, time_t> downloadTarball( +DownloadTarballResult downloadTarball( ref<Store> store, const std::string & url, const std::string & name, @@ -131,8 +135,9 @@ std::pair<Tree, time_t> downloadTarball( if (cached && !cached->expired) return { - Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, - getIntAttr(cached->infoAttrs, "lastModified") + .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, + .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), + .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; auto res = downloadFile(store, url, name, locked, headers); @@ -160,6 +165,9 @@ std::pair<Tree, time_t> downloadTarball( {"etag", res.etag}, }); + if (res.immutableUrl) + infoAttrs.emplace("immutableUrl", *res.immutableUrl); + getCache()->add( store, inAttrs, @@ -168,8 +176,9 @@ std::pair<Tree, time_t> downloadTarball( locked); return { - Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, - lastModified, + .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, + .lastModified = lastModified, + .immutableUrl = res.immutableUrl, }; } @@ -189,21 +198,33 @@ struct CurlInputScheme : InputScheme virtual bool isValidURL(const ParsedURL & url) const = 0; - std::optional<Input> inputFromURL(const ParsedURL & url) const override + std::optional<Input> inputFromURL(const ParsedURL & _url) const override { - if (!isValidURL(url)) + if (!isValidURL(_url)) return std::nullopt; Input input; - auto urlWithoutApplicationScheme = url; - urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport; + auto url = _url; + + url.scheme = parseUrlScheme(url.scheme).transport; - input.attrs.insert_or_assign("type", inputType()); - input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string()); auto narHash = url.query.find("narHash"); if (narHash != url.query.end()) input.attrs.insert_or_assign("narHash", narHash->second); + + if (auto i = get(url.query, "rev")) + input.attrs.insert_or_assign("rev", *i); + + if (auto i = get(url.query, "revCount")) + if (auto n = string2Int<uint64_t>(*i)) + input.attrs.insert_or_assign("revCount", *n); + + url.query.erase("rev"); + url.query.erase("revCount"); + + input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("url", url.to_string()); return input; } @@ -212,7 +233,8 @@ struct CurlInputScheme : InputScheme auto type = maybeGetStrAttr(attrs, "type"); if (type != inputType()) return {}; - std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack"}; + // FIXME: some of these only apply to TarballInputScheme. + std::set<std::string> allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount"}; for (auto & [name, value] : attrs) if (!allowedNames.count(name)) throw Error("unsupported %s input attribute '%s'", *type, name); @@ -275,10 +297,22 @@ struct TarballInputScheme : CurlInputScheme : hasTarballExtension(url.path)); } - std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override + std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override { - auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first; - return {std::move(tree.storePath), input}; + Input input(_input); + auto url = getStrAttr(input.attrs, "url"); + auto result = downloadTarball(store, url, input.getName(), false); + + if (result.immutableUrl) { + auto immutableInput = Input::fromURL(*result.immutableUrl); + // FIXME: would be nice to support arbitrary flakerefs + // here, e.g. git flakes. + if (immutableInput.getType() != "tarball") + throw Error("tarball 'Link' headers that redirect to non-tarball URLs are not supported"); + input = immutableInput; + } + + return {result.tree.storePath, std::move(input)}; } }; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5b1c923cd..5e37f7ecb 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -9,6 +9,7 @@ #include "archive.hh" #include "compression.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "local-store.hh" // TODO remove, along with remaining downcasts @@ -1150,9 +1151,11 @@ HookReply DerivationGoal::tryBuildHook() throw; } + WorkerProto::WriteConn conn { hook->sink }; + /* Tell the hook all the inputs that have to be copied to the remote system. */ - worker_proto::write(worker.store, hook->sink, inputPaths); + WorkerProto::write(worker.store, conn, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1163,7 +1166,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - worker_proto::write(worker.store, hook->sink, missingOutputs); + WorkerProto::write(worker.store, conn, missingOutputs); } hook->sink = FdSink(); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 74eae0692..4aa4d6dca 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -31,11 +31,11 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod } if (failed.size() == 1 && ex) { - ex->status = worker.exitStatus(); + ex->status = worker.failingExitStatus(); throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); + throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); } } @@ -102,15 +102,15 @@ void Store::ensurePath(const StorePath & path) if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { - goal->ex->status = worker.exitStatus(); + goal->ex->status = worker.failingExitStatus(); throw std::move(*goal->ex); } else - throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } } -void LocalStore::repairPath(const StorePath & path) +void Store::repairPath(const StorePath & path) { Worker worker(*this, *this); GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair); @@ -128,7 +128,7 @@ void LocalStore::repairPath(const StorePath & path) goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair)); worker.run(goals); } else - throw Error(worker.exitStatus(), "cannot repair path '%s'", printStorePath(path)); + throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); } } diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 075ad554f..337c60bd4 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -5,14 +5,14 @@ namespace nix { HookInstance::HookInstance() { - debug("starting build hook '%s'", settings.buildHook); + debug("starting build hook '%s'", concatStringsSep(" ", settings.buildHook.get())); - auto buildHookArgs = tokenizeString<std::list<std::string>>(settings.buildHook.get()); + auto buildHookArgs = settings.buildHook.get(); if (buildHookArgs.empty()) throw Error("'build-hook' setting is empty"); - auto buildHook = buildHookArgs.front(); + auto buildHook = canonPath(buildHookArgs.front()); buildHookArgs.pop_front(); Strings args; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 9f21a711a..53e6998e8 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -4,13 +4,12 @@ #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "references.hh" +#include "path-references.hh" #include "finally.hh" #include "util.hh" #include "archive.hh" #include "compression.hh" #include "daemon.hh" -#include "worker-protocol.hh" #include "topo-sort.hh" #include "callback.hh" #include "json-utils.hh" @@ -65,8 +64,9 @@ void handleDiffHook( const Path & tryA, const Path & tryB, const Path & drvPath, const Path & tmpDir) { - auto diffHook = settings.diffHook; - if (diffHook != "" && settings.runDiffHook) { + auto & diffHookOpt = settings.diffHook.get(); + if (diffHookOpt && settings.runDiffHook) { + auto & diffHook = *diffHookOpt; try { auto diffRes = runProgram(RunOptions { .program = diffHook, @@ -357,7 +357,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() for (auto & [_, status] : initialOutputs) { if (!status.known) continue; if (buildMode != bmCheck && status.known->isValid()) continue; - auto p = worker.store.printStorePath(status.known->path); + auto p = worker.store.toRealPath(status.known->path); if (pathExists(chrootRootDir + p)) renameFile((chrootRootDir + p), p); } @@ -395,8 +395,9 @@ static void linkOrCopy(const Path & from, const Path & to) bind-mount in this case? It can also fail with EPERM in BeegFS v7 and earlier versions + or fail with EXDEV in OpenAFS which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM) + if (errno != EMLINK && errno != EPERM && errno != EXDEV) throw SysError("linking '%s' to '%s'", to, from); copyPath(from, to); } @@ -1423,7 +1424,8 @@ void LocalDerivationGoal::startDaemon() Store::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; - params["root"] = getLocalStore().rootDir; + if (auto & optRoot = getLocalStore().rootDir.get()) + params["root"] = *optRoot; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; auto store = make_ref<RestrictedStore>(params, @@ -1452,7 +1454,7 @@ void LocalDerivationGoal::startDaemon() (struct sockaddr *) &remoteAddr, &remoteAddrLen); if (!remote) { if (errno == EINTR || errno == EAGAIN) continue; - if (errno == EINVAL) break; + if (errno == EINVAL || errno == ECONNABORTED) break; throw SysError("accepting connection"); } @@ -1482,8 +1484,22 @@ void LocalDerivationGoal::startDaemon() void LocalDerivationGoal::stopDaemon() { - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); + if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) { + // According to the POSIX standard, the 'shutdown' function should + // return an ENOTCONN error when attempting to shut down a socket that + // hasn't been connected yet. This situation occurs when the 'accept' + // function is called on a socket without any accepted connections, + // leaving the socket unconnected. While Linux doesn't seem to produce + // an error for sockets that have only been accepted, more + // POSIX-compliant operating systems like OpenBSD, macOS, and others do + // return the ENOTCONN error. Therefore, we handle this error here to + // avoid raising an exception for compliant behaviour. + if (errno == ENOTCONN) { + daemonSocket.close(); + } else { + throw SysError("shutting down daemon socket"); + } + } if (daemonThread.joinable()) daemonThread.join(); @@ -1494,7 +1510,8 @@ void LocalDerivationGoal::stopDaemon() thread.join(); daemonWorkerThreads.clear(); - daemonSocket = -1; + // release the socket. + daemonSocket.close(); } @@ -1771,6 +1788,9 @@ void LocalDerivationGoal::runChild() for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts" }) if (pathExists(path)) ss.push_back(path); + + if (settings.caFile != "") + dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); } for (auto & i : ss) dirsInChroot.emplace(i, i); @@ -2371,18 +2391,21 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() continue; auto references = *referencesOpt; - auto rewriteOutput = [&]() { + auto rewriteOutput = [&](const StringMap & rewrites) { /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { + if (!rewrites.empty()) { debug("rewriting hashes in '%1%'; cross fingers", actualPath); - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); + /* FIXME: Is this actually streaming? */ + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink(rewrites, nextSink); + dumpPath(actualPath, rsink); + rsink.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); deletePath(actualPath); - sink.s = rewriteStrings(sink.s, outputRewrites); - StringSource source(sink.s); - restorePath(actualPath, source); + movePath(tmpPath, actualPath); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ @@ -2431,7 +2454,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", actualPath); } - rewriteOutput(); + rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; HashModuloSink caSink { outputHash.hashType, oldHashPart }; @@ -2469,16 +2492,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() Hash::dummy, }; if (*scratchPath != newInfo0.path) { - // Also rewrite the output path - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - dumpPath(actualPath, rsink2); - rsink2.flush(); - }); - Path tmpPath = actualPath + ".tmp"; - restorePath(tmpPath, *source); - deletePath(actualPath); - movePath(tmpPath, actualPath); + // If the path has some self-references, we need to rewrite + // them. + // (note that this doesn't invalidate the ca hash we calculated + // above because it's computed *modulo the self-references*, so + // it already takes this rewrite into account). + rewriteOutput( + StringMap{{oldHashPart, + std::string(newInfo0.path.hashPart())}}); } HashResult narHashAndSize = hashPath(htSHA256, actualPath); @@ -2500,7 +2521,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() outputRewrites.insert_or_assign( std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); + rewriteOutput(outputRewrites); auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; diff --git a/src/libstore/build/personality.cc b/src/libstore/build/personality.cc index 4ad477869..1a6201758 100644 --- a/src/libstore/build/personality.cc +++ b/src/libstore/build/personality.cc @@ -21,7 +21,8 @@ void setPersonality(std::string_view system) && (std::string_view(SYSTEM) == "x86_64-linux" || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) || system == "armv7l-linux" - || system == "armv6l-linux") + || system == "armv6l-linux" + || system == "armv5tel-linux") { if (personality(PER_LINUX32) == -1) throw SysError("cannot set 32-bit personality"); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index ee334d54a..a9ca9cbbc 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -468,16 +468,9 @@ void Worker::waitForInput() } -unsigned int Worker::exitStatus() +unsigned int Worker::failingExitStatus() { - /* - * 1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - */ + // See API docs in header for explanation unsigned int mask = 0; bool buildFailure = permanentFailure || timedOut || hashMismatch; if (buildFailure) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 63624d910..5abceca0d 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -280,7 +280,28 @@ public: */ void waitForInput(); - unsigned int exitStatus(); + /*** + * The exit status in case of failure. + * + * In the case of a build failure, returned value follows this + * bitmask: + * + * ``` + * 0b1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + * ``` + * + * In other words, the failure code is at least 100 (0b1100100), but + * might also be greater. + * + * Otherwise (no build failure, but some other sort of failure by + * assumption), this returned value is 1. + */ + unsigned int failingExitStatus(); /** * Check whether the given valid path exists and has the right diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 5083497a9..ad3dee1a2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,6 +1,7 @@ #include "daemon.hh" #include "monitor-fd.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" #include "store-cast.hh" @@ -259,13 +260,13 @@ struct ClientSettings } }; -static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, Source & from) +static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, WorkerProto::ReadConn conn) { std::vector<DerivedPath> reqs; if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {}); + reqs = WorkerProto::Serialise<std::vector<DerivedPath>>::read(store, conn); } else { - for (auto & s : readStrings<Strings>(from)) + for (auto & s : readStrings<Strings>(conn.from)) reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); } return reqs; @@ -273,11 +274,14 @@ static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int cli static void performOp(TunnelLogger * logger, ref<Store> store, TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, - Source & from, BufferedSink & to, unsigned int op) + Source & from, BufferedSink & to, WorkerProto::Op op) { + WorkerProto::ReadConn rconn { .from = from }; + WorkerProto::WriteConn wconn { .to = to }; + switch (op) { - case wopIsValidPath: { + case WorkerProto::Op::IsValidPath: { auto path = store->parseStorePath(readString(from)); logger->startWork(); bool result = store->isValidPath(path); @@ -286,8 +290,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopQueryValidPaths: { - auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + case WorkerProto::Op::QueryValidPaths: { + auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); SubstituteFlag substitute = NoSubstitute; if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { @@ -300,11 +304,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } auto res = store->queryValidPaths(paths, substitute); logger->stopWork(); - worker_proto::write(*store, to, res); + WorkerProto::write(*store, wconn, res); break; } - case wopHasSubstitutes: { + case WorkerProto::Op::HasSubstitutes: { auto path = store->parseStorePath(readString(from)); logger->startWork(); StorePathSet paths; // FIXME @@ -315,16 +319,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopQuerySubstitutablePaths: { - auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + case WorkerProto::Op::QuerySubstitutablePaths: { + auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); logger->startWork(); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - worker_proto::write(*store, to, res); + WorkerProto::write(*store, wconn, res); break; } - case wopQueryPathHash: { + case WorkerProto::Op::QueryPathHash: { auto path = store->parseStorePath(readString(from)); logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; @@ -333,27 +337,27 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopQueryReferences: - case wopQueryReferrers: - case wopQueryValidDerivers: - case wopQueryDerivationOutputs: { + case WorkerProto::Op::QueryReferences: + case WorkerProto::Op::QueryReferrers: + case WorkerProto::Op::QueryValidDerivers: + case WorkerProto::Op::QueryDerivationOutputs: { auto path = store->parseStorePath(readString(from)); logger->startWork(); StorePathSet paths; - if (op == wopQueryReferences) + if (op == WorkerProto::Op::QueryReferences) for (auto & i : store->queryPathInfo(path)->references) paths.insert(i); - else if (op == wopQueryReferrers) + else if (op == WorkerProto::Op::QueryReferrers) store->queryReferrers(path, paths); - else if (op == wopQueryValidDerivers) + else if (op == WorkerProto::Op::QueryValidDerivers) paths = store->queryValidDerivers(path); else paths = store->queryDerivationOutputs(path); logger->stopWork(); - worker_proto::write(*store, to, paths); + WorkerProto::write(*store, wconn, paths); break; } - case wopQueryDerivationOutputNames: { + case WorkerProto::Op::QueryDerivationOutputNames: { auto path = store->parseStorePath(readString(from)); logger->startWork(); auto names = store->readDerivation(path).outputNames(); @@ -362,16 +366,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopQueryDerivationOutputMap: { + case WorkerProto::Op::QueryDerivationOutputMap: { auto path = store->parseStorePath(readString(from)); logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); - worker_proto::write(*store, to, outputs); + WorkerProto::write(*store, wconn, outputs); break; } - case wopQueryDeriver: { + case WorkerProto::Op::QueryDeriver: { auto path = store->parseStorePath(readString(from)); logger->startWork(); auto info = store->queryPathInfo(path); @@ -380,7 +384,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopQueryPathFromHashPart: { + case WorkerProto::Op::QueryPathFromHashPart: { auto hashPart = readString(from); logger->startWork(); auto path = store->queryPathFromHashPart(hashPart); @@ -389,11 +393,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopAddToStore: { + case WorkerProto::Op::AddToStore: { if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { auto name = readString(from); auto camStr = readString(from); - auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); bool repairBool; from >> repairBool; auto repair = RepairFlag{repairBool}; @@ -475,7 +479,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopAddMultipleToStore: { + case WorkerProto::Op::AddMultipleToStore: { bool repair, dontCheckSigs; from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) @@ -492,10 +496,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopAddTextToStore: { + case WorkerProto::Op::AddTextToStore: { std::string suffix = readString(from); std::string s = readString(from); - auto refs = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + auto refs = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); logger->startWork(); auto path = store->addTextToStore(suffix, s, refs, NoRepair); logger->stopWork(); @@ -503,7 +507,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopExportPath: { + case WorkerProto::Op::ExportPath: { auto path = store->parseStorePath(readString(from)); readInt(from); // obsolete logger->startWork(); @@ -514,7 +518,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopImportPaths: { + case WorkerProto::Op::ImportPaths: { logger->startWork(); TunnelSource source(from, to); auto paths = store->importPaths(source, @@ -526,8 +530,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopBuildPaths: { - auto drvs = readDerivedPaths(*store, clientVersion, from); + case WorkerProto::Op::BuildPaths: { + auto drvs = readDerivedPaths(*store, clientVersion, rconn); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -551,8 +555,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopBuildPathsWithResults: { - auto drvs = readDerivedPaths(*store, clientVersion, from); + case WorkerProto::Op::BuildPathsWithResults: { + auto drvs = readDerivedPaths(*store, clientVersion, rconn); BuildMode mode = bmNormal; mode = (BuildMode) readInt(from); @@ -567,12 +571,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store, auto results = store->buildPathsWithResults(drvs, mode); logger->stopWork(); - worker_proto::write(*store, to, results); + WorkerProto::write(*store, wconn, results); break; } - case wopBuildDerivation: { + case WorkerProto::Op::BuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); @@ -644,12 +648,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store, DrvOutputs builtOutputs; for (auto & [output, realisation] : res.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - worker_proto::write(*store, to, builtOutputs); + WorkerProto::write(*store, wconn, builtOutputs); } break; } - case wopEnsurePath: { + case WorkerProto::Op::EnsurePath: { auto path = store->parseStorePath(readString(from)); logger->startWork(); store->ensurePath(path); @@ -658,7 +662,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopAddTempRoot: { + case WorkerProto::Op::AddTempRoot: { auto path = store->parseStorePath(readString(from)); logger->startWork(); store->addTempRoot(path); @@ -667,7 +671,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopAddIndirectRoot: { + case WorkerProto::Op::AddIndirectRoot: { Path path = absPath(readString(from)); logger->startWork(); @@ -680,14 +684,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } // Obsolete. - case wopSyncWithGC: { + case WorkerProto::Op::SyncWithGC: { logger->startWork(); logger->stopWork(); to << 1; break; } - case wopFindRoots: { + case WorkerProto::Op::FindRoots: { logger->startWork(); auto & gcStore = require<GcStore>(*store); Roots roots = gcStore.findRoots(!trusted); @@ -706,10 +710,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopCollectGarbage: { + case WorkerProto::Op::CollectGarbage: { GCOptions options; options.action = (GCOptions::GCAction) readInt(from); - options.pathsToDelete = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + options.pathsToDelete = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields readInt(from); @@ -730,7 +734,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopSetOptions: { + case WorkerProto::Op::SetOptions: { ClientSettings clientSettings; @@ -767,7 +771,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopQuerySubstitutablePathInfo: { + case WorkerProto::Op::QuerySubstitutablePathInfo: { auto path = store->parseStorePath(readString(from)); logger->startWork(); SubstitutablePathInfos infos; @@ -779,22 +783,22 @@ static void performOp(TunnelLogger * logger, ref<Store> store, else { to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); - worker_proto::write(*store, to, i->second.references); + WorkerProto::write(*store, wconn, i->second.references); to << i->second.downloadSize << i->second.narSize; } break; } - case wopQuerySubstitutablePathInfos: { + case WorkerProto::Op::QuerySubstitutablePathInfos: { SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; if (GET_PROTOCOL_MINOR(clientVersion) < 22) { - auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); } else - pathsMap = worker_proto::read(*store, from, Phantom<StorePathCAMap> {}); + pathsMap = WorkerProto::Serialise<StorePathCAMap>::read(*store, rconn); logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); @@ -802,21 +806,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store, for (auto & i : infos) { to << store->printStorePath(i.first) << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); - worker_proto::write(*store, to, i.second.references); + WorkerProto::write(*store, wconn, i.second.references); to << i.second.downloadSize << i.second.narSize; } break; } - case wopQueryAllValidPaths: { + case WorkerProto::Op::QueryAllValidPaths: { logger->startWork(); auto paths = store->queryAllValidPaths(); logger->stopWork(); - worker_proto::write(*store, to, paths); + WorkerProto::write(*store, wconn, paths); break; } - case wopQueryPathInfo: { + case WorkerProto::Op::QueryPathInfo: { auto path = store->parseStorePath(readString(from)); std::shared_ptr<const ValidPathInfo> info; logger->startWork(); @@ -837,14 +841,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopOptimiseStore: + case WorkerProto::Op::OptimiseStore: logger->startWork(); store->optimiseStore(); logger->stopWork(); to << 1; break; - case wopVerifyStore: { + case WorkerProto::Op::VerifyStore: { bool checkContents, repair; from >> checkContents >> repair; logger->startWork(); @@ -856,19 +860,17 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopAddSignatures: { + case WorkerProto::Op::AddSignatures: { auto path = store->parseStorePath(readString(from)); StringSet sigs = readStrings<StringSet>(from); logger->startWork(); - if (!trusted) - throw Error("you are not privileged to add signatures"); store->addSignatures(path, sigs); logger->stopWork(); to << 1; break; } - case wopNarFromPath: { + case WorkerProto::Op::NarFromPath: { auto path = store->parseStorePath(readString(from)); logger->startWork(); logger->stopWork(); @@ -876,7 +878,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopAddToStoreNar: { + case WorkerProto::Op::AddToStoreNar: { bool repair, dontCheckSigs; auto path = store->parseStorePath(readString(from)); auto deriver = readString(from); @@ -884,7 +886,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, from, Phantom<StorePathSet> {}); + info.references = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); from >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(from); info.ca = ContentAddress::parseOpt(readString(from)); @@ -928,21 +930,21 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } - case wopQueryMissing: { - auto targets = readDerivedPaths(*store, clientVersion, from); + case WorkerProto::Op::QueryMissing: { + auto targets = readDerivedPaths(*store, clientVersion, rconn); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); logger->stopWork(); - worker_proto::write(*store, to, willBuild); - worker_proto::write(*store, to, willSubstitute); - worker_proto::write(*store, to, unknown); + WorkerProto::write(*store, wconn, willBuild); + WorkerProto::write(*store, wconn, willSubstitute); + WorkerProto::write(*store, wconn, unknown); to << downloadSize << narSize; break; } - case wopRegisterDrvOutput: { + case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); if (GET_PROTOCOL_MINOR(clientVersion) < 31) { auto outputId = DrvOutput::parse(readString(from)); @@ -950,14 +952,14 @@ static void performOp(TunnelLogger * logger, ref<Store> store, store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath}); } else { - auto realisation = worker_proto::read(*store, from, Phantom<Realisation>()); + auto realisation = WorkerProto::Serialise<Realisation>::read(*store, rconn); store->registerDrvOutput(realisation); } logger->stopWork(); break; } - case wopQueryRealisation: { + case WorkerProto::Op::QueryRealisation: { logger->startWork(); auto outputId = DrvOutput::parse(readString(from)); auto info = store->queryRealisation(outputId); @@ -965,16 +967,16 @@ static void performOp(TunnelLogger * logger, ref<Store> store, if (GET_PROTOCOL_MINOR(clientVersion) < 31) { std::set<StorePath> outPaths; if (info) outPaths.insert(info->outPath); - worker_proto::write(*store, to, outPaths); + WorkerProto::write(*store, wconn, outPaths); } else { std::set<Realisation> realisations; if (info) realisations.insert(*info); - worker_proto::write(*store, to, realisations); + WorkerProto::write(*store, wconn, realisations); } break; } - case wopAddBuildLog: { + case WorkerProto::Op::AddBuildLog: { StorePath path{readString(from)}; logger->startWork(); if (!trusted) @@ -991,6 +993,10 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } + case WorkerProto::Op::QueryFailedPaths: + case WorkerProto::Op::ClearFailedPaths: + throw Error("Removed operation %1%", op); + default: throw Error("invalid operation %1%", op); } @@ -1045,7 +1051,8 @@ void processConnection( auto temp = trusted ? store->isTrustedClient() : std::optional { NotTrusted }; - worker_proto::write(*store, to, temp); + WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::write(*store, wconn, temp); } /* Send startup error messages to the client. */ @@ -1058,9 +1065,9 @@ void processConnection( /* Process client requests. */ while (true) { - WorkerOp op; + WorkerProto::Op op; try { - op = (WorkerOp) readInt(from); + op = (enum WorkerProto::Op) readInt(from); } catch (Interrupted & e) { break; } catch (EndOfFile & e) { diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index d56dc727b..6f63685d4 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1,9 +1,11 @@ #include "derivations.hh" +#include "downstream-placeholder.hh" #include "store-api.hh" #include "globals.hh" #include "util.hh" #include "split.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "fs-accessor.hh" #include <boost/container/small_vector.hpp> #include <nlohmann/json.hpp> @@ -748,7 +750,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = worker_proto::read(store, in, Phantom<StorePathSet> {}); + drv.inputSrcs = WorkerProto::Serialise<StorePathSet>::read(store, + WorkerProto::ReadConn { .from = in }); in >> drv.platform >> drv.builder; drv.args = readStrings<Strings>(in); @@ -796,7 +799,9 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, }, i.second.raw()); } - worker_proto::write(store, out, drv.inputSrcs); + WorkerProto::write(store, + WorkerProto::WriteConn { .to = out }, + drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); for (auto & i : drv.env) @@ -810,13 +815,7 @@ std::string hashPlaceholder(const std::string_view outputName) return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); } -std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName) -{ - auto drvNameWithExtension = drvPath.name(); - auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); - auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); - return "/" + hashString(htSHA256, clearText).to_string(Base32, false); -} + static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) @@ -880,7 +879,7 @@ std::optional<BasicDerivation> Derivation::tryResolve( for (auto & outputName : inputOutputs) { if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { inputRewrites.emplace( - downstreamPlaceholder(store, inputDrv, outputName), + DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), store.printStorePath(*actualPath)); resolved.inputSrcs.insert(*actualPath); } else { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 1e2143f31..fa79f77fd 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -6,6 +6,7 @@ #include "hash.hh" #include "content-address.hh" #include "repair-flag.hh" +#include "derived-path.hh" #include "sync.hh" #include "comparator.hh" @@ -495,17 +496,6 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr */ std::string hashPlaceholder(const std::string_view outputName); -/** - * This creates an opaque and almost certainly unique string - * deterministically from a derivation path and output name. - * - * It is used as a placeholder to allow derivations to refer to - * content-addressed paths whose content --- and thus the path - * themselves --- isn't yet known. This occurs when a derivation has a - * dependency which is a CA derivation. - */ -std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); - extern const Hash impureOutputHash; } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc new file mode 100644 index 000000000..1752738f2 --- /dev/null +++ b/src/libstore/downstream-placeholder.cc @@ -0,0 +1,39 @@ +#include "downstream-placeholder.hh" +#include "derivations.hh" + +namespace nix { + +std::string DownstreamPlaceholder::render() const +{ + return "/" + hash.to_string(Base32, false); +} + + +DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( + const StorePath & drvPath, + std::string_view outputName) +{ + auto drvNameWithExtension = drvPath.name(); + auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); + auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); + return DownstreamPlaceholder { + hashString(htSHA256, clearText) + }; +} + +DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( + const DownstreamPlaceholder & placeholder, + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::DynamicDerivations); + auto compressed = compressHash(placeholder.hash, 20); + auto clearText = "nix-computed-output:" + + compressed.to_string(Base32, false) + + ":" + std::string { outputName }; + return DownstreamPlaceholder { + hashString(htSHA256, clearText) + }; +} + +} diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh new file mode 100644 index 000000000..f0c0dee77 --- /dev/null +++ b/src/libstore/downstream-placeholder.hh @@ -0,0 +1,75 @@ +#pragma once +///@file + +#include "hash.hh" +#include "path.hh" + +namespace nix { + +/** + * Downstream Placeholders are opaque and almost certainly unique values + * used to allow derivations to refer to store objects which are yet to + * be built and for we do not yet have store paths for. + * + * They correspond to `DerivedPaths` that are not `DerivedPath::Opaque`, + * except for the cases involving input addressing or fixed outputs + * where we do know a store path for the derivation output in advance. + * + * Unlike `DerivationPath`, however, `DownstreamPlaceholder` is + * purposefully opaque and obfuscated. This is so they are hard to + * create by accident, and so substituting them (once we know what the + * path to store object is) is unlikely to capture other stuff it + * shouldn't. + * + * We use them with `Derivation`: the `render()` method is called to + * render an opaque string which can be used in the derivation, and the + * resolving logic can substitute those strings for store paths when + * resolving `Derivation.inputDrvs` to `BasicDerivation.inputSrcs`. + */ +class DownstreamPlaceholder +{ + /** + * `DownstreamPlaceholder` is just a newtype of `Hash`. + * This its only field. + */ + Hash hash; + + /** + * Newtype constructor + */ + DownstreamPlaceholder(Hash hash) : hash(hash) { } + +public: + /** + * This creates an opaque and almost certainly unique string + * deterministically from the placeholder. + */ + std::string render() const; + + /** + * Create a placeholder for an unknown output of a content-addressed + * derivation. + * + * The derivation itself is known (we have a store path for it), but + * the output doesn't yet have a known store path. + */ + static DownstreamPlaceholder unknownCaOutput( + const StorePath & drvPath, + std::string_view outputName); + + /** + * Create a placehold for the output of an unknown derivation. + * + * The derivation is not yet known because it is a dynamic + * derivaiton --- it is itself an output of another derivation --- + * and we just have (another) placeholder for it. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DownstreamPlaceholder unknownDerivation( + const DownstreamPlaceholder & drvPlaceholder, + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); +}; + +} diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 4eb838b68..e866aeb42 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -2,6 +2,7 @@ #include "store-api.hh" #include "archive.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include <algorithm> @@ -45,7 +46,9 @@ void Store::exportPath(const StorePath & path, Sink & sink) teeSink << exportMagic << printStorePath(path); - worker_proto::write(*this, teeSink, info->references); + WorkerProto::write(*this, + WorkerProto::WriteConn { .to = teeSink }, + info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; @@ -73,7 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = worker_proto::read(*this, source, Phantom<StorePathSet> {}); + auto references = WorkerProto::Serialise<StorePathSet>::read(*this, + WorkerProto::ReadConn { .from = source }); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 2346accbe..38b691279 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -186,9 +186,9 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; std::string line((char *) contents, realSize); printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); + static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); - std::smatch match; - if (std::regex_match(line, match, statusLine)) { + if (std::smatch match; std::regex_match(line, match, statusLine)) { result.etag = ""; result.data.clear(); result.bodySize = 0; @@ -196,9 +196,11 @@ struct curlFileTransfer : public FileTransfer acceptRanges = false; encoding = ""; } else { + auto i = line.find(':'); if (i != std::string::npos) { std::string name = toLower(trim(line.substr(0, i))); + if (name == "etag") { result.etag = trim(line.substr(i + 1)); /* Hack to work around a GitHub bug: it sends @@ -212,10 +214,22 @@ struct curlFileTransfer : public FileTransfer debug("shutting down on 200 HTTP response with expected ETag"); return 0; } - } else if (name == "content-encoding") + } + + else if (name == "content-encoding") encoding = trim(line.substr(i + 1)); + else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes") acceptRanges = true; + + else if (name == "link" || name == "x-amz-meta-link") { + auto value = trim(line.substr(i + 1)); + static std::regex linkRegex("<([^>]*)>; rel=\"immutable\"", std::regex::extended | std::regex::icase); + if (std::smatch match; std::regex_match(value, match, linkRegex)) + result.immutableUrl = match.str(1); + else + debug("got invalid link header '%s'", value); + } } } return realSize; @@ -345,7 +359,7 @@ struct curlFileTransfer : public FileTransfer { auto httpStatus = getHTTPStatus(); - char * effectiveUriCStr; + char * effectiveUriCStr = nullptr; curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); if (effectiveUriCStr) result.effectiveUri = effectiveUriCStr; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 378c6ff78..a3b0dde1f 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -80,6 +80,10 @@ struct FileTransferResult std::string effectiveUri; std::string data; uint64_t bodySize = 0; + /* An "immutable" URL for this resource (i.e. one whose contents + will never change), as returned by the `Link: <url>; + rel="immutable"` header. */ + std::optional<std::string> immutableUrl; }; class Store; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0038ec802..20720fb99 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -110,6 +110,11 @@ void LocalStore::createTempRootsFile() void LocalStore::addTempRoot(const StorePath & path) { + if (readOnly) { + debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); + return; + } + createTempRootsFile(); /* Open/create the global GC lock file. */ @@ -563,7 +568,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* On macOS, accepted sockets inherit the non-blocking flag from the server socket, so explicitly make it blocking. */ - if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1) + if (fcntl(fdClient.get(), F_SETFL, fcntl(fdClient.get(), F_GETFL) & ~O_NONBLOCK) == -1) abort(); while (true) { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4c66d08ee..5a4cb1824 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -77,7 +77,33 @@ Settings::Settings() allowedImpureHostPrefixes = tokenizeString<StringSet>("/System/Library /usr/lib /dev /bin/sh"); #endif - buildHook = getSelfExe().value_or("nix") + " __build-remote"; + /* Set the build hook location + + For builds we perform a self-invocation, so Nix has to be self-aware. + That is, it has to know where it is installed. We don't think it's sentient. + + Normally, nix is installed according to `nixBinDir`, which is set at compile time, + but can be overridden. This makes for a great default that works even if this + code is linked as a library into some other program whose main is not aware + that it might need to be a build remote hook. + + However, it may not have been installed at all. For example, if it's a static build, + there's a good chance that it has been moved out of its installation directory. + That makes `nixBinDir` useless. Instead, we'll query the OS for the path to the + current executable, using `getSelfExe()`. + + As a last resort, we resort to `PATH`. Hopefully we find a `nix` there that's compatible. + If you're porting Nix to a new platform, that might be good enough for a while, but + you'll want to improve `getSelfExe()` to work on your platform. + */ + std::string nixExePath = nixBinDir + "/nix"; + if (!pathExists(nixExePath)) { + nixExePath = getSelfExe().value_or("nix"); + } + buildHook = { + nixExePath, + "__build-remote", + }; } void loadConfFile() @@ -183,7 +209,7 @@ bool Settings::isWSL1() Path Settings::getDefaultSSLCertFile() { for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) - if (pathExists(fn)) return fn; + if (pathAccessible(fn)) return fn; return ""; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a7cf36d83..a19b43086 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -236,7 +236,7 @@ public: )", {"build-timeout"}}; - PathSetting buildHook{this, true, "", "build-hook", + Setting<Strings> buildHook{this, {}, "build-hook", R"( The path to the helper program that executes remote builds. @@ -575,8 +575,8 @@ public: line. )"}; - PathSetting diffHook{ - this, true, "", "diff-hook", + OptionalPathSetting diffHook{ + this, std::nullopt, "diff-hook", R"( Absolute path to an executable capable of diffing build results. The hook is executed if `run-diff-hook` is true, and the @@ -710,32 +710,29 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - to be used as substituters, separated by whitespace. - Substituters are tried based on their Priority value, which each substituter can set - independently. Lower value means higher priority. - The default is `https://cache.nixos.org`, with a Priority of 40. + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace. + A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. - At least one of the following conditions must be met for Nix to use - a substituter: + Substituters are tried based on their priority value, which each substituter can set independently. + Lower value means higher priority. + The default is `https://cache.nixos.org`, which has a priority of 40. - - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + At least one of the following conditions must be met for Nix to use a substituter: - In addition, each store path should be trusted as described - in [`trusted-public-keys`](#conf-trusted-public-keys) + - The substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list + - The user calling Nix is in the [`trusted-users`](#conf-trusted-users) list + + In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; Setting<StringSet> trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), - separated by whitespace. These are - not used by default, but can be enabled by users of the Nix daemon - by specifying `--option substituters urls` on the command - line. Unprivileged users are only allowed to pass a subset of the - URLs listed in `substituters` and `trusted-substituters`. + A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. + These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters). + + Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. )", {"trusted-binary-caches"}}; @@ -915,12 +912,11 @@ public: this, {}, "hashed-mirrors", R"( A list of web servers used by `builtins.fetchurl` to obtain files by - hash. The default is `http://tarballs.nixos.org/`. Given a hash type - *ht* and a base-16 hash *h*, Nix will try to download the file from - *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if - they have disappeared from their original URI. For example, given - the default mirror `http://tarballs.nixos.org/`, when building the - derivation + hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to + download the file from *hashed-mirror*/*ht*/*h*. This allows files to + be downloaded even if they have disappeared from their original URI. + For example, given an example mirror `http://tarballs.nixos.org/`, + when building the derivation ```nix builtins.fetchurl { @@ -1014,6 +1010,18 @@ public: | `~/.nix-profile` | `$XDG_STATE_HOME/nix/profile` | | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | + + If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option. + If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`. + This can be achieved with the following shell commands: + + ```sh + nix_state_home=${XDG_STATE_HOME-$HOME/.local/state}/nix + mkdir -p $nix_state_home + mv $HOME/.nix-profile $nix_state_home/profile + mv $HOME/.nix-defexpr $nix_state_home/defexpr + mv $HOME/.nix-channels $nix_state_home/channels + ``` )" }; }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 2012584e0..fa17d606d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -7,6 +7,7 @@ #include "store-api.hh" #include "path-with-outputs.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" @@ -47,6 +48,42 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor FdSource from; int remoteVersion; bool good = true; + + /** + * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the + * factored out worker protocol searlizers with a + * `LegacySSHStore::Connection`. + * + * The worker protocol connection types are unidirectional, unlike + * this type. + * + * @todo Use server protocol serializers, not worker protocol + * serializers, once we have made that distiction. + */ + operator WorkerProto::ReadConn () + { + return WorkerProto::ReadConn { + .from = from, + }; + } + + /* + * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the + * factored out worker protocol searlizers with a + * `LegacySSHStore::Connection`. + * + * The worker protocol connection types are unidirectional, unlike + * this type. + * + * @todo Use server protocol serializers, not worker protocol + * serializers, once we have made that distiction. + */ + operator WorkerProto::WriteConn () + { + return WorkerProto::WriteConn { + .to = to, + }; + } }; std::string host; @@ -133,7 +170,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); - conn->to << cmdQueryPathInfos << PathSet{printStorePath(path)}; + conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)}; conn->to.flush(); auto p = readString(conn->from); @@ -146,7 +183,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info->references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -176,11 +213,11 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { conn->to - << cmdAddToStoreNar + << ServeProto::Command::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -198,7 +235,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor } else { conn->to - << cmdImportPaths + << ServeProto::Command::ImportPaths << 1; try { copyNAR(source, conn->to); @@ -209,7 +246,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - worker_proto::write(*this, conn->to, info.references); + WorkerProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -226,7 +263,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { auto conn(connections->get()); - conn->to << cmdDumpStorePath << printStorePath(path); + conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); conn->to.flush(); copyNAR(conn->from, sink); } @@ -279,7 +316,7 @@ public: auto conn(connections->get()); conn->to - << cmdBuildDerivation + << ServeProto::Command::BuildDerivation << printStorePath(drvPath); writeDerivation(conn->to, *this, drv); @@ -294,7 +331,7 @@ public: if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {}); + auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) status.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -310,7 +347,7 @@ public: auto conn(connections->get()); - conn->to << cmdBuildPaths; + conn->to << ServeProto::Command::BuildPaths; Strings ss; for (auto & p : drvPaths) { auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); @@ -344,6 +381,17 @@ public: virtual ref<FSAccessor> getFSAccessor() override { unsupported("getFSAccessor"); } + /** + * The default instance would schedule the work on the client side, but + * for consistency with `buildPaths` and `buildDerivation` it should happen + * on the remote side. + * + * We make this fail for now so we can add implement this properly later + * without it being a breaking change. + */ + void repairPath(const StorePath & path) override + { unsupported("repairPath"); } + void computeFSClosure(const StorePathSet & paths, StorePathSet & out, bool flipDirection = false, bool includeOutputs = false, bool includeDerivers = false) override @@ -356,12 +404,12 @@ public: auto conn(connections->get()); conn->to - << cmdQueryClosure + << ServeProto::Command::QueryClosure << includeOutputs; - worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})) + for (auto & i : WorkerProto::Serialise<StorePathSet>::read(*this, *conn)) out.insert(i); } @@ -371,13 +419,13 @@ public: auto conn(connections->get()); conn->to - << cmdQueryValidPaths + << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, *conn, paths); conn->to.flush(); - return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + return WorkerProto::Serialise<StorePathSet>::read(*this, *conn); } void connect() override diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index a03bb88f5..2ee2ef0c8 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -15,22 +15,22 @@ struct LocalFSStoreConfig : virtual StoreConfig // it to omit the call to the Setting constructor. Clang works fine // either way. - const PathSetting rootDir{(StoreConfig*) this, true, "", + const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt, "root", "Directory prefixed to all other paths."}; - const PathSetting stateDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, + const PathSetting stateDir{(StoreConfig*) this, + rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, "state", "Directory where Nix will store state."}; - const PathSetting logDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, + const PathSetting logDir{(StoreConfig*) this, + rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store log files."}; - const PathSetting realStoreDir{(StoreConfig*) this, false, - rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + const PathSetting realStoreDir{(StoreConfig*) this, + rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7fb312c37..e69460e6c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -190,7 +190,11 @@ LocalStore::LocalStore(const Params & params) /* Create missing state directories if they don't already exist. */ createDirs(realStoreDir); - makeStoreWritable(); + if (readOnly) { + experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); + } else { + makeStoreWritable(); + } createDirs(linksDir); Path profilesDir = stateDir + "/profiles"; createDirs(profilesDir); @@ -204,8 +208,10 @@ LocalStore::LocalStore(const Params & params) for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); - if (chmod(perUserDir.c_str(), 0755) == -1) - throw SysError("could not set permissions on '%s' to 755", perUserDir); + if (!readOnly) { + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } } /* Optionally, create directories and set permissions for a @@ -269,10 +275,12 @@ LocalStore::LocalStore(const Params & params) /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ - Path globalLockPath = dbDir + "/big-lock"; - globalLock = openLockFile(globalLockPath.c_str(), true); + if (!readOnly) { + Path globalLockPath = dbDir + "/big-lock"; + globalLock = openLockFile(globalLockPath.c_str(), true); + } - if (!lockFile(globalLock.get(), ltRead, false)) { + if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) { printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -280,6 +288,14 @@ LocalStore::LocalStore(const Params & params) /* Check the current database schema and if necessary do an upgrade. */ int curSchema = getSchema(); + if (readOnly && curSchema < nixSchemaVersion) { + debug("current schema version: %d", curSchema); + debug("supported schema version: %d", nixSchemaVersion); + throw Error(curSchema == 0 ? + "database does not exist, and cannot be created in read-only mode" : + "database schema needs migrating, but this cannot be done in read-only mode"); + } + if (curSchema > nixSchemaVersion) throw Error("current Nix store schema is version %1%, but I only support %2%", curSchema, nixSchemaVersion); @@ -344,7 +360,11 @@ LocalStore::LocalStore(const Params & params) else openDB(*state, false); if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + if (!readOnly) { + migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); + } else { + throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode"); + } } /* Prepare SQL statements. */ @@ -475,13 +495,20 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { - if (access(dbDir.c_str(), R_OK | W_OK)) + if (create && readOnly) { + throw Error("cannot create database while in read-only mode"); + } + + if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - state.db = SQLite(dbPath, create); + auto openMode = readOnly ? SQLiteOpenMode::Immutable + : create ? SQLiteOpenMode::Normal + : SQLiteOpenMode::NoCreate; + state.db = SQLite(dbPath, openMode); #ifdef __CYGWIN__ /* The cygwin version of sqlite3 has a patch which calls diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 55add18dd..8a3b0b43f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -46,6 +46,23 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig "require-sigs", "Whether store paths copied into this store should have a trusted signature."}; + Setting<bool> readOnly{(StoreConfig*) this, + false, + "read-only", + R"( + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + )"}; + const std::string name() override { return "Local Store"; } std::string doc() override; @@ -240,8 +257,6 @@ public: void vacuumDB(); - void repairPath(const StorePath & path) override; - void addSignatures(const StorePath & storePath, const StringSet & sigs) override; /** @@ -271,6 +286,10 @@ public: private: + /** + * Retrieve the current version of the database schema. + * If the database does not exist yet, the version returned will be 0. + */ int getSchema(); void openDB(State & state, bool create); diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 53fe04704..626a22480 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -80,4 +80,15 @@ std::map<StorePath, StorePath> makeContentAddressed( return remappings; } +StorePath makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePath & fromPath) +{ + auto remappings = makeContentAddressed(srcStore, dstStore, StorePathSet { fromPath }); + auto i = remappings.find(fromPath); + assert(i != remappings.end()); + return i->second; +} + } diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh index 2ce6ec7bc..60bb2b477 100644 --- a/src/libstore/make-content-addressed.hh +++ b/src/libstore/make-content-addressed.hh @@ -5,9 +5,20 @@ namespace nix { +/** Rewrite a closure of store paths to be completely content addressed. + */ std::map<StorePath, StorePath> makeContentAddressed( Store & srcStore, Store & dstStore, - const StorePathSet & storePaths); + const StorePathSet & rootPaths); + +/** Rewrite a closure of a store path to be completely content addressed. + * + * This is a convenience function for the case where you only have one root path. + */ +StorePath makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePath & rootPath); } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index e60d7abe0..981bbfb14 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,5 +1,7 @@ #include "path-info.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" +#include "store-api.hh" namespace nix { @@ -131,7 +133,8 @@ ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned auto narHash = Hash::parseAny(readString(source), htSHA256); ValidPathInfo info(path, narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = worker_proto::read(store, source, Phantom<StorePathSet> {}); + info.references = WorkerProto::Serialise<StorePathSet>::read(store, + WorkerProto::ReadConn { .from = source }); source >> info.registrationTime >> info.narSize; if (format >= 16) { source >> info.ultimate; @@ -152,7 +155,9 @@ void ValidPathInfo::write( sink << store.printStorePath(path); sink << (deriver ? store.printStorePath(*deriver) : "") << narHash.to_string(Base16, false); - worker_proto::write(store, sink, references); + WorkerProto::write(store, + WorkerProto::WriteConn { .to = sink }, + references); sink << registrationTime << narSize; if (format >= 16) { sink << ultimate diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc new file mode 100644 index 000000000..33cf66ce3 --- /dev/null +++ b/src/libstore/path-references.cc @@ -0,0 +1,73 @@ +#include "path-references.hh" +#include "hash.hh" +#include "util.hh" +#include "archive.hh" + +#include <map> +#include <cstdlib> +#include <mutex> +#include <algorithm> + + +namespace nix { + + +PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap) + : RefScanSink(std::move(hashes)) + , backMap(std::move(backMap)) +{ } + +PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) +{ + StringSet hashes; + std::map<std::string, StorePath> backMap; + + for (auto & i : refs) { + std::string hashPart(i.hashPart()); + auto inserted = backMap.emplace(hashPart, i).second; + assert(inserted); + hashes.insert(hashPart); + } + + return PathRefScanSink(std::move(hashes), std::move(backMap)); +} + +StorePathSet PathRefScanSink::getResultPaths() +{ + /* Map the hashes found back to their store paths. */ + StorePathSet found; + for (auto & i : getResult()) { + auto j = backMap.find(i); + assert(j != backMap.end()); + found.insert(j->second); + } + + return found; +} + + +std::pair<StorePathSet, HashResult> scanForReferences( + const std::string & path, + const StorePathSet & refs) +{ + HashSink hashSink { htSHA256 }; + auto found = scanForReferences(hashSink, path, refs); + auto hash = hashSink.finish(); + return std::pair<StorePathSet, HashResult>(found, hash); +} + +StorePathSet scanForReferences( + Sink & toTee, + const Path & path, + const StorePathSet & refs) +{ + PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); + TeeSink sink { refsSink, toTee }; + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + return refsSink.getResultPaths(); +} + +} diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh new file mode 100644 index 000000000..7b44e3261 --- /dev/null +++ b/src/libstore/path-references.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "references.hh" +#include "path.hh" + +namespace nix { + +std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs); + +StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); + +class PathRefScanSink : public RefScanSink +{ + std::map<std::string, StorePath> backMap; + + PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap); + +public: + + static PathRefScanSink fromPaths(const StorePathSet & refs); + + StorePathSet getResultPaths(); +}; + +} diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 46be54281..552e83114 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -9,8 +9,8 @@ static void checkName(std::string_view path, std::string_view name) if (name.empty()) throw BadStorePath("store path '%s' has an empty name", path); if (name.size() > StorePath::MaxPathLen) - throw BadStorePath("store path '%s' has a name longer than '%d characters", - StorePath::MaxPathLen, path); + throw BadStorePath("store path '%s' has a name longer than %d characters", + path, StorePath::MaxPathLen); // See nameRegexStr for the definition for (auto c : name) if (!((c >= '0' && c <= '9') diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ba5c8583f..4e9955948 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -13,8 +13,10 @@ namespace nix { -/* Parse a generation name of the format - `<profilename>-<number>-link'. */ +/** + * Parse a generation name of the format + * `<profilename>-<number>-link'. + */ static std::optional<GenerationNumber> parseName(const std::string & profileName, const std::string & name) { if (name.substr(0, profileName.size() + 1) != profileName + "-") return {}; @@ -28,7 +30,6 @@ static std::optional<GenerationNumber> parseName(const std::string & profileName } - std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile) { Generations gens; @@ -61,15 +62,16 @@ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path pro } -static void makeName(const Path & profile, GenerationNumber num, - Path & outLink) +/** + * Create a generation name that can be parsed by `parseName()`. + */ +static Path makeName(const Path & profile, GenerationNumber num) { - Path prefix = fmt("%1%-%2%", profile, num); - outLink = prefix + "-link"; + return fmt("%s-%s-link", profile, num); } -Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath) +Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath) { /* The new generation number should be higher than old the previous ones. */ @@ -79,7 +81,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath) if (gens.size() > 0) { Generation last = gens.back(); - if (readLink(last.path) == store->printStorePath(outPath)) { + if (readLink(last.path) == store.printStorePath(outPath)) { /* We only create a new generation symlink if it differs from the last one. @@ -89,7 +91,7 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath) return last.path; } - num = gens.back().number; + num = last.number; } else { num = 0; } @@ -100,9 +102,8 @@ Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath) to the permanent roots (of which the GC would have a stale view). If we didn't do it this way, the GC might remove the user environment etc. we've just built. */ - Path generation; - makeName(profile, num + 1, generation); - store->addPermRoot(outPath, generation); + Path generation = makeName(profile, num + 1); + store.addPermRoot(outPath, generation); return generation; } @@ -117,12 +118,19 @@ static void removeFile(const Path & path) void deleteGeneration(const Path & profile, GenerationNumber gen) { - Path generation; - makeName(profile, gen, generation); + Path generation = makeName(profile, gen); removeFile(generation); } - +/** + * Delete a generation with dry-run mode. + * + * Like `deleteGeneration()` but: + * + * - We log what we are going to do. + * + * - We only actually delete if `dryRun` is false. + */ static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun) { if (dryRun) @@ -150,27 +158,36 @@ void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & } } +/** + * Advanced the iterator until the given predicate `cond` returns `true`. + */ +static inline void iterDropUntil(Generations & gens, auto && i, auto && cond) +{ + for (; i != gens.rend() && !cond(*i); ++i); +} + void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun) { + if (max == 0) + throw Error("Must keep at least one generation, otherwise the current one would be deleted"); + PathLocks lock; lockProfile(lock, profile); - bool fromCurGen = false; - auto [gens, curGen] = findGenerations(profile); - for (auto i = gens.rbegin(); i != gens.rend(); ++i) { - if (i->number == curGen) { - fromCurGen = true; - max--; - continue; - } - if (fromCurGen) { - if (max) { - max--; - continue; - } - deleteGeneration2(profile, i->number, dryRun); - } - } + auto [gens, _curGen] = findGenerations(profile); + auto curGen = _curGen; + + auto i = gens.rbegin(); + + // Find the current generation + iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; }); + + // Skip over `max` generations, preserving them + for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep); + + // Delete the rest + for (; i != gens.rend(); ++i) + deleteGeneration2(profile, i->number, dryRun); } void deleteOldGenerations(const Path & profile, bool dryRun) @@ -193,23 +210,33 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) auto [gens, curGen] = findGenerations(profile); - bool canDelete = false; - for (auto i = gens.rbegin(); i != gens.rend(); ++i) - if (canDelete) { - assert(i->creationTime < t); - if (i->number != curGen) - deleteGeneration2(profile, i->number, dryRun); - } else if (i->creationTime < t) { - /* We may now start deleting generations, but we don't - delete this generation yet, because this generation was - still the one that was active at the requested point in - time. */ - canDelete = true; - } + auto i = gens.rbegin(); + + // Predicate that the generation is older than the given time. + auto older = [&](auto & g) { return g.creationTime < t; }; + + // Find the first older generation, if one exists + iterDropUntil(gens, i, older); + + /* Take the previous generation + + We don't want delete this one yet because it + existed at the requested point in time, and + we want to be able to roll back to it. */ + if (i != gens.rend()) ++i; + + // Delete all previous generations (unless current). + for (; i != gens.rend(); ++i) { + /* Creating date and generations should be monotonic, so lower + numbered derivations should also be older. */ + assert(older(*i)); + if (i->number != curGen) + deleteGeneration2(profile, i->number, dryRun); + } } -void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun) +time_t parseOlderThanTimeSpec(std::string_view timeSpec) { if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd') throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec); @@ -221,9 +248,7 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, if (!days || *days < 1) throw UsageError("invalid number of days specifier '%1%'", timeSpec); - time_t oldTime = curTime - *days * 24 * 3600; - - deleteGenerationsOlderThan(profile, oldTime, dryRun); + return curTime - *days * 24 * 3600; } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 4e1f42e83..193c0bf21 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -1,7 +1,11 @@ #pragma once -///@file +/** + * @file Implementation of Profiles. + * + * See the manual for additional information. + */ - #include "types.hh" +#include "types.hh" #include "pathlocks.hh" #include <time.h> @@ -12,41 +16,166 @@ namespace nix { class StorePath; +/** + * A positive number identifying a generation for a given profile. + * + * Generation numbers are assigned sequentially. Each new generation is + * assigned 1 + the current highest generation number. + */ typedef uint64_t GenerationNumber; +/** + * A generation is a revision of a profile. + * + * Each generation is a mapping (key-value pair) from an identifier + * (`number`) to a store object (specified by `path`). + */ struct Generation { + /** + * The number of a generation is its unique identifier within the + * profile. + */ GenerationNumber number; + /** + * The store path identifies the store object that is the contents + * of the generation. + * + * These store paths / objects are not unique to the generation + * within a profile. Nix tries to ensure successive generations have + * distinct contents to avoid bloat, but nothing stops two + * non-adjacent generations from having the same contents. + * + * @todo Use `StorePath` instead of `Path`? + */ Path path; + + /** + * When the generation was created. This is extra metadata about the + * generation used to make garbage collecting old generations more + * convenient. + */ time_t creationTime; }; +/** + * All the generations of a profile + */ typedef std::list<Generation> Generations; /** - * Returns the list of currently present generations for the specified - * profile, sorted by generation number. Also returns the number of - * the current generation. + * Find all generations for the given profile. + * + * @param profile A profile specified by its name and location combined + * into a path. E.g. if "foo" is the name of the profile, and "/bar/baz" + * is the directory it is in, then the path "/bar/baz/foo" would be the + * argument for this parameter. + * + * @return The pair of: + * + * - The list of currently present generations for the specified profile, + * sorted by ascending generation number. + * + * - The number of the current/active generation. + * + * Note that the current/active generation need not be the latest one. */ std::pair<Generations, std::optional<GenerationNumber>> findGenerations(Path profile); class LocalFSStore; -Path createGeneration(ref<LocalFSStore> store, Path profile, StorePath outPath); +/** + * Create a new generation of the given profile + * + * If the previous generation (not the currently active one!) has a + * distinct store object, a fresh generation number is mapped to the + * given store object, referenced by path. Otherwise, the previous + * generation is assumed. + * + * The behavior of reusing existing generations like this makes this + * procedure idempotent. It also avoids clutter. + */ +Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath); +/** + * Unconditionally delete a generation + * + * @param profile A profile specified by its name and location combined into a path. + * + * @param gen The generation number specifying exactly which generation + * to delete. + * + * Because there is no check of whether the generation to delete is + * active, this is somewhat unsafe. + * + * @todo Should we expose this at all? + */ void deleteGeneration(const Path & profile, GenerationNumber gen); +/** + * Delete the given set of generations. + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param gensToDelete The generations to delete, specified by a set of + * numbers. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + * + * Trying to delete the currently active generation will fail, and cause + * no generations to be deleted. + */ void deleteGenerations(const Path & profile, const std::set<GenerationNumber> & gensToDelete, bool dryRun); +/** + * Delete generations older than `max` passed the current generation. + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param max How many generations to keep up to the current one. Must + * be at least 1 so we don't delete the current one. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + */ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun); +/** + * Delete all generations other than the current one + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + */ void deleteOldGenerations(const Path & profile, bool dryRun); +/** + * Delete generations older than `t`, except for the most recent one + * older than `t`. + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + */ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun); -void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun); +/** + * Parse a temp spec intended for `deleteGenerationsOlderThan()`. + * + * Throws an exception if `timeSpec` fails to parse. + */ +time_t parseOlderThanTimeSpec(std::string_view timeSpec); +/** + * Smaller wrapper around `replaceSymlink` for replacing the current + * generation of a profile. Does not enforce proper structure. + * + * @todo Always use `switchGeneration()` instead, and delete this. + */ void switchLink(Path link, Path target); /** diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh new file mode 100644 index 000000000..d32d91a60 --- /dev/null +++ b/src/libstore/remote-store-connection.hh @@ -0,0 +1,97 @@ +#include "remote-store.hh" +#include "worker-protocol.hh" + +namespace nix { + +/** + * Bidirectional connection (send and receive) used by the Remote Store + * implementation. + * + * Contains `Source` and `Sink` for actual communication, along with + * other information learned when negotiating the connection. + */ +struct RemoteStore::Connection +{ + /** + * Send with this. + */ + FdSink to; + + /** + * Receive with this. + */ + FdSource from; + + /** + * Worker protocol version used for the connection. + * + * Despite its name, I think it is actually the maximum version both + * sides support. (If the maximum doesn't exist, we would fail to + * establish a connection and produce a value of this type.) + */ + unsigned int daemonVersion; + + /** + * Whether the remote side trusts us or not. + * + * 3 values: "yes", "no", or `std::nullopt` for "unknown". + * + * Note that the "remote side" might not be just the end daemon, but + * also an intermediary forwarder that can make its own trusting + * decisions. This would be the intersection of all their trust + * decisions, since it takes only one link in the chain to start + * denying operations. + */ + std::optional<TrustedFlag> remoteTrustsUs; + + /** + * The version of the Nix daemon that is processing our requests. + * + * Do note, it may or may not communicating with another daemon, + * rather than being an "end" `LocalStore` or similar. + */ + std::optional<std::string> daemonNixVersion; + + /** + * Time this connection was established. + */ + std::chrono::time_point<std::chrono::steady_clock> startTime; + + /** + * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the + * factored out worker protocol searlizers with a + * `RemoteStore::Connection`. + * + * The worker protocol connection types are unidirectional, unlike + * this type. + */ + operator WorkerProto::ReadConn () + { + return WorkerProto::ReadConn { + .from = from, + }; + } + + /** + * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the + * factored out worker protocol searlizers with a + * `RemoteStore::Connection`. + * + * The worker protocol connection types are unidirectional, unlike + * this type. + */ + operator WorkerProto::WriteConn () + { + return WorkerProto::WriteConn { + .to = to, + }; + } + + virtual ~Connection(); + + virtual void closeWrite() = 0; + + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); +}; + +} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0ed17a6ce..1e2104e1f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -5,7 +5,9 @@ #include "remote-fs-accessor.hh" #include "build-result.hh" #include "remote-store.hh" +#include "remote-store-connection.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "archive.hh" #include "globals.hh" #include "derivations.hh" @@ -18,189 +20,6 @@ namespace nix { -namespace worker_proto { - -std::string read(const Store & store, Source & from, Phantom<std::string> _) -{ - return readString(from); -} - -void write(const Store & store, Sink & out, const std::string & str) -{ - out << str; -} - - -StorePath read(const Store & store, Source & from, Phantom<StorePath> _) -{ - return store.parseStorePath(readString(from)); -} - -void write(const Store & store, Sink & out, const StorePath & storePath) -{ - out << store.printStorePath(storePath); -} - - -std::optional<TrustedFlag> read(const Store & store, Source & from, Phantom<std::optional<TrustedFlag>> _) -{ - auto temp = readNum<uint8_t>(from); - switch (temp) { - case 0: - return std::nullopt; - case 1: - return { Trusted }; - case 2: - return { NotTrusted }; - default: - throw Error("Invalid trusted status from remote"); - } -} - -void write(const Store & store, Sink & out, const std::optional<TrustedFlag> & optTrusted) -{ - if (!optTrusted) - out << (uint8_t)0; - else { - switch (*optTrusted) { - case Trusted: - out << (uint8_t)1; - break; - case NotTrusted: - out << (uint8_t)2; - break; - default: - assert(false); - }; - } -} - - -ContentAddress read(const Store & store, Source & from, Phantom<ContentAddress> _) -{ - return ContentAddress::parse(readString(from)); -} - -void write(const Store & store, Sink & out, const ContentAddress & ca) -{ - out << renderContentAddress(ca); -} - - -DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _) -{ - auto s = readString(from); - return DerivedPath::parseLegacy(store, s); -} - -void write(const Store & store, Sink & out, const DerivedPath & req) -{ - out << req.to_string_legacy(store); -} - - -Realisation read(const Store & store, Source & from, Phantom<Realisation> _) -{ - std::string rawInput = readString(from); - return Realisation::fromJSON( - nlohmann::json::parse(rawInput), - "remote-protocol" - ); -} - -void write(const Store & store, Sink & out, const Realisation & realisation) -{ - out << realisation.toJSON().dump(); -} - - -DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _) -{ - return DrvOutput::parse(readString(from)); -} - -void write(const Store & store, Sink & out, const DrvOutput & drvOutput) -{ - out << drvOutput.to_string(); -} - - -KeyedBuildResult read(const Store & store, Source & from, Phantom<KeyedBuildResult> _) -{ - auto path = worker_proto::read(store, from, Phantom<DerivedPath> {}); - auto br = worker_proto::read(store, from, Phantom<BuildResult> {}); - return KeyedBuildResult { - std::move(br), - /* .path = */ std::move(path), - }; -} - -void write(const Store & store, Sink & to, const KeyedBuildResult & res) -{ - worker_proto::write(store, to, res.path); - worker_proto::write(store, to, static_cast<const BuildResult &>(res)); -} - - -BuildResult read(const Store & store, Source & from, Phantom<BuildResult> _) -{ - BuildResult res; - res.status = (BuildResult::Status) readInt(from); - from - >> res.errorMsg - >> res.timesBuilt - >> res.isNonDeterministic - >> res.startTime - >> res.stopTime; - auto builtOutputs = worker_proto::read(store, from, Phantom<DrvOutputs> {}); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - return res; -} - -void write(const Store & store, Sink & to, const BuildResult & res) -{ - to - << res.status - << res.errorMsg - << res.timesBuilt - << res.isNonDeterministic - << res.startTime - << res.stopTime; - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - worker_proto::write(store, to, builtOutputs); -} - - -std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _) -{ - auto s = readString(from); - return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s); -} - -void write(const Store & store, Sink & out, const std::optional<StorePath> & storePathOpt) -{ - out << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); -} - - -std::optional<ContentAddress> read(const Store & store, Source & from, Phantom<std::optional<ContentAddress>> _) -{ - return ContentAddress::parseOpt(readString(from)); -} - -void write(const Store & store, Sink & out, const std::optional<ContentAddress> & caOpt) -{ - out << (caOpt ? renderContentAddress(*caOpt) : ""); -} - -} - - /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : RemoteStoreConfig(params) @@ -283,7 +102,7 @@ void RemoteStore::initConnection(Connection & conn) } if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { - conn.remoteTrustsUs = worker_proto::read(*this, conn.from, Phantom<std::optional<TrustedFlag>> {}); + conn.remoteTrustsUs = WorkerProto::Serialise<std::optional<TrustedFlag>>::read(*this, conn); } else { // We don't know the answer; protocol to old. conn.remoteTrustsUs = std::nullopt; @@ -302,7 +121,7 @@ void RemoteStore::initConnection(Connection & conn) void RemoteStore::setOptions(Connection & conn) { - conn.to << wopSetOptions + conn.to << WorkerProto::Op::SetOptions << settings.keepFailed << settings.keepGoing << settings.tryFallback @@ -367,6 +186,7 @@ struct ConnectionHandle } RemoteStore::Connection * operator -> () { return &*handle; } + RemoteStore::Connection & operator * () { return *handle; } void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) { @@ -394,7 +214,7 @@ void RemoteStore::setOptions() bool RemoteStore::isValidPathUncached(const StorePath & path) { auto conn(getConnection()); - conn->to << wopIsValidPath << printStorePath(path); + conn->to << WorkerProto::Op::IsValidPath << printStorePath(path); conn.processStderr(); return readInt(conn->from); } @@ -409,13 +229,13 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute if (isValidPath(i)) res.insert(i); return res; } else { - conn->to << wopQueryValidPaths; - worker_proto::write(*this, conn->to, paths); + conn->to << WorkerProto::Op::QueryValidPaths; + WorkerProto::write(*this, *conn, paths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { conn->to << (settings.buildersUseSubstitutes ? 1 : 0); } conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + return WorkerProto::Serialise<StorePathSet>::read(*this, *conn); } } @@ -423,9 +243,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute StorePathSet RemoteStore::queryAllValidPaths() { auto conn(getConnection()); - conn->to << wopQueryAllValidPaths; + conn->to << WorkerProto::Op::QueryAllValidPaths; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + return WorkerProto::Serialise<StorePathSet>::read(*this, *conn); } @@ -435,16 +255,16 @@ StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { StorePathSet res; for (auto & i : paths) { - conn->to << wopHasSubstitutes << printStorePath(i); + conn->to << WorkerProto::Op::HasSubstitutes << printStorePath(i); conn.processStderr(); if (readInt(conn->from)) res.insert(i); } return res; } else { - conn->to << wopQuerySubstitutablePaths; - worker_proto::write(*this, conn->to, paths); + conn->to << WorkerProto::Op::QuerySubstitutablePaths; + WorkerProto::write(*this, *conn, paths); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + return WorkerProto::Serialise<StorePathSet>::read(*this, *conn); } } @@ -459,14 +279,14 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S for (auto & i : pathsMap) { SubstitutablePathInfo info; - conn->to << wopQuerySubstitutablePathInfo << printStorePath(i.first); + conn->to << WorkerProto::Op::QuerySubstitutablePathInfo << printStorePath(i.first); conn.processStderr(); unsigned int reply = readInt(conn->from); if (reply == 0) continue; auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info.references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); infos.insert_or_assign(i.first, std::move(info)); @@ -474,14 +294,14 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } else { - conn->to << wopQuerySubstitutablePathInfos; + conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); - worker_proto::write(*this, conn->to, paths); + WorkerProto::write(*this, *conn, paths); } else - worker_proto::write(*this, conn->to, pathsMap); + WorkerProto::write(*this, *conn, pathsMap); conn.processStderr(); size_t count = readNum<size_t>(conn->from); for (size_t n = 0; n < count; n++) { @@ -489,7 +309,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto deriver = readString(conn->from); if (deriver != "") info.deriver = parseStorePath(deriver); - info.references = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + info.references = WorkerProto::Serialise<StorePathSet>::read(*this, *conn); info.downloadSize = readLongLong(conn->from); info.narSize = readLongLong(conn->from); } @@ -505,7 +325,7 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, std::shared_ptr<const ValidPathInfo> info; { auto conn(getConnection()); - conn->to << wopQueryPathInfo << printStorePath(path); + conn->to << WorkerProto::Op::QueryPathInfo << printStorePath(path); try { conn.processStderr(); } catch (Error & e) { @@ -530,9 +350,9 @@ void RemoteStore::queryReferrers(const StorePath & path, StorePathSet & referrers) { auto conn(getConnection()); - conn->to << wopQueryReferrers << printStorePath(path); + conn->to << WorkerProto::Op::QueryReferrers << printStorePath(path); conn.processStderr(); - for (auto & i : worker_proto::read(*this, conn->from, Phantom<StorePathSet> {})) + for (auto & i : WorkerProto::Serialise<StorePathSet>::read(*this, *conn)) referrers.insert(i); } @@ -540,9 +360,9 @@ void RemoteStore::queryReferrers(const StorePath & path, StorePathSet RemoteStore::queryValidDerivers(const StorePath & path) { auto conn(getConnection()); - conn->to << wopQueryValidDerivers << printStorePath(path); + conn->to << WorkerProto::Op::QueryValidDerivers << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + return WorkerProto::Serialise<StorePathSet>::read(*this, *conn); } @@ -552,9 +372,9 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) return Store::queryDerivationOutputs(path); } auto conn(getConnection()); - conn->to << wopQueryDerivationOutputs << printStorePath(path); + conn->to << WorkerProto::Op::QueryDerivationOutputs << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + return WorkerProto::Serialise<StorePathSet>::read(*this, *conn); } @@ -562,9 +382,9 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat { if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { auto conn(getConnection()); - conn->to << wopQueryDerivationOutputMap << printStorePath(path); + conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom<std::map<std::string, std::optional<StorePath>>> {}); + return WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(*this, *conn); } else { // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an @@ -585,7 +405,7 @@ std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivat std::optional<StorePath> RemoteStore::queryPathFromHashPart(const std::string & hashPart) { auto conn(getConnection()); - conn->to << wopQueryPathFromHashPart << hashPart; + conn->to << WorkerProto::Op::QueryPathFromHashPart << hashPart; conn.processStderr(); Path path = readString(conn->from); if (path.empty()) return {}; @@ -607,10 +427,10 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { conn->to - << wopAddToStore + << WorkerProto::Op::AddToStore << name << caMethod.render(hashType); - worker_proto::write(*this, conn->to, references); + WorkerProto::write(*this, *conn, references); conn->to << repair; // The dump source may invoke the store, so we need to make some room. @@ -634,13 +454,13 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", name, printHashType(hashType)); std::string s = dump.drain(); - conn->to << wopAddTextToStore << name << s; - worker_proto::write(*this, conn->to, references); + conn->to << WorkerProto::Op::AddTextToStore << name << s; + WorkerProto::write(*this, *conn, references); conn.processStderr(); }, [&](const FileIngestionMethod & fim) -> void { conn->to - << wopAddToStore + << WorkerProto::Op::AddToStore << name << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << (fim == FileIngestionMethod::Recursive ? 1 : 0) @@ -692,7 +512,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { - conn->to << wopImportPaths; + conn->to << WorkerProto::Op::ImportPaths; auto source2 = sinkToSource([&](Sink & sink) { sink << 1 // == path follows @@ -701,7 +521,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, sink << exportMagic << printStorePath(info.path); - worker_proto::write(*this, sink, info.references); + WorkerProto::write(*this, *conn, info.references); sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature @@ -711,16 +531,16 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn.processStderr(0, source2.get()); - auto importedPaths = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + auto importedPaths = WorkerProto::Serialise<StorePathSet>::read(*this, *conn); assert(importedPaths.size() <= 1); } else { - conn->to << wopAddToStoreNar + conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") << info.narHash.to_string(Base16, false); - worker_proto::write(*this, conn->to, info.references); + WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; @@ -764,7 +584,7 @@ void RemoteStore::addMultipleToStore( if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) { auto conn(getConnection()); conn->to - << wopAddMultipleToStore + << WorkerProto::Op::AddMultipleToStore << repair << !checkSigs; conn.withFramedSink([&](Sink & sink) { @@ -788,12 +608,12 @@ StorePath RemoteStore::addTextToStore( void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); - conn->to << wopRegisterDrvOutput; + conn->to << WorkerProto::Op::RegisterDrvOutput; if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string()); } else { - worker_proto::write(*this, conn->to, info); + WorkerProto::write(*this, *conn, info); } conn.processStderr(); } @@ -809,20 +629,20 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, return callback(nullptr); } - conn->to << wopQueryRealisation; + conn->to << WorkerProto::Op::QueryRealisation; conn->to << id.to_string(); conn.processStderr(); auto real = [&]() -> std::shared_ptr<const Realisation> { if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { - auto outPaths = worker_proto::read( - *this, conn->from, Phantom<std::set<StorePath>> {}); + auto outPaths = WorkerProto::Serialise<std::set<StorePath>>::read( + *this, *conn); if (outPaths.empty()) return nullptr; return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() }); } else { - auto realisations = worker_proto::read( - *this, conn->from, Phantom<std::set<Realisation>> {}); + auto realisations = WorkerProto::Serialise<std::set<Realisation>>::read( + *this, *conn); if (realisations.empty()) return nullptr; return std::make_shared<const Realisation>(*realisations.begin()); @@ -833,10 +653,10 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, } catch (...) { return callback.rethrow(); } } -static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs) +static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & conn, const std::vector<DerivedPath> & reqs) { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 30) { - worker_proto::write(store, conn->to, reqs); + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 30) { + WorkerProto::write(store, conn, reqs); } else { Strings ss; for (auto & p : reqs) { @@ -848,12 +668,12 @@ static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, cons [&](const StorePath & drvPath) { throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn->daemonVersion), - GET_PROTOCOL_MINOR(conn->daemonVersion)); + GET_PROTOCOL_MAJOR(conn.daemonVersion), + GET_PROTOCOL_MINOR(conn.daemonVersion)); }, }, sOrDrvPath); } - conn->to << ss; + conn.to << ss; } } @@ -877,9 +697,9 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod copyDrvsFromEvalStore(drvPaths, evalStore); auto conn(getConnection()); - conn->to << wopBuildPaths; + conn->to << WorkerProto::Op::BuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeDerivedPaths(*this, conn, drvPaths); + writeDerivedPaths(*this, *conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -902,11 +722,11 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults( auto & conn = *conn_; if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { - conn->to << wopBuildPathsWithResults; - writeDerivedPaths(*this, conn, paths); + conn->to << WorkerProto::Op::BuildPathsWithResults; + writeDerivedPaths(*this, *conn, paths); conn->to << buildMode; conn.processStderr(); - return worker_proto::read(*this, conn->from, Phantom<std::vector<KeyedBuildResult>> {}); + return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn); } else { // Avoid deadlock. conn_.reset(); @@ -978,7 +798,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD BuildMode buildMode) { auto conn(getConnection()); - conn->to << wopBuildDerivation << printStorePath(drvPath); + conn->to << WorkerProto::Op::BuildDerivation << printStorePath(drvPath); writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); @@ -989,7 +809,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; } if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {}); + auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, *conn); for (auto && [output, realisation] : builtOutputs) res.builtOutputs.insert_or_assign( std::move(output.outputName), @@ -1002,7 +822,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD void RemoteStore::ensurePath(const StorePath & path) { auto conn(getConnection()); - conn->to << wopEnsurePath << printStorePath(path); + conn->to << WorkerProto::Op::EnsurePath << printStorePath(path); conn.processStderr(); readInt(conn->from); } @@ -1011,7 +831,7 @@ void RemoteStore::ensurePath(const StorePath & path) void RemoteStore::addTempRoot(const StorePath & path) { auto conn(getConnection()); - conn->to << wopAddTempRoot << printStorePath(path); + conn->to << WorkerProto::Op::AddTempRoot << printStorePath(path); conn.processStderr(); readInt(conn->from); } @@ -1020,7 +840,7 @@ void RemoteStore::addTempRoot(const StorePath & path) void RemoteStore::addIndirectRoot(const Path & path) { auto conn(getConnection()); - conn->to << wopAddIndirectRoot << path; + conn->to << WorkerProto::Op::AddIndirectRoot << path; conn.processStderr(); readInt(conn->from); } @@ -1029,7 +849,7 @@ void RemoteStore::addIndirectRoot(const Path & path) Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); - conn->to << wopFindRoots; + conn->to << WorkerProto::Op::FindRoots; conn.processStderr(); size_t count = readNum<size_t>(conn->from); Roots result; @@ -1047,8 +867,8 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) auto conn(getConnection()); conn->to - << wopCollectGarbage << options.action; - worker_proto::write(*this, conn->to, options.pathsToDelete); + << WorkerProto::Op::CollectGarbage << options.action; + WorkerProto::write(*this, *conn, options.pathsToDelete); conn->to << options.ignoreLiveness << options.maxFreed /* removed options */ @@ -1070,7 +890,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) void RemoteStore::optimiseStore() { auto conn(getConnection()); - conn->to << wopOptimiseStore; + conn->to << WorkerProto::Op::OptimiseStore; conn.processStderr(); readInt(conn->from); } @@ -1079,7 +899,7 @@ void RemoteStore::optimiseStore() bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { auto conn(getConnection()); - conn->to << wopVerifyStore << checkContents << repair; + conn->to << WorkerProto::Op::VerifyStore << checkContents << repair; conn.processStderr(); return readInt(conn->from); } @@ -1088,7 +908,7 @@ bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { auto conn(getConnection()); - conn->to << wopAddSignatures << printStorePath(storePath) << sigs; + conn->to << WorkerProto::Op::AddSignatures << printStorePath(storePath) << sigs; conn.processStderr(); readInt(conn->from); } @@ -1104,12 +924,12 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets, // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; - conn->to << wopQueryMissing; - writeDerivedPaths(*this, conn, targets); + conn->to << WorkerProto::Op::QueryMissing; + writeDerivedPaths(*this, *conn, targets); conn.processStderr(); - willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); - willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); - unknown = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {}); + willBuild = WorkerProto::Serialise<StorePathSet>::read(*this, *conn); + willSubstitute = WorkerProto::Serialise<StorePathSet>::read(*this, *conn); + unknown = WorkerProto::Serialise<StorePathSet>::read(*this, *conn); conn->from >> downloadSize >> narSize; return; } @@ -1123,7 +943,7 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets, void RemoteStore::addBuildLog(const StorePath & drvPath, std::string_view log) { auto conn(getConnection()); - conn->to << wopAddBuildLog << drvPath.to_string(); + conn->to << WorkerProto::Op::AddBuildLog << drvPath.to_string(); StringSource source(log); conn.withFramedSink([&](Sink & sink) { source.drainInto(sink); @@ -1175,7 +995,7 @@ RemoteStore::Connection::~Connection() void RemoteStore::narFromPath(const StorePath & path, Sink & sink) { auto conn(connections->get()); - conn->to << wopNarFromPath << printStorePath(path); + conn->to << WorkerProto::Op::NarFromPath << printStorePath(path); conn->processStderr(); copyNAR(conn->from, sink); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 82e4656ab..cb7a71acf 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -137,6 +137,17 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; + /** + * The default instance would schedule the work on the client side, but + * for consistency with `buildPaths` and `buildDerivation` it should happen + * on the remote side. + * + * We make this fail for now so we can add implement this properly later + * without it being a breaking change. + */ + void repairPath(const StorePath & path) override + { unsupported("repairPath"); } + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; void queryMissing(const std::vector<DerivedPath> & targets, @@ -155,21 +166,7 @@ public: void flushBadConnections(); - struct Connection - { - FdSink to; - FdSource from; - unsigned int daemonVersion; - std::optional<TrustedFlag> remoteTrustsUs; - std::optional<std::string> daemonNixVersion; - std::chrono::time_point<std::chrono::steady_clock> startTime; - - virtual ~Connection(); - - virtual void closeWrite() = 0; - - std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); - }; + struct Connection; ref<Connection> openConnectionWrapper(); diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 553fd3a09..7e43b3969 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -10,16 +10,52 @@ namespace nix { #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -typedef enum { - cmdQueryValidPaths = 1, - cmdQueryPathInfos = 2, - cmdDumpStorePath = 3, - cmdImportPaths = 4, - cmdExportPaths = 5, - cmdBuildPaths = 6, - cmdQueryClosure = 7, - cmdBuildDerivation = 8, - cmdAddToStoreNar = 9, -} ServeCommand; +/** + * The "serve protocol", used by ssh:// stores. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. + */ +struct ServeProto +{ + /** + * Enumeration of all the request types for the protocol. + */ + enum struct Command : uint64_t; +}; + +enum struct ServeProto::Command : uint64_t +{ + QueryValidPaths = 1, + QueryPathInfos = 2, + DumpStorePath = 3, + ImportPaths = 4, + ExportPaths = 5, + BuildPaths = 6, + QueryClosure = 7, + BuildDerivation = 8, + AddToStoreNar = 9, +}; + +/** + * Convenience for sending operation codes. + * + * @todo Switch to using `ServeProto::Serialize` instead probably. But + * this was not done at this time so there would be less churn. + */ +inline Sink & operator << (Sink & sink, ServeProto::Command op) +{ + return sink << (uint64_t) op; +} + +/** + * Convenience for debugging. + * + * @todo Perhaps render known opcodes more nicely. + */ +inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) +{ + return s << (uint64_t) op; +} } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index df334c23c..7c8decb74 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,6 +1,7 @@ #include "sqlite.hh" #include "globals.hh" #include "util.hh" +#include "url.hh" #include <sqlite3.h> @@ -50,15 +51,17 @@ static void traceSQL(void * x, const char * sql) notice("SQL<[%1%]>", sql); }; -SQLite::SQLite(const Path & path, bool create) +SQLite::SQLite(const Path & path, SQLiteOpenMode mode) { // useSQLiteWAL also indicates what virtual file system we need. Using // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem // for Linux (WSL) where useSQLiteWAL should be false by default. const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; - int flags = SQLITE_OPEN_READWRITE; - if (create) flags |= SQLITE_OPEN_CREATE; - int ret = sqlite3_open_v2(path.c_str(), &db, flags, vfs); + bool immutable = mode == SQLiteOpenMode::Immutable; + int flags = immutable ? SQLITE_OPEN_READONLY : SQLITE_OPEN_READWRITE; + if (mode == SQLiteOpenMode::Normal) flags |= SQLITE_OPEN_CREATE; + auto uri = "file:" + percentEncode(path) + "?immutable=" + (immutable ? "1" : "0"); + int ret = sqlite3_open_v2(uri.c_str(), &db, SQLITE_OPEN_URI | flags, vfs); if (ret != SQLITE_OK) { const char * err = sqlite3_errstr(ret); throw Error("cannot open SQLite database '%s': %s", path, err); diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 6e14852cb..0c08267f7 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -11,6 +11,27 @@ struct sqlite3_stmt; namespace nix { +enum class SQLiteOpenMode { + /** + * Open the database in read-write mode. + * If the database does not exist, it will be created. + */ + Normal, + /** + * Open the database in read-write mode. + * Fails with an error if the database does not exist. + */ + NoCreate, + /** + * Open the database in immutable mode. + * In addition to the database being read-only, + * no wal or journal files will be created by sqlite. + * Use this mode if the database is on a read-only filesystem. + * Fails with an error if the database does not exist. + */ + Immutable, +}; + /** * RAII wrapper to close a SQLite database automatically. */ @@ -18,7 +39,7 @@ struct SQLite { sqlite3 * db = 0; SQLite() { } - SQLite(const Path & path, bool create = true); + SQLite(const Path & path, SQLiteOpenMode mode = SQLiteOpenMode::Normal); SQLite(const SQLite & from) = delete; SQLite& operator = (const SQLite & from) = delete; SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 962221ad2..0200076c0 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,6 +1,7 @@ #include "ssh-store-config.hh" #include "store-api.hh" #include "remote-store.hh" +#include "remote-store-connection.hh" #include "remote-fs-accessor.hh" #include "archive.hh" #include "worker-protocol.hh" diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 6f6deda51..fae99d75b 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -41,6 +41,11 @@ void SSHMaster::addCommonSSHOpts(Strings & args) args.push_back("-oLocalCommand=echo started"); } +bool SSHMaster::isMasterRunning() { + auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true}); + return res.first == 0; +} + std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string & command) { Path socketPath = startMaster(); @@ -97,7 +102,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string // Wait for the SSH connection to be established, // So that we don't overwrite the password prompt with our progress bar. - if (!fakeSSH && !useMaster) { + if (!fakeSSH && !useMaster && !isMasterRunning()) { std::string reply; try { reply = readLine(out.readSide.get()); @@ -133,6 +138,8 @@ Path SSHMaster::startMaster() logger->pause(); Finally cleanup = [&]() { logger->resume(); }; + bool wasMasterRunning = isMasterRunning(); + state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -152,13 +159,15 @@ Path SSHMaster::startMaster() out.writeSide = -1; - std::string reply; - try { - reply = readLine(out.readSide.get()); - } catch (EndOfFile & e) { } + if (!wasMasterRunning) { + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } - if (reply != "started") - throw Error("failed to start SSH master connection to '%s'", host); + if (reply != "started") + throw Error("failed to start SSH master connection to '%s'", host); + } return state->socketPath; } diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index c86a8a986..94b952af9 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -28,6 +28,7 @@ private: Sync<State> state_; void addCommonSSHOpts(Strings & args); + bool isMasterRunning(); public: diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index bad610014..14a862eef 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -114,7 +114,7 @@ struct StoreConfig : public Config return ""; } - const PathSetting storeDir_{this, false, settings.nixStore, + const PathSetting storeDir_{this, settings.nixStore, "store", R"( Logical location of the Nix store, usually @@ -679,8 +679,7 @@ public: * Repair the contents of the given path by redownloading it using * a substituter (if available). */ - virtual void repairPath(const StorePath & path) - { unsupported("repairPath"); } + virtual void repairPath(const StorePath & path); /** * Add signatures to the specified store path. The signatures are diff --git a/src/libstore/tests/downstream-placeholder.cc b/src/libstore/tests/downstream-placeholder.cc new file mode 100644 index 000000000..ec3e1000f --- /dev/null +++ b/src/libstore/tests/downstream-placeholder.cc @@ -0,0 +1,33 @@ +#include <gtest/gtest.h> + +#include "downstream-placeholder.hh" + +namespace nix { + +TEST(DownstreamPlaceholder, unknownCaOutput) { + ASSERT_EQ( + DownstreamPlaceholder::unknownCaOutput( + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, + "out").render(), + "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz"); +} + +TEST(DownstreamPlaceholder, unknownDerivation) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + ASSERT_EQ( + DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::unknownCaOutput( + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" }, + "out"), + "out", + mockXpSettings).render(), + "/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w"); +} + +} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 0fb7c38e9..69dae2da5 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -13,6 +13,14 @@ namespace nix { +std::string UDSRemoteStoreConfig::doc() +{ + return + #include "uds-remote-store.md" + ; +} + + UDSRemoteStore::UDSRemoteStore(const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index bd1dcb67c..2bd6517fa 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -2,6 +2,7 @@ ///@file #include "remote-store.hh" +#include "remote-store-connection.hh" #include "local-fs-store.hh" namespace nix { @@ -17,12 +18,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon const std::string name() override { return "Local Daemon Store"; } - std::string doc() override - { - return - #include "uds-remote-store.md" - ; - } + std::string doc() override; }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh new file mode 100644 index 000000000..d3d2792ff --- /dev/null +++ b/src/libstore/worker-protocol-impl.hh @@ -0,0 +1,78 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "worker-protocol.hh" + +namespace nix { + +template<typename T> +std::vector<T> WorkerProto::Serialise<std::vector<T>>::read(const Store & store, WorkerProto::ReadConn conn) +{ + std::vector<T> resSet; + auto size = readNum<size_t>(conn.from); + while (size--) { + resSet.push_back(WorkerProto::Serialise<T>::read(store, conn)); + } + return resSet; +} + +template<typename T> +void WorkerProto::Serialise<std::vector<T>>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector<T> & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + WorkerProto::Serialise<T>::write(store, conn, key); + } +} + +template<typename T> +std::set<T> WorkerProto::Serialise<std::set<T>>::read(const Store & store, WorkerProto::ReadConn conn) +{ + std::set<T> resSet; + auto size = readNum<size_t>(conn.from); + while (size--) { + resSet.insert(WorkerProto::Serialise<T>::read(store, conn)); + } + return resSet; +} + +template<typename T> +void WorkerProto::Serialise<std::set<T>>::write(const Store & store, WorkerProto::WriteConn conn, const std::set<T> & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + WorkerProto::Serialise<T>::write(store, conn, key); + } +} + +template<typename K, typename V> +std::map<K, V> WorkerProto::Serialise<std::map<K, V>>::read(const Store & store, WorkerProto::ReadConn conn) +{ + std::map<K, V> resMap; + auto size = readNum<size_t>(conn.from); + while (size--) { + auto k = WorkerProto::Serialise<K>::read(store, conn); + auto v = WorkerProto::Serialise<V>::read(store, conn); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template<typename K, typename V> +void WorkerProto::Serialise<std::map<K, V>>::write(const Store & store, WorkerProto::WriteConn conn, const std::map<K, V> & resMap) +{ + conn.to << resMap.size(); + for (auto & i : resMap) { + WorkerProto::Serialise<K>::write(store, conn, i.first); + WorkerProto::Serialise<V>::write(store, conn, i.second); + } +} + +} diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc new file mode 100644 index 000000000..a23130743 --- /dev/null +++ b/src/libstore/worker-protocol.cc @@ -0,0 +1,193 @@ +#include "serialise.hh" +#include "util.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "worker-protocol.hh" +#include "worker-protocol-impl.hh" +#include "archive.hh" +#include "derivations.hh" + +#include <nlohmann/json.hpp> + +namespace nix { + +std::string WorkerProto::Serialise<std::string>::read(const Store & store, WorkerProto::ReadConn conn) +{ + return readString(conn.from); +} + +void WorkerProto::Serialise<std::string>::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str) +{ + conn.to << str; +} + + +StorePath WorkerProto::Serialise<StorePath>::read(const Store & store, WorkerProto::ReadConn conn) +{ + return store.parseStorePath(readString(conn.from)); +} + +void WorkerProto::Serialise<StorePath>::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath) +{ + conn.to << store.printStorePath(storePath); +} + + +std::optional<TrustedFlag> WorkerProto::Serialise<std::optional<TrustedFlag>>::read(const Store & store, WorkerProto::ReadConn conn) +{ + auto temp = readNum<uint8_t>(conn.from); + switch (temp) { + case 0: + return std::nullopt; + case 1: + return { Trusted }; + case 2: + return { NotTrusted }; + default: + throw Error("Invalid trusted status from remote"); + } +} + +void WorkerProto::Serialise<std::optional<TrustedFlag>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<TrustedFlag> & optTrusted) +{ + if (!optTrusted) + conn.to << (uint8_t)0; + else { + switch (*optTrusted) { + case Trusted: + conn.to << (uint8_t)1; + break; + case NotTrusted: + conn.to << (uint8_t)2; + break; + default: + assert(false); + }; + } +} + + +ContentAddress WorkerProto::Serialise<ContentAddress>::read(const Store & store, WorkerProto::ReadConn conn) +{ + return ContentAddress::parse(readString(conn.from)); +} + +void WorkerProto::Serialise<ContentAddress>::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca) +{ + conn.to << renderContentAddress(ca); +} + + +DerivedPath WorkerProto::Serialise<DerivedPath>::read(const Store & store, WorkerProto::ReadConn conn) +{ + auto s = readString(conn.from); + return DerivedPath::parseLegacy(store, s); +} + +void WorkerProto::Serialise<DerivedPath>::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) +{ + conn.to << req.to_string_legacy(store); +} + + +Realisation WorkerProto::Serialise<Realisation>::read(const Store & store, WorkerProto::ReadConn conn) +{ + std::string rawInput = readString(conn.from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} + +void WorkerProto::Serialise<Realisation>::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation) +{ + conn.to << realisation.toJSON().dump(); +} + + +DrvOutput WorkerProto::Serialise<DrvOutput>::read(const Store & store, WorkerProto::ReadConn conn) +{ + return DrvOutput::parse(readString(conn.from)); +} + +void WorkerProto::Serialise<DrvOutput>::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput) +{ + conn.to << drvOutput.to_string(); +} + + +KeyedBuildResult WorkerProto::Serialise<KeyedBuildResult>::read(const Store & store, WorkerProto::ReadConn conn) +{ + auto path = WorkerProto::Serialise<DerivedPath>::read(store, conn); + auto br = WorkerProto::Serialise<BuildResult>::read(store, conn); + return KeyedBuildResult { + std::move(br), + /* .path = */ std::move(path), + }; +} + +void WorkerProto::Serialise<KeyedBuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) +{ + WorkerProto::write(store, conn, res.path); + WorkerProto::write(store, conn, static_cast<const BuildResult &>(res)); +} + + +BuildResult WorkerProto::Serialise<BuildResult>::read(const Store & store, WorkerProto::ReadConn conn) +{ + BuildResult res; + res.status = (BuildResult::Status) readInt(conn.from); + conn.from + >> res.errorMsg + >> res.timesBuilt + >> res.isNonDeterministic + >> res.startTime + >> res.stopTime; + auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + return res; +} + +void WorkerProto::Serialise<BuildResult>::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res) +{ + conn.to + << res.status + << res.errorMsg + << res.timesBuilt + << res.isNonDeterministic + << res.startTime + << res.stopTime; + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + WorkerProto::write(store, conn, builtOutputs); +} + + +std::optional<StorePath> WorkerProto::Serialise<std::optional<StorePath>>::read(const Store & store, WorkerProto::ReadConn conn) +{ + auto s = readString(conn.from); + return s == "" ? std::optional<StorePath> {} : store.parseStorePath(s); +} + +void WorkerProto::Serialise<std::optional<StorePath>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<StorePath> & storePathOpt) +{ + conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional<ContentAddress> WorkerProto::Serialise<std::optional<ContentAddress>>::read(const Store & store, WorkerProto::ReadConn conn) +{ + return ContentAddress::parseOpt(readString(conn.from)); +} + +void WorkerProto::Serialise<std::optional<ContentAddress>>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional<ContentAddress> & caOpt) +{ + conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 34b2fc17b..ff762c924 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "store-api.hh" #include "serialise.hh" namespace nix { @@ -15,57 +14,6 @@ namespace nix { #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -/** - * Enumeration of all the request types for the "worker protocol", used - * by unix:// and ssh-ng:// stores. - */ -typedef enum { - wopIsValidPath = 1, - wopHasSubstitutes = 3, - wopQueryPathHash = 4, // obsolete - wopQueryReferences = 5, // obsolete - wopQueryReferrers = 6, - wopAddToStore = 7, - wopAddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use wopAddToStore - wopBuildPaths = 9, - wopEnsurePath = 10, - wopAddTempRoot = 11, - wopAddIndirectRoot = 12, - wopSyncWithGC = 13, - wopFindRoots = 14, - wopExportPath = 16, // obsolete - wopQueryDeriver = 18, // obsolete - wopSetOptions = 19, - wopCollectGarbage = 20, - wopQuerySubstitutablePathInfo = 21, - wopQueryDerivationOutputs = 22, // obsolete - wopQueryAllValidPaths = 23, - wopQueryFailedPaths = 24, - wopClearFailedPaths = 25, - wopQueryPathInfo = 26, - wopImportPaths = 27, // obsolete - wopQueryDerivationOutputNames = 28, // obsolete - wopQueryPathFromHashPart = 29, - wopQuerySubstitutablePathInfos = 30, - wopQueryValidPaths = 31, - wopQuerySubstitutablePaths = 32, - wopQueryValidDerivers = 33, - wopOptimiseStore = 34, - wopVerifyStore = 35, - wopBuildDerivation = 36, - wopAddSignatures = 37, - wopNarFromPath = 38, - wopAddToStoreNar = 39, - wopQueryMissing = 40, - wopQueryDerivationOutputMap = 41, - wopRegisterDrvOutput = 42, - wopQueryRealisation = 43, - wopAddMultipleToStore = 44, - wopAddBuildLog = 45, - wopBuildPathsWithResults = 46, -} WorkerOp; - - #define STDERR_NEXT 0x6f6c6d67 #define STDERR_READ 0x64617461 // data needed from source #define STDERR_WRITE 0x64617416 // data for sink @@ -79,45 +27,208 @@ typedef enum { class Store; struct Source; +// items being serialised +struct DerivedPath; +struct DrvOutput; +struct Realisation; +struct BuildResult; +struct KeyedBuildResult; +enum TrustedFlag : bool; + + /** - * Used to guide overloading + * The "worker protocol", used by unix:// and ssh-ng:// stores. * - * See https://en.cppreference.com/w/cpp/language/adl for the broader - * concept of what is going on here. + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. */ -template<typename T> -struct Phantom {}; +struct WorkerProto +{ + /** + * Enumeration of all the request types for the protocol. + */ + enum struct Op : uint64_t; + + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + * + * This currently is just a `Source &`, but more fields will be added + * later. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + * + * This currently is just a `Sink &`, but more fields will be added + * later. + */ + struct WriteConn { + Sink & to; + }; + + /** + * Data type for canonical pairs of serialisers for the worker protocol. + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ + template<typename T> + struct Serialise; + // This is the definition of `Serialise` we *want* to put here, but + // do not do so. + // + // The problem is that if we do so, C++ will think we have + // seralisers for *all* types. We don't, of course, but that won't + // cause an error until link time. That makes for long debug cycles + // when there is a missing serialiser. + // + // By not defining it globally, and instead letting individual + // serialisers specialise the type, we get back the compile-time + // errors we would like. When no serialiser exists, C++ sees an + // abstract "incomplete" type with no definition, and any attempt to + // use `to` or `from` static methods is a compile-time error because + // they don't exist on an incomplete type. + // + // This makes for a quicker debug cycle, as desired. +#if 0 + { + static T read(const Store & store, ReadConn conn); + static void write(const Store & store, WriteConn conn, const T & t); + }; +#endif + + /** + * Wrapper function around `WorkerProto::Serialise<T>::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template<typename T> + static void write(const Store & store, WriteConn conn, const T & t) + { + WorkerProto::Serialise<T>::write(store, conn, t); + } +}; +enum struct WorkerProto::Op : uint64_t +{ + IsValidPath = 1, + HasSubstitutes = 3, + QueryPathHash = 4, // obsolete + QueryReferences = 5, // obsolete + QueryReferrers = 6, + AddToStore = 7, + AddTextToStore = 8, // obsolete since 1.25, Nix 3.0. Use WorkerProto::Op::AddToStore + BuildPaths = 9, + EnsurePath = 10, + AddTempRoot = 11, + AddIndirectRoot = 12, + SyncWithGC = 13, + FindRoots = 14, + ExportPath = 16, // obsolete + QueryDeriver = 18, // obsolete + SetOptions = 19, + CollectGarbage = 20, + QuerySubstitutablePathInfo = 21, + QueryDerivationOutputs = 22, // obsolete + QueryAllValidPaths = 23, + QueryFailedPaths = 24, + ClearFailedPaths = 25, + QueryPathInfo = 26, + ImportPaths = 27, // obsolete + QueryDerivationOutputNames = 28, // obsolete + QueryPathFromHashPart = 29, + QuerySubstitutablePathInfos = 30, + QueryValidPaths = 31, + QuerySubstitutablePaths = 32, + QueryValidDerivers = 33, + OptimiseStore = 34, + VerifyStore = 35, + BuildDerivation = 36, + AddSignatures = 37, + NarFromPath = 38, + AddToStoreNar = 39, + QueryMissing = 40, + QueryDerivationOutputMap = 41, + RegisterDrvOutput = 42, + QueryRealisation = 43, + AddMultipleToStore = 44, + AddBuildLog = 45, + BuildPathsWithResults = 46, +}; -namespace worker_proto { -/* FIXME maybe move more stuff inside here */ +/** + * Convenience for sending operation codes. + * + * @todo Switch to using `WorkerProto::Serialise` instead probably. But + * this was not done at this time so there would be less churn. + */ +inline Sink & operator << (Sink & sink, WorkerProto::Op op) +{ + return sink << (uint64_t) op; +} -#define MAKE_WORKER_PROTO(TEMPLATE, T) \ - TEMPLATE T read(const Store & store, Source & from, Phantom< T > _); \ - TEMPLATE void write(const Store & store, Sink & out, const T & str) +/** + * Convenience for debugging. + * + * @todo Perhaps render known opcodes more nicely. + */ +inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) +{ + return s << (uint64_t) op; +} -MAKE_WORKER_PROTO(, std::string); -MAKE_WORKER_PROTO(, StorePath); -MAKE_WORKER_PROTO(, ContentAddress); -MAKE_WORKER_PROTO(, DerivedPath); -MAKE_WORKER_PROTO(, Realisation); -MAKE_WORKER_PROTO(, DrvOutput); -MAKE_WORKER_PROTO(, BuildResult); -MAKE_WORKER_PROTO(, KeyedBuildResult); -MAKE_WORKER_PROTO(, std::optional<TrustedFlag>); +/** + * Declare a canonical serialiser pair for the worker protocol. + * + * We specialise the struct merely to indicate that we are implementing + * the function for the given type. + * + * Some sort of `template<...>` must be used with the caller for this to + * be legal specialization syntax. See below for what that looks like in + * practice. + */ +#define MAKE_WORKER_PROTO(T) \ + struct WorkerProto::Serialise< T > { \ + static T read(const Store & store, WorkerProto::ReadConn conn); \ + static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ + }; + +template<> +MAKE_WORKER_PROTO(std::string); +template<> +MAKE_WORKER_PROTO(StorePath); +template<> +MAKE_WORKER_PROTO(ContentAddress); +template<> +MAKE_WORKER_PROTO(DerivedPath); +template<> +MAKE_WORKER_PROTO(Realisation); +template<> +MAKE_WORKER_PROTO(DrvOutput); +template<> +MAKE_WORKER_PROTO(BuildResult); +template<> +MAKE_WORKER_PROTO(KeyedBuildResult); +template<> +MAKE_WORKER_PROTO(std::optional<TrustedFlag>); -MAKE_WORKER_PROTO(template<typename T>, std::vector<T>); -MAKE_WORKER_PROTO(template<typename T>, std::set<T>); +template<typename T> +MAKE_WORKER_PROTO(std::vector<T>); +template<typename T> +MAKE_WORKER_PROTO(std::set<T>); -#define X_ template<typename K, typename V> -#define Y_ std::map<K, V> -MAKE_WORKER_PROTO(X_, Y_); +template<typename K, typename V> +#define X_ std::map<K, V> +MAKE_WORKER_PROTO(X_); #undef X_ -#undef Y_ /** * These use the empty string for the null case, relying on the fact - * that the underlying types never serialize to the empty string. + * that the underlying types never serialise to the empty string. * * We do this instead of a generic std::optional<T> instance because * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For @@ -129,72 +240,9 @@ MAKE_WORKER_PROTO(X_, Y_); * worker protocol harder to implement in other languages where such * specializations may not be allowed. */ -MAKE_WORKER_PROTO(, std::optional<StorePath>); -MAKE_WORKER_PROTO(, std::optional<ContentAddress>); - -template<typename T> -std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _) -{ - std::vector<T> resSet; - auto size = readNum<size_t>(from); - while (size--) { - resSet.push_back(read(store, from, Phantom<T> {})); - } - return resSet; -} - -template<typename T> -void write(const Store & store, Sink & out, const std::vector<T> & resSet) -{ - out << resSet.size(); - for (auto & key : resSet) { - write(store, out, key); - } -} - -template<typename T> -std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _) -{ - std::set<T> resSet; - auto size = readNum<size_t>(from); - while (size--) { - resSet.insert(read(store, from, Phantom<T> {})); - } - return resSet; -} - -template<typename T> -void write(const Store & store, Sink & out, const std::set<T> & resSet) -{ - out << resSet.size(); - for (auto & key : resSet) { - write(store, out, key); - } -} - -template<typename K, typename V> -std::map<K, V> read(const Store & store, Source & from, Phantom<std::map<K, V>> _) -{ - std::map<K, V> resMap; - auto size = readNum<size_t>(from); - while (size--) { - auto k = read(store, from, Phantom<K> {}); - auto v = read(store, from, Phantom<V> {}); - resMap.insert_or_assign(std::move(k), std::move(v)); - } - return resMap; -} - -template<typename K, typename V> -void write(const Store & store, Sink & out, const std::map<K, V> & resMap) -{ - out << resMap.size(); - for (auto & i : resMap) { - write(store, out, i.first); - write(store, out, i.second); - } -} - -} +template<> +MAKE_WORKER_PROTO(std::optional<StorePath>); +template<> +MAKE_WORKER_PROTO(std::optional<ContentAddress>); } diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index 7b6c3fcb5..d506dfb74 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -3,6 +3,7 @@ #include <nlohmann/json.hpp> #include "config.hh" +#include "json-utils.hh" namespace nix { template<typename T> diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 081dbeb28..3cf3ed9ca 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,10 +1,9 @@ #include "args.hh" #include "hash.hh" +#include "json-utils.hh" #include <glob.h> -#include <nlohmann/json.hpp> - namespace nix { void Args::addFlag(Flag && flag_) @@ -247,11 +246,7 @@ nlohmann::json Args::toJSON() j["arity"] = flag->handler.arity; if (!flag->labels.empty()) j["labels"] = flag->labels; - // TODO With C++23 use `std::optional::tranform` - if (auto & xp = flag->experimentalFeature) - j["experimental-feature"] = showExperimentalFeature(*xp); - else - j["experimental-feature"] = nullptr; + j["experimental-feature"] = flag->experimentalFeature; flags[name] = std::move(j); } @@ -416,11 +411,7 @@ nlohmann::json MultiCommand::toJSON() cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); j["category"] = std::move(cat); - // TODO With C++23 use `std::optional::tranform` - if (auto xp = command->experimentalFeature()) - cat["experimental-feature"] = showExperimentalFeature(*xp); - else - cat["experimental-feature"] = nullptr; + cat["experimental-feature"] = command->experimentalFeature(); cmds[name] = std::move(j); } diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b6cae5ec3..b9639e761 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -4,6 +4,9 @@ * * Template implementations (as opposed to mere declarations). * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + * * One only needs to include this when one is declaring a * `BaseClass<CustomType>` setting, or as derived class of such an * instantiation. @@ -50,8 +53,11 @@ template<> void BaseSetting<std::set<ExperimentalFeature>>::appendOrSet(std::set template<typename T> void BaseSetting<T>::appendOrSet(T && newValue, bool append) { - static_assert(!trait::appendable, "using default `appendOrSet` implementation with an appendable type"); + static_assert( + !trait::appendable, + "using default `appendOrSet` implementation with an appendable type"); assert(!append); + value = std::move(newValue); } @@ -68,4 +74,60 @@ void BaseSetting<T>::set(const std::string & str, bool append) } } +template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string & category); + +template<typename T> +void BaseSetting<T>::convertToArg(Args & args, const std::string & category) +{ + args.addFlag({ + .longName = name, + .description = fmt("Set the `%s` setting.", name), + .category = category, + .labels = {"value"}, + .handler = {[this](std::string s) { overridden = true; set(s); }}, + .experimentalFeature = experimentalFeature, + }); + + if (isAppendable()) + args.addFlag({ + .longName = "extra-" + name, + .description = fmt("Append to the `%s` setting.", name), + .category = category, + .labels = {"value"}, + .handler = {[this](std::string s) { overridden = true; set(s, true); }}, + .experimentalFeature = experimentalFeature, + }); +} + +#define DECLARE_CONFIG_SERIALISER(TY) \ + template<> TY BaseSetting< TY >::parse(const std::string & str) const; \ + template<> std::string BaseSetting< TY >::to_string() const; + +DECLARE_CONFIG_SERIALISER(std::string) +DECLARE_CONFIG_SERIALISER(std::optional<std::string>) +DECLARE_CONFIG_SERIALISER(bool) +DECLARE_CONFIG_SERIALISER(Strings) +DECLARE_CONFIG_SERIALISER(StringSet) +DECLARE_CONFIG_SERIALISER(StringMap) +DECLARE_CONFIG_SERIALISER(std::set<ExperimentalFeature>) + +template<typename T> +T BaseSetting<T>::parse(const std::string & str) const +{ + static_assert(std::is_integral<T>::value, "Integer required."); + + if (auto n = string2Int<T>(str)) + return *n; + else + throw UsageError("setting '%s' has invalid value '%s'", name, str); +} + +template<typename T> +std::string BaseSetting<T>::to_string() const +{ + static_assert(std::is_integral<T>::value, "Integer required."); + + return std::to_string(value); +} + } diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 085a884dc..38d406e8a 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -219,29 +219,6 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } -template<typename T> -void BaseSetting<T>::convertToArg(Args & args, const std::string & category) -{ - args.addFlag({ - .longName = name, - .description = fmt("Set the `%s` setting.", name), - .category = category, - .labels = {"value"}, - .handler = {[this](std::string s) { overridden = true; set(s); }}, - .experimentalFeature = experimentalFeature, - }); - - if (isAppendable()) - args.addFlag({ - .longName = "extra-" + name, - .description = fmt("Append to the `%s` setting.", name), - .category = category, - .labels = {"value"}, - .handler = {[this](std::string s) { overridden = true; set(s, true); }}, - .experimentalFeature = experimentalFeature, - }); -} - template<> std::string BaseSetting<std::string>::parse(const std::string & str) const { return str; @@ -252,21 +229,17 @@ template<> std::string BaseSetting<std::string>::to_string() const return value; } -template<typename T> -T BaseSetting<T>::parse(const std::string & str) const +template<> std::optional<std::string> BaseSetting<std::optional<std::string>>::parse(const std::string & str) const { - static_assert(std::is_integral<T>::value, "Integer required."); - if (auto n = string2Int<T>(str)) - return *n; + if (str == "") + return std::nullopt; else - throw UsageError("setting '%s' has invalid value '%s'", name, str); + return { str }; } -template<typename T> -std::string BaseSetting<T>::to_string() const +template<> std::string BaseSetting<std::optional<std::string>>::to_string() const { - static_assert(std::is_integral<T>::value, "Integer required."); - return std::to_string(value); + return value ? *value : ""; } template<> bool BaseSetting<bool>::parse(const std::string & str) const @@ -403,17 +376,27 @@ template class BaseSetting<StringSet>; template class BaseSetting<StringMap>; template class BaseSetting<std::set<ExperimentalFeature>>; -Path PathSetting::parse(const std::string & str) const +static Path parsePath(const AbstractSetting & s, const std::string & str) { - if (str == "") { - if (allowEmpty) - return ""; - else - throw UsageError("setting '%s' cannot be empty", name); - } else + if (str == "") + throw UsageError("setting '%s' is a path and paths cannot be empty", s.name); + else return canonPath(str); } +Path PathSetting::parse(const std::string & str) const +{ + return parsePath(*this, str); +} + +std::optional<Path> OptionalPathSetting::parse(const std::string & str) const +{ + if (str == "") + return std::nullopt; + else + return parsePath(*this, str); +} + bool GlobalConfig::set(const std::string & name, const std::string & value) { for (auto & config : *configRegistrations) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 2675baed7..cc8532587 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -353,21 +353,20 @@ public: /** * A special setting for Paths. These are automatically canonicalised * (e.g. "/foo//bar/" becomes "/foo/bar"). + * + * It is mandatory to specify a path; i.e. the empty string is not + * permitted. */ class PathSetting : public BaseSetting<Path> { - bool allowEmpty; - public: PathSetting(Config * options, - bool allowEmpty, const Path & def, const std::string & name, const std::string & description, const std::set<std::string> & aliases = {}) : BaseSetting<Path>(def, true, name, description, aliases) - , allowEmpty(allowEmpty) { options->addSetting(this); } @@ -379,6 +378,30 @@ public: void operator =(const Path & v) { this->assign(v); } }; +/** + * Like `PathSetting`, but the absence of a path is also allowed. + * + * `std::optional` is used instead of the empty string for clarity. + */ +class OptionalPathSetting : public BaseSetting<std::optional<Path>> +{ +public: + + OptionalPathSetting(Config * options, + const std::optional<Path> & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}) + : BaseSetting<std::optional<Path>>(def, true, name, description, aliases) + { + options->addSetting(this); + } + + std::optional<Path> parse(const std::string & str) const override; + + void operator =(const std::optional<Path> & v) { this->assign(v); } +}; + struct GlobalConfig : public AbstractConfig { typedef std::vector<Config*> ConfigRegistrations; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index ad0ec0427..7c4112d32 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{ +constexpr std::array<ExperimentalFeatureDetails, 15> xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -50,6 +50,8 @@ constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{ or other impure derivations can rely on impure derivations. Finally, an impure derivation cannot also be [content-addressed](#xp-feature-ca-derivations). + + This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). )", }, { @@ -207,6 +209,23 @@ constexpr std::array<ExperimentalFeatureDetails, 13> xpFeatureDetails = {{ - "text hashing" derivation outputs, so we can build .drv files. + + - dependencies in derivations on the outputs of + derivations that are themselves derivations outputs. + )", + }, + { + .tag = Xp::ParseTomlTimestamps, + .name = "parse-toml-timestamps", + .description = R"( + Allow parsing of timestamps in builtins.fromTOML. + )", + }, + { + .tag = Xp::ReadOnlyLocalStore, + .name = "read-only-local-store", + .description = R"( + Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. )", }, }}; @@ -243,7 +262,7 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag) return xpFeatureDetails[(size_t)tag].name; } -nlohmann::json documentExperimentalFeatures() +nlohmann::json documentExperimentalFeatures() { StringMap res; for (auto & xpFeature : xpFeatureDetails) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 409100592..faf2e9398 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -3,7 +3,7 @@ #include "comparator.hh" #include "error.hh" -#include "nlohmann/json_fwd.hpp" +#include "json-utils.hh" #include "types.hh" namespace nix { @@ -30,6 +30,8 @@ enum struct ExperimentalFeature DiscardReferences, DaemonTrustOverride, DynamicDerivations, + ParseTomlTimestamps, + ReadOnlyLocalStore, }; /** @@ -92,4 +94,10 @@ public: void to_json(nlohmann::json &, const ExperimentalFeature &); void from_json(const nlohmann::json &, ExperimentalFeature &); +/** + * It is always rendered as a string + */ +template<> +struct json_avoids_null<ExperimentalFeature> : std::true_type {}; + } diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 56be76ecc..11cc0c0e7 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -63,30 +63,19 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) return {std::move(fd), tmpl}; } -void createSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime) +void createSymlink(const Path & target, const Path & link) { if (symlink(target.c_str(), link.c_str())) throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } } -void replaceSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime) +void replaceSymlink(const Path & target, const Path & link) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp, mtime); + createSymlink(target, tmp); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc new file mode 100644 index 000000000..d7220e71d --- /dev/null +++ b/src/libutil/json-utils.cc @@ -0,0 +1,19 @@ +#include "json-utils.hh" + +namespace nix { + +const nlohmann::json * get(const nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +nlohmann::json * get(nlohmann::json & map, const std::string & key) +{ + auto i = map.find(key); + if (i == map.end()) return nullptr; + return &*i; +} + +} diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index eb00e954f..5e63c1af4 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -2,21 +2,77 @@ ///@file #include <nlohmann/json.hpp> +#include <list> namespace nix { -const nlohmann::json * get(const nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; -} +const nlohmann::json * get(const nlohmann::json & map, const std::string & key); + +nlohmann::json * get(nlohmann::json & map, const std::string & key); + +/** + * For `adl_serializer<std::optional<T>>` below, we need to track what + * types are not already using `null`. Only for them can we use `null` + * to represent `std::nullopt`. + */ +template<typename T> +struct json_avoids_null; + +/** + * Handle numbers in default impl + */ +template<typename T> +struct json_avoids_null : std::bool_constant<std::is_integral<T>::value> {}; + +template<> +struct json_avoids_null<std::nullptr_t> : std::false_type {}; + +template<> +struct json_avoids_null<bool> : std::true_type {}; + +template<> +struct json_avoids_null<std::string> : std::true_type {}; + +template<typename T> +struct json_avoids_null<std::vector<T>> : std::true_type {}; + +template<typename T> +struct json_avoids_null<std::list<T>> : std::true_type {}; + +template<typename K, typename V> +struct json_avoids_null<std::map<K, V>> : std::true_type {}; -nlohmann::json * get(nlohmann::json & map, const std::string & key) -{ - auto i = map.find(key); - if (i == map.end()) return nullptr; - return &*i; } +namespace nlohmann { + +/** + * This "instance" is widely requested, see + * https://github.com/nlohmann/json/issues/1749, but momentum has stalled + * out. Writing there here in Nix as a stop-gap. + * + * We need to make sure the underlying type does not use `null` for this to + * round trip. We do that with a static assert. + */ +template<typename T> +struct adl_serializer<std::optional<T>> { + static std::optional<T> from_json(const json & json) { + static_assert( + nix::json_avoids_null<T>::value, + "null is already in use for underlying type's JSON"); + return json.is_null() + ? std::nullopt + : std::optional { adl_serializer<T>::from_json(json) }; + } + static void to_json(json & json, std::optional<T> t) { + static_assert( + nix::json_avoids_null<T>::value, + "null is already in use for underlying type's JSON"); + if (t) + adl_serializer<T>::to_json(json, *t); + else + json = nullptr; + } +}; + } diff --git a/src/libstore/references.cc b/src/libutil/references.cc index 345f4528b..7f59b4c09 100644 --- a/src/libstore/references.cc +++ b/src/libutil/references.cc @@ -6,6 +6,7 @@ #include <map> #include <cstdlib> #include <mutex> +#include <algorithm> namespace nix { @@ -66,69 +67,20 @@ void RefScanSink::operator () (std::string_view data) } -PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap) - : RefScanSink(std::move(hashes)) - , backMap(std::move(backMap)) -{ } - -PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) +RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) + : RewritingSink({{from, to}}, nextSink) { - StringSet hashes; - std::map<std::string, StorePath> backMap; - - for (auto & i : refs) { - std::string hashPart(i.hashPart()); - auto inserted = backMap.emplace(hashPart, i).second; - assert(inserted); - hashes.insert(hashPart); - } - - return PathRefScanSink(std::move(hashes), std::move(backMap)); } -StorePathSet PathRefScanSink::getResultPaths() +RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink) + : rewrites(rewrites), nextSink(nextSink) { - /* Map the hashes found back to their store paths. */ - StorePathSet found; - for (auto & i : getResult()) { - auto j = backMap.find(i); - assert(j != backMap.end()); - found.insert(j->second); + std::string::size_type maxRewriteSize = 0; + for (auto & [from, to] : rewrites) { + assert(from.size() == to.size()); + maxRewriteSize = std::max(maxRewriteSize, from.size()); } - - return found; -} - - -std::pair<StorePathSet, HashResult> scanForReferences( - const std::string & path, - const StorePathSet & refs) -{ - HashSink hashSink { htSHA256 }; - auto found = scanForReferences(hashSink, path, refs); - auto hash = hashSink.finish(); - return std::pair<StorePathSet, HashResult>(found, hash); -} - -StorePathSet scanForReferences( - Sink & toTee, - const Path & path, - const StorePathSet & refs) -{ - PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); - TeeSink sink { refsSink, toTee }; - - /* Look for the hashes in the NAR dump of the path. */ - dumpPath(path, sink); - - return refsSink.getResultPaths(); -} - - -RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) - : from(from), to(to), nextSink(nextSink) -{ - assert(from.size() == to.size()); + this->maxRewriteSize = maxRewriteSize; } void RewritingSink::operator () (std::string_view data) @@ -136,13 +88,13 @@ void RewritingSink::operator () (std::string_view data) std::string s(prev); s.append(data); - size_t j = 0; - while ((j = s.find(from, j)) != std::string::npos) { - matches.push_back(pos + j); - s.replace(j, from.size(), to); - } + s = rewriteStrings(s, rewrites); - prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1); + prev = s.size() < maxRewriteSize + ? s + : maxRewriteSize == 0 + ? "" + : std::string(s, s.size() - maxRewriteSize + 1, maxRewriteSize - 1); auto consumed = s.size() - prev.size(); diff --git a/src/libstore/references.hh b/src/libutil/references.hh index 52d71b333..f0baeffe1 100644 --- a/src/libstore/references.hh +++ b/src/libutil/references.hh @@ -2,14 +2,9 @@ ///@file #include "hash.hh" -#include "path.hh" namespace nix { -std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs); - -StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); - class RefScanSink : public Sink { StringSet hashes; @@ -28,28 +23,18 @@ public: void operator () (std::string_view data) override; }; -class PathRefScanSink : public RefScanSink -{ - std::map<std::string, StorePath> backMap; - - PathRefScanSink(StringSet && hashes, std::map<std::string, StorePath> && backMap); - -public: - - static PathRefScanSink fromPaths(const StorePathSet & refs); - - StorePathSet getResultPaths(); -}; - struct RewritingSink : Sink { - std::string from, to, prev; + const StringMap rewrites; + std::string::size_type maxRewriteSize; + std::string prev; Sink & nextSink; uint64_t pos = 0; std::vector<uint64_t> matches; RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + RewritingSink(const StringMap & rewrites, Sink & nextSink); void operator () (std::string_view data) override; diff --git a/src/libutil/tests/references.cc b/src/libutil/tests/references.cc new file mode 100644 index 000000000..a517d9aa1 --- /dev/null +++ b/src/libutil/tests/references.cc @@ -0,0 +1,46 @@ +#include "references.hh" +#include <gtest/gtest.h> + +namespace nix { + +using std::string; + +struct RewriteParams { + string originalString, finalString; + StringMap rewrites; + + friend std::ostream& operator<<(std::ostream& os, const RewriteParams& bar) { + StringSet strRewrites; + for (auto & [from, to] : bar.rewrites) + strRewrites.insert(from + "->" + to); + return os << + "OriginalString: " << bar.originalString << std::endl << + "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl << + "Expected result: " << bar.finalString; + } +}; + +class RewriteTest : public ::testing::TestWithParam<RewriteParams> { +}; + +TEST_P(RewriteTest, IdentityRewriteIsIdentity) { + RewriteParams param = GetParam(); + StringSink rewritten; + auto rewriter = RewritingSink(param.rewrites, rewritten); + rewriter(param.originalString); + rewriter.flush(); + ASSERT_EQ(rewritten.s, param.finalString); +} + +INSTANTIATE_TEST_CASE_P( + references, + RewriteTest, + ::testing::Values( + RewriteParams{ "foooo", "baroo", {{"foo", "bar"}, {"bar", "baz"}}}, + RewriteParams{ "foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}}, + RewriteParams{ "foooo", "foooo", {}} + ) +); + +} + diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 250e83a38..f3c1e8248 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -202,7 +202,7 @@ namespace nix { } TEST(pathExists, bogusPathDoesNotExist) { - ASSERT_FALSE(pathExists("/home/schnitzel/darmstadt/pommes")); + ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); } /* ---------------------------------------------------------------------------- diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 21d1c8dcd..26f9dc8a8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -266,6 +266,17 @@ bool pathExists(const Path & path) return false; } +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + Path readLink(const Path & path) { @@ -1141,9 +1152,9 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss) } std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional<std::string> & input) + const std::optional<std::string> & input, bool isInteractive) { - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input}); + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); if (!statusOk(res.first)) throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); @@ -1193,6 +1204,16 @@ void runProgram2(const RunOptions & options) // case), so we can't use it if we alter the environment processOptions.allowVfork = !options.environment; + std::optional<Finally<std::function<void()>>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + /* Fork. */ Pid pid = startProcess([&]() { if (options.environment) @@ -1832,6 +1853,7 @@ void setStackSize(size_t stackSize) #if __linux__ static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; #endif void saveMountNamespace() @@ -1839,10 +1861,11 @@ void saveMountNamespace() #if __linux__ static std::once_flag done; std::call_once(done, []() { - AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); - if (!fd) + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) throw SysError("saving parent mount namespace"); - fdSavedMountNamespace = std::move(fd); + + fdSavedRoot = open("/proc/self/root", O_RDONLY); }); #endif } @@ -1855,9 +1878,16 @@ void restoreMountNamespace() if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); - if (chdir(savedCwd.c_str()) == -1) { - throw SysError("restoring cwd"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); } catch (Error & e) { debug(e.msg()); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 040fed68f..b302d6f45 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -121,6 +121,14 @@ struct stat lstat(const Path & path); bool pathExists(const Path & path); /** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + +/** * Read the contents (target) of a symbolic link. The result is not * in any way canonicalised. */ @@ -248,14 +256,12 @@ inline Paths createDirs(PathView path) /** * Create a symlink. */ -void createSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime = {}); +void createSymlink(const Path & target, const Path & link); /** * Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link, - std::optional<time_t> mtime = {}); +void replaceSymlink(const Path & target, const Path & link); void renameFile(const Path & src, const Path & dst); @@ -415,7 +421,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = P */ std::string runProgram(Path program, bool searchPath = false, const Strings & args = Strings(), - const std::optional<std::string> & input = {}); + const std::optional<std::string> & input = {}, bool isInteractive = false); struct RunOptions { @@ -430,6 +436,7 @@ struct RunOptions Source * standardIn = nullptr; Sink * standardOut = nullptr; bool mergeStderrToStdout = false; + bool isInteractive = false; }; std::pair<int, std::string> runProgram(RunOptions && options); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 740737ffe..c1c8edd1d 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -177,6 +177,7 @@ static int main_nix_channel(int argc, char ** argv) cRemove, cList, cUpdate, + cListGenerations, cRollback } cmd = cNone; std::vector<std::string> args; @@ -193,6 +194,8 @@ static int main_nix_channel(int argc, char ** argv) cmd = cList; } else if (*arg == "--update") { cmd = cUpdate; + } else if (*arg == "--list-generations") { + cmd = cListGenerations; } else if (*arg == "--rollback") { cmd = cRollback; } else { @@ -237,6 +240,11 @@ static int main_nix_channel(int argc, char ** argv) case cUpdate: update(StringSet(args.begin(), args.end())); break; + case cListGenerations: + if (!args.empty()) + throw UsageError("'--list-generations' expects no arguments"); + std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush; + break; case cRollback: if (args.size() > 1) throw UsageError("'--rollback' has at most one argument"); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 3cc57af4e..70af53b28 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -41,9 +41,10 @@ void removeOldGenerations(std::string dir) } if (link.find("link") != std::string::npos) { printInfo("removing old generations of profile %s", path); - if (deleteOlderThan != "") - deleteGenerationsOlderThan(path, deleteOlderThan, dryRun); - else + if (deleteOlderThan != "") { + auto t = parseOlderThanTimeSpec(deleteOlderThan); + deleteGenerationsOlderThan(path, t, dryRun); + } else deleteOldGenerations(path, dryRun); } } else if (type == DT_DIR) { @@ -77,7 +78,12 @@ static int main_nix_collect_garbage(int argc, char * * argv) return true; }); - if (removeOld) removeOldGenerations(profilesDir()); + if (removeOld) { + std::set<Path> dirsToClean = { + profilesDir(), settings.nixStateDir + "/profiles", dirOf(getDefaultProfile())}; + for (auto & dir : dirsToClean) + removeOldGenerations(dir); + } // Run the actual garbage collector. if (!dryRun) { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 5e94f2d14..91b073b49 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -772,7 +772,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) debug("switching to new user environment"); Path generation = createGeneration( - ref<LocalFSStore>(store2), + *store2, globals.profile, drv.queryOutPath()); switchLink(globals.profile, generation); @@ -1356,13 +1356,14 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr if (opArgs.size() == 1 && opArgs.front() == "old") { deleteOldGenerations(globals.profile, globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('d') != std::string::npos) { - deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); + auto t = parseOlderThanTimeSpec(opArgs.front()); + deleteGenerationsOlderThan(globals.profile, t, globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('+') != std::string::npos) { if (opArgs.front().size() < 2) throw Error("invalid number of generations '%1%'", opArgs.front()); auto str_max = opArgs.front().substr(1); auto max = string2Int<GenerationNumber>(str_max); - if (!max || *max == 0) + if (!max) throw Error("invalid number of generations to keep '%1%'", opArgs.front()); deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun); } else { diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 9e916abc4..d12d70f33 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -158,7 +158,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, } debug("switching to new user environment"); - Path generation = createGeneration(ref<LocalFSStore>(store2), profile, topLevelOut); + Path generation = createGeneration(*store2, profile, topLevelOut); switchLink(profile, generation); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 40f30eb63..caa0248f1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -12,6 +12,7 @@ #include "shared.hh" #include "util.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" @@ -806,6 +807,9 @@ static void opServe(Strings opFlags, Strings opArgs) out.flush(); unsigned int clientVersion = readInt(in); + WorkerProto::ReadConn rconn { .from = in }; + WorkerProto::WriteConn wconn { .to = out }; + auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're // building through the daemon. @@ -837,19 +841,19 @@ static void opServe(Strings opFlags, Strings opArgs) }; while (true) { - ServeCommand cmd; + ServeProto::Command cmd; try { - cmd = (ServeCommand) readInt(in); + cmd = (ServeProto::Command) readInt(in); } catch (EndOfFile & e) { break; } switch (cmd) { - case cmdQueryValidPaths: { + case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = worker_proto::read(*store, in, Phantom<StorePathSet> {}); + auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -858,19 +862,19 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - worker_proto::write(*store, out, store->queryValidPaths(paths)); + WorkerProto::write(*store, wconn, store->queryValidPaths(paths)); break; } - case cmdQueryPathInfos: { - auto paths = worker_proto::read(*store, in, Phantom<StorePathSet> {}); + case ServeProto::Command::QueryPathInfos: { + auto paths = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - worker_proto::write(*store, out, info->references); + WorkerProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; @@ -885,24 +889,24 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case cmdDumpStorePath: + case ServeProto::Command::DumpStorePath: store->narFromPath(store->parseStorePath(readString(in)), out); break; - case cmdImportPaths: { + case ServeProto::Command::ImportPaths: { if (!writeAllowed) throw Error("importing paths is not allowed"); store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking? out << 1; // indicate success break; } - case cmdExportPaths: { + case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(worker_proto::read(*store, in, Phantom<StorePathSet> {}), out); + store->exportPaths(WorkerProto::Serialise<StorePathSet>::read(*store, rconn), out); break; } - case cmdBuildPaths: { + case ServeProto::Command::BuildPaths: { if (!writeAllowed) throw Error("building paths is not allowed"); @@ -923,7 +927,7 @@ static void opServe(Strings opFlags, Strings opArgs) break; } - case cmdBuildDerivation: { /* Used by hydra-queue-runner. */ + case ServeProto::Command::BuildDerivation: { /* Used by hydra-queue-runner. */ if (!writeAllowed) throw Error("building paths is not allowed"); @@ -944,22 +948,22 @@ static void opServe(Strings opFlags, Strings opArgs) DrvOutputs builtOutputs; for (auto & [output, realisation] : status.builtOutputs) builtOutputs.insert_or_assign(realisation.id, realisation); - worker_proto::write(*store, out, builtOutputs); + WorkerProto::write(*store, wconn, builtOutputs); } break; } - case cmdQueryClosure: { + case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(worker_proto::read(*store, in, Phantom<StorePathSet> {}), + store->computeFSClosure(WorkerProto::Serialise<StorePathSet>::read(*store, rconn), closure, false, includeOutputs); - worker_proto::write(*store, out, closure); + WorkerProto::write(*store, wconn, closure); break; } - case cmdAddToStoreNar: { + case ServeProto::Command::AddToStoreNar: { if (!writeAllowed) throw Error("importing paths is not allowed"); auto path = readString(in); @@ -970,7 +974,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = worker_proto::read(*store, in, Phantom<StorePathSet> {}); + info.references = WorkerProto::Serialise<StorePathSet>::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings<StringSet>(in); info.ca = ContentAddress::parseOpt(readString(in)); diff --git a/src/nix/app.cc b/src/nix/app.cc index fd4569bb4..e678b54f0 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -7,6 +7,7 @@ #include "names.hh" #include "command.hh" #include "derivations.hh" +#include "downstream-placeholder.hh" namespace nix { @@ -23,7 +24,7 @@ StringPairs resolveRewrites( if (auto drvDep = std::get_if<BuiltPathBuilt>(&dep.path)) for (auto & [ outputName, outputPath ] : drvDep->outputs) res.emplace( - downstreamPlaceholder(store, drvDep->drvPath, outputName), + DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), store.printStorePath(outputPath) ); return res; diff --git a/src/nix/build.md b/src/nix/build.md index ee414dc86..0fbb39cc3 100644 --- a/src/nix/build.md +++ b/src/nix/build.md @@ -44,7 +44,7 @@ R""( `release.nix`: ```console - # nix build -f release.nix build.x86_64-linux + # nix build --file release.nix build.x86_64-linux ``` * Build a NixOS system configuration from a flake, and make a profile diff --git a/src/nix/copy.md b/src/nix/copy.md index 25e0ddadc..199006436 100644 --- a/src/nix/copy.md +++ b/src/nix/copy.md @@ -15,7 +15,7 @@ R""( SSH: ```console - # nix copy -s --to ssh://server /run/current-system + # nix copy --substitute-on-destination --to ssh://server /run/current-system ``` The `-s` flag causes the remote machine to try to substitute missing diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index c1a91c63d..1511f9e6e 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -4,6 +4,7 @@ #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" +#include "remote-store-connection.hh" #include "util.hh" #include "serialise.hh" #include "archive.hh" @@ -24,6 +25,7 @@ #include <sys/stat.h> #include <sys/socket.h> #include <sys/un.h> +#include <sys/select.h> #include <errno.h> #include <pwd.h> #include <grp.h> @@ -54,19 +56,16 @@ struct AuthorizationSettings : Config { Setting<Strings> trustedUsers{ this, {"root"}, "trusted-users", R"( - A list of names of users (separated by whitespace) that have - additional rights when connecting to the Nix daemon, such as the - ability to specify additional binary caches, or to import unsigned - NARs. You can also specify groups by prefixing them with `@`; for - instance, `@wheel` means all users in the `wheel` group. The default - is `root`. + A list of user names, separated by whitespace. + These users will have additional rights when connecting to the Nix daemon, such as the ability to specify additional [substituters](#conf-substituters), or to import unsigned [NARs](@docroot@/glossary.md#gloss-nar). + + You can also specify groups by prefixing names with `@`. + For instance, `@wheel` means all users in the `wheel` group. > **Warning** > - > Adding a user to `trusted-users` is essentially equivalent to - > giving that user root access to the system. For example, the user - > can set `sandbox-paths` and thereby obtain read access to - > directories that are otherwise inacessible to them. + > Adding a user to `trusted-users` is essentially equivalent to giving that user root access to the system. + > For example, the user can access or replace store path contents that are critical for system security. )"}; /** @@ -75,12 +74,16 @@ struct AuthorizationSettings : Config { Setting<Strings> allowedUsers{ this, {"*"}, "allowed-users", R"( - A list of names of users (separated by whitespace) that are allowed - to connect to the Nix daemon. As with the `trusted-users` option, - you can specify groups by prefixing them with `@`. Also, you can - allow all users by specifying `*`. The default is `*`. + A list user names, separated by whitespace. + These users are allowed to connect to the Nix daemon. + + You can specify groups by prefixing names with `@`. + For instance, `@wheel` means all users in the `wheel` group. + Also, you can allow all users by specifying `*`. - Note that trusted users are always allowed to connect. + > **Note** + > + > Trusted users (set in [`trusted-users`](#conf-trusted-users)) can always connect to the Nix daemon. )"}; }; diff --git a/src/nix/develop.md b/src/nix/develop.md index c49b39669..1b5a8aeba 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -69,7 +69,7 @@ R""( * Run a series of script commands: ```console - # nix develop --command bash -c "mkdir build && cmake .. && make" + # nix develop --command bash --command "mkdir build && cmake .. && make" ``` # Description diff --git a/src/nix/eval.md b/src/nix/eval.md index 3b510737a..48d5aa597 100644 --- a/src/nix/eval.md +++ b/src/nix/eval.md @@ -18,7 +18,7 @@ R""( * Evaluate a Nix expression from a file: ```console - # nix eval -f ./my-nixpkgs hello.name + # nix eval --file ./my-nixpkgs hello.name ``` * Get the current version of the `nixpkgs` flake: diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index 07031c909..c8307f8d8 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -68,6 +68,6 @@ The following flake output attributes must be In addition, the `hydraJobs` output is evaluated in the same way as Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested attribute set of derivations). Similarly, the -`legacyPackages`.*system* output is evaluated like `nix-env -qa`. +`legacyPackages`.*system* output is evaluated like `nix-env --query --available `. )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3db655aeb..b5f5d0cac 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -179,6 +179,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(Base16, false); + if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) + j["dirtyRevision"] = *dirtyRev; if (auto revCount = flake.lockedRef.input.getRevCount()) j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) @@ -204,6 +206,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", rev->to_string(Base16, false)); + if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) + logger->cout( + ANSI_BOLD "Revision:" ANSI_NORMAL " %s", + *dirtyRev); if (auto revCount = flake.lockedRef.input.getRevCount()) logger->cout( ANSI_BOLD "Revisions:" ANSI_NORMAL " %s", @@ -259,6 +265,7 @@ struct CmdFlakeInfo : CmdFlakeMetadata struct CmdFlakeCheck : FlakeCommand { bool build = true; + bool checkAllSystems = false; CmdFlakeCheck() { @@ -267,6 +274,11 @@ struct CmdFlakeCheck : FlakeCommand .description = "Do not build checks.", .handler = {&build, false} }); + addFlag({ + .longName = "all-systems", + .description = "Check the outputs for all systems.", + .handler = {&checkAllSystems, true} + }); } std::string description() override @@ -292,6 +304,7 @@ struct CmdFlakeCheck : FlakeCommand lockFlags.applyNixConfig = true; auto flake = lockFlake(); + auto localSystem = std::string(settings.thisSystem.get()); bool hasErrors = false; auto reportError = [&](const Error & e) { @@ -307,6 +320,8 @@ struct CmdFlakeCheck : FlakeCommand } }; + std::set<std::string> omittedSystems; + // FIXME: rewrite to use EvalCache. auto resolve = [&] (PosIdx p) { @@ -327,6 +342,15 @@ struct CmdFlakeCheck : FlakeCommand reportError(Error("'%s' is not a valid system type, at %s", system, resolve(pos))); }; + auto checkSystemType = [&](const std::string & system, const PosIdx pos) { + if (!checkAllSystems && system != localSystem) { + omittedSystems.insert(system); + return false; + } else { + return true; + } + }; + auto checkDerivation = [&](const std::string & attrPath, Value & v, const PosIdx pos) -> std::optional<StorePath> { try { auto drvInfo = getDerivation(*state, v, false); @@ -362,8 +386,10 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { state->forceValue(v, pos); - if (!v.isLambda() - || v.lambda.fun->hasFormals() + if (!v.isLambda()) { + throw Error("overlay is not a function, but %s instead", showType(v)); + } + if (v.lambda.fun->hasFormals() || !argHasName(v.lambda.fun->arg, "final")) throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body); @@ -509,16 +535,18 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) { - auto drvPath = checkDerivation( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); - if (drvPath && attr_name == settings.thisSystem.get()) { - drvPaths.push_back(DerivedPath::Built { - .drvPath = *drvPath, - .outputs = OutputsSpec::All { }, - }); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) { + auto drvPath = checkDerivation( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + if (drvPath && attr_name == settings.thisSystem.get()) { + drvPaths.push_back(DerivedPath::Built { + .drvPath = *drvPath, + .outputs = OutputsSpec::All { }, + }); + } } } } @@ -529,9 +557,11 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkApp( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkApp( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } @@ -540,11 +570,13 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) - checkDerivation( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) + checkDerivation( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + }; } } @@ -553,11 +585,13 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) - checkApp( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) + checkApp( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + }; } } @@ -566,9 +600,11 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkDerivation( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkDerivation( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } @@ -577,9 +613,11 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkApp( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos) ) { + checkApp( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } @@ -587,6 +625,7 @@ struct CmdFlakeCheck : FlakeCommand state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs) { checkSystemName(state->symbols[attr.name], attr.pos); + checkSystemType(state->symbols[attr.name], attr.pos); // FIXME: do getDerivations? } } @@ -636,9 +675,11 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - checkBundler( - fmt("%s.%s", name, attr_name), - *attr.value, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkBundler( + fmt("%s.%s", name, attr_name), + *attr.value, attr.pos); + }; } } @@ -647,12 +688,14 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *vOutput.attrs) { const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs) { - checkBundler( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, attr2.pos); - } + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs) { + checkBundler( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, attr2.pos); + } + }; } } @@ -685,7 +728,15 @@ struct CmdFlakeCheck : FlakeCommand } if (hasErrors) throw Error("some errors were encountered during the evaluation"); - } + + if (!omittedSystems.empty()) { + warn( + "The check omitted these incompatible systems: %s\n" + "Use '--all-systems' to check all.", + concatStringsSep(", ", omittedSystems) + ); + }; + }; }; static Strings defaultTemplateAttrPathsPrefixes{"templates."}; diff --git a/src/nix/flake.md b/src/nix/flake.md index 456fd0ea1..92f477917 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -71,8 +71,6 @@ inputs.nixpkgs = { Here are some examples of flake references in their URL-like representation: -* `.`: The flake in the current directory. -* `/home/alice/src/patchelf`: A flake in some other directory. * `nixpkgs`: The `nixpkgs` entry in the flake registry. * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` entry in the flake registry, with its Git revision overridden to a @@ -93,6 +91,23 @@ Here are some examples of flake references in their URL-like representation: * `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball flake. +## Path-like syntax + +Flakes corresponding to a local path can also be referred to by a direct path reference, either `/absolute/path/to/the/flake` or `./relative/path/to/the/flake` (note that the leading `./` is mandatory for relative paths to avoid any ambiguity). + +The semantic of such a path is as follows: + +* If the directory is part of a Git repository, then the input will be treated as a `git+file:` URL, otherwise it will be treated as a `path:` url; +* If the directory doesn't contain a `flake.nix` file, then Nix will search for such a file upwards in the file system hierarchy until it finds any of: + 1. The Git repository root, or + 2. The filesystem root (/), or + 3. A folder on a different mount point. + +### Examples + +* `.`: The flake to which the current directory belongs to. +* `/home/alice/src/patchelf`: A flake in some other directory. + ## Flake reference attributes The following generic flake reference attributes are supported: diff --git a/src/nix/main.cc b/src/nix/main.cc index ce0bed2a3..650c79d14 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -352,7 +352,7 @@ void mainWrapped(int argc, char * * argv) return; } - if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { + if (argc == 2 && std::string(argv[1]) == "__dump-language") { experimentalFeatureSettings.experimentalFeatures = { Xp::Flakes, Xp::FetchClosure, @@ -360,17 +360,34 @@ void mainWrapped(int argc, char * * argv) evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); auto res = nlohmann::json::object(); - auto builtins = state.baseEnv.values[0]->attrs; - for (auto & builtin : *builtins) { - auto b = nlohmann::json::object(); - if (!builtin.value->isPrimOp()) continue; - auto primOp = builtin.value->primOp; - if (!primOp->doc) continue; - b["arity"] = primOp->arity; - b["args"] = primOp->args; - b["doc"] = trim(stripIndentation(primOp->doc)); - res[state.symbols[builtin.name]] = std::move(b); - } + res["builtins"] = ({ + auto builtinsJson = nlohmann::json::object(); + auto builtins = state.baseEnv.values[0]->attrs; + for (auto & builtin : *builtins) { + auto b = nlohmann::json::object(); + if (!builtin.value->isPrimOp()) continue; + auto primOp = builtin.value->primOp; + if (!primOp->doc) continue; + b["arity"] = primOp->arity; + b["args"] = primOp->args; + b["doc"] = trim(stripIndentation(primOp->doc)); + b["experimental-feature"] = primOp->experimentalFeature; + builtinsJson[state.symbols[builtin.name]] = std::move(b); + } + std::move(builtinsJson); + }); + res["constants"] = ({ + auto constantsJson = nlohmann::json::object(); + for (auto & [name, info] : state.constantInfos) { + auto c = nlohmann::json::object(); + if (!info.doc) continue; + c["doc"] = trim(stripIndentation(info.doc)); + c["type"] = showType(info.type, false); + c["impure-only"] = info.impureOnly; + constantsJson[name] = std::move(c); + } + std::move(constantsJson); + }); logger->cout("%s", res); return; } diff --git a/src/nix/nar-ls.md b/src/nix/nar-ls.md index d373f9715..5a03c5d82 100644 --- a/src/nix/nar-ls.md +++ b/src/nix/nar-ls.md @@ -5,7 +5,7 @@ R""( * To list a specific file in a NAR: ```console - # nix nar ls -l ./hello.nar /bin/hello + # nix nar ls --long ./hello.nar /bin/hello -r-xr-xr-x 38184 hello ``` @@ -13,7 +13,7 @@ R""( format: ```console - # nix nar ls --json -R ./hello.nar /bin + # nix nar ls --json --recursive ./hello.nar /bin {"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}} ``` diff --git a/src/nix/nix.md b/src/nix/nix.md index 1ef6c7fcd..e0f459d6b 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -63,7 +63,7 @@ The following types of installable are supported by most commands: - [Nix file](#nix-file), optionally qualified by an attribute path - [Nix expression](#nix-expression), optionally qualified by an attribute path -For most commands, if no installable is specified, `.` as assumed. +For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. ### Flake output attribute @@ -102,6 +102,7 @@ way: available in the flake. If this is undesirable, specify `path:<directory>` explicitly; For example, if `/foo/bar` is a git repository with the following structure: + ``` . └── baz @@ -197,7 +198,7 @@ operate are determined as follows: of all outputs of the `glibc` package in the binary cache: ```console - # nix path-info -S --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*' + # nix path-info --closure-size --eval-store auto --store https://cache.nixos.org 'nixpkgs#glibc^*' /nix/store/g02b1lpbddhymmcjb923kf0l7s9nww58-glibc-2.33-123 33208200 /nix/store/851dp95qqiisjifi639r0zzg5l465ny4-glibc-2.33-123-bin 36142896 /nix/store/kdgs3q6r7xdff1p7a9hnjr43xw2404z7-glibc-2.33-123-debug 155787312 @@ -208,7 +209,7 @@ operate are determined as follows: and likewise, using a store path to a "drv" file to specify the derivation: ```console - # nix path-info -S '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*' + # nix path-info --closure-size '/nix/store/gzaflydcr6sb3567hap9q6srzx8ggdgg-glibc-2.33-78.drv^*' … ``` * If you didn't specify the desired outputs, but the derivation has an diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 6ad23a02e..2dda866d0 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -13,7 +13,7 @@ R""( closure, sorted by size: ```console - # nix path-info -rS /run/current-system | sort -nk2 + # nix path-info --recursive --closure-size /run/current-system | sort -nk2 /nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty 96 /nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers 112 … @@ -25,7 +25,7 @@ R""( readable sizes: ```console - # nix path-info -rsSh nixpkgs#rustc + # nix path-info --recursive --size --closure-size --human-readable nixpkgs#rustc /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7K 69.1M /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9M 37.4M … @@ -34,7 +34,7 @@ R""( * Check the existence of a path in a binary cache: ```console - # nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/ + # nix path-info --recursive /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/ path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid ``` @@ -57,7 +57,7 @@ R""( size: ```console - # nix path-info --json --all -S \ + # nix path-info --json --all --closure-size \ | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' [ …, diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md index fa786162f..5d7fcc0ec 100644 --- a/src/nix/profile-list.md +++ b/src/nix/profile-list.md @@ -6,26 +6,48 @@ R""( ```console # nix profile list - 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 - 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 - 2 flake:blender-bin#packages.x86_64-linux.default github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#packages.x86_64-linux.default /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 + Index: 0 + Flake attribute: legacyPackages.x86_64-linux.gdb + Original flake URL: flake:nixpkgs + Locked flake URL: github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca + Store paths: /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1 + + Index: 1 + Flake attribute: packages.x86_64-linux.default + Original flake URL: flake:blender-bin + Locked flake URL: github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender + Store paths: /nix/store/i798sxl3j40wpdi1rgf391id1b5klw7g-blender-bin-3.1.2 ``` + Note that you can unambiguously rebuild a package from a profile + through its locked flake URL and flake attribute, e.g. + + ```console + # nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default + ``` + + will build the package with index 1 shown above. + # Description This command shows what packages are currently installed in a -profile. The output consists of one line per package, with the -following fields: +profile. For each installed package, it shows the following +information: + +* `Index`: An integer that can be used to unambiguously identify the + package in invocations of `nix profile remove` and `nix profile + upgrade`. -* An integer that can be used to unambiguously identify the package in - invocations of `nix profile remove` and `nix profile upgrade`. +* `Flake attribute`: The flake output attribute path that provides the + package (e.g. `packages.x86_64-linux.hello`). -* The original ("unlocked") flake reference and output attribute path - used at installation time. +* `Original flake URL`: The original ("unlocked") flake reference + specified by the user when the package was first installed via `nix + profile install`. -* The locked flake reference to which the unlocked flake reference was - resolved. +* `Locked flake URL`: The locked flake reference to which the original + flake reference was resolved. -* The store path(s) of the package. +* `Store paths`: The store path(s) of the package. )"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index fd63b3519..b833b5192 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -21,7 +21,7 @@ struct ProfileElementSource { FlakeRef originalRef; // FIXME: record original attrpath. - FlakeRef resolvedRef; + FlakeRef lockedRef; std::string attrPath; ExtendedOutputsSpec outputs; @@ -31,6 +31,11 @@ struct ProfileElementSource std::tuple(originalRef.to_string(), attrPath, outputs) < std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs); } + + std::string to_string() const + { + return fmt("%s#%s%s", originalRef, attrPath, outputs.to_string()); + } }; const int defaultPriority = 5; @@ -42,16 +47,30 @@ struct ProfileElement bool active = true; int priority = defaultPriority; - std::string describe() const + std::string identifier() const { if (source) - return fmt("%s#%s%s", source->originalRef, source->attrPath, source->outputs.to_string()); + return source->to_string(); StringSet names; for (auto & path : storePaths) names.insert(DrvName(path.name()).name); return concatStringsSep(", ", names); } + /** + * Return a string representing an installable corresponding to the current + * element, either a flakeref or a plain store path + */ + std::set<std::string> toInstallables(Store & store) + { + if (source) + return {source->to_string()}; + StringSet rawPaths; + for (auto & path : storePaths) + rawPaths.insert(store.printStorePath(path)); + return rawPaths; + } + std::string versions() const { StringSet versions; @@ -62,7 +81,7 @@ struct ProfileElement bool operator < (const ProfileElement & other) const { - return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); + return std::tuple(identifier(), storePaths) < std::tuple(other.identifier(), other.storePaths); } void updateStorePaths( @@ -149,7 +168,7 @@ struct ProfileManifest } } - std::string toJSON(Store & store) const + nlohmann::json toJSON(Store & store) const { auto array = nlohmann::json::array(); for (auto & element : elements) { @@ -162,7 +181,7 @@ struct ProfileManifest obj["priority"] = element.priority; if (element.source) { obj["originalUrl"] = element.source->originalRef.to_string(); - obj["url"] = element.source->resolvedRef.to_string(); + obj["url"] = element.source->lockedRef.to_string(); obj["attrPath"] = element.source->attrPath; obj["outputs"] = element.source->outputs; } @@ -171,7 +190,7 @@ struct ProfileManifest nlohmann::json json; json["version"] = 2; json["elements"] = array; - return json.dump(); + return json; } StorePath build(ref<Store> store) @@ -191,7 +210,7 @@ struct ProfileManifest buildProfile(tempDir, std::move(pkgs)); - writeFile(tempDir + "/manifest.json", toJSON(*store)); + writeFile(tempDir + "/manifest.json", toJSON(*store).dump()); /* Add the symlink tree to the store. */ StringSink sink; @@ -237,13 +256,13 @@ struct ProfileManifest bool changes = false; while (i != prevElems.end() || j != curElems.end()) { - if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) { - logger->cout("%s%s: ∅ -> %s", indent, j->describe(), j->versions()); + if (j != curElems.end() && (i == prevElems.end() || i->identifier() > j->identifier())) { + logger->cout("%s%s: ∅ -> %s", indent, j->identifier(), j->versions()); changes = true; ++j; } - else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) { - logger->cout("%s%s: %s -> ∅", indent, i->describe(), i->versions()); + else if (i != prevElems.end() && (j == curElems.end() || i->identifier() < j->identifier())) { + logger->cout("%s%s: %s -> ∅", indent, i->identifier(), i->versions()); changes = true; ++i; } @@ -251,7 +270,7 @@ struct ProfileManifest auto v1 = i->versions(); auto v2 = j->versions(); if (v1 != v2) { - logger->cout("%s%s: %s -> %s", indent, i->describe(), v1, v2); + logger->cout("%s%s: %s -> %s", indent, i->identifier(), v1, v2); changes = true; } ++i; @@ -330,7 +349,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile if (auto * info2 = dynamic_cast<ExtraPathInfoFlake *>(&*info)) { element.source = ProfileElementSource { .originalRef = info2->flake.originalRef, - .resolvedRef = info2->flake.resolvedRef, + .lockedRef = info2->flake.lockedRef, .attrPath = info2->value.attrPath, .outputs = info2->value.extendedOutputsSpec, }; @@ -363,10 +382,10 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile auto profileElement = *it; for (auto & storePath : profileElement.storePaths) { if (conflictError.fileA.starts_with(store->printStorePath(storePath))) { - return std::pair(conflictError.fileA, profileElement.source->originalRef); + return std::pair(conflictError.fileA, profileElement.toInstallables(*store)); } if (conflictError.fileB.starts_with(store->printStorePath(storePath))) { - return std::pair(conflictError.fileB, profileElement.source->originalRef); + return std::pair(conflictError.fileB, profileElement.toInstallables(*store)); } } } @@ -375,9 +394,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile // There are 2 conflicting files. We need to find out which one is from the already installed package and // which one is the package that is the new package that is being installed. // The first matching package is the one that was already installed (original). - auto [originalConflictingFilePath, originalConflictingRef] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end()); + auto [originalConflictingFilePath, originalConflictingRefs] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end()); // The last matching package is the one that was going to be installed (new). - auto [newConflictingFilePath, newConflictingRef] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend()); + auto [newConflictingFilePath, newConflictingRefs] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend()); throw Error( "An existing package already provides the following file:\n" @@ -403,8 +422,8 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile " nix profile install %4% --priority %7%\n", originalConflictingFilePath, newConflictingFilePath, - originalConflictingRef.to_string(), - newConflictingRef.to_string(), + concatStringsSep(" ", originalConflictingRefs), + concatStringsSep(" ", newConflictingRefs), conflictError.priority, conflictError.priority - 1, conflictError.priority + 1 @@ -491,7 +510,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem if (!matches(*store, element, i, matchers)) { newManifest.elements.push_back(std::move(element)); } else { - notice("removing '%s'", element.describe()); + notice("removing '%s'", element.identifier()); } } @@ -569,14 +588,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf assert(infop); auto & info = *infop; - if (element.source->resolvedRef == info.flake.resolvedRef) continue; + if (element.source->lockedRef == info.flake.lockedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, info.flake.resolvedRef); + element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); element.source = ProfileElementSource { .originalRef = installable->flakeRef, - .resolvedRef = info.flake.resolvedRef, + .lockedRef = info.flake.lockedRef, .attrPath = info.value.attrPath, .outputs = installable->extendedOutputsSpec, }; @@ -616,7 +635,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } }; -struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile, MixJSON { std::string description() override { @@ -634,12 +653,22 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro { ProfileManifest manifest(*getEvalState(), *profile); - for (size_t i = 0; i < manifest.elements.size(); ++i) { - auto & element(manifest.elements[i]); - logger->cout("%d %s %s %s", i, - element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", - element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", - concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + if (json) { + std::cout << manifest.toJSON(*store).dump() << "\n"; + } else { + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + if (i) logger->cout(""); + logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", + i, + element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); + if (element.source) { + logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); + logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); + logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); + } + logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + } } } }; @@ -787,9 +816,10 @@ struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRu void run(ref<Store> store) override { - if (minAge) - deleteGenerationsOlderThan(*profile, *minAge, dryRun); - else + if (minAge) { + auto t = parseOlderThanTimeSpec(*minAge); + deleteGenerationsOlderThan(*profile, t, dryRun); + } else deleteOldGenerations(*profile, dryRun); } }; diff --git a/src/nix/search.md b/src/nix/search.md index 4caa90654..0c5d22549 100644 --- a/src/nix/search.md +++ b/src/nix/search.md @@ -52,12 +52,12 @@ R""( * Search for packages containing `neovim` but hide ones containing either `gui` or `python`: ```console - # nix search nixpkgs neovim -e 'python|gui' + # nix search nixpkgs neovim --exclude 'python|gui' ``` or ```console - # nix search nixpkgs neovim -e 'python' -e 'gui' + # nix search nixpkgs neovim --exclude 'python' --exclude 'gui' ``` # Description diff --git a/src/nix/shell.md b/src/nix/shell.md index 13a389103..1668104b1 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -19,26 +19,26 @@ R""( * Run GNU Hello: ```console - # nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!' + # nix shell nixpkgs#hello --command hello --greeting 'Hi everybody!' Hi everybody! ``` * Run multiple commands in a shell environment: ```console - # nix shell nixpkgs#gnumake -c sh -c "cd src && make" + # nix shell nixpkgs#gnumake --command sh --command "cd src && make" ``` * Run GNU Hello in a chroot store: ```console - # nix shell --store ~/my-nix nixpkgs#hello -c hello + # nix shell --store ~/my-nix nixpkgs#hello --command hello ``` * Start a shell providing GNU Hello in a chroot store: ```console - # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash + # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive --command bash ``` Note that it's necessary to specify `bash` explicitly because your diff --git a/src/nix/store-ls.md b/src/nix/store-ls.md index 836efce42..14c4627c9 100644 --- a/src/nix/store-ls.md +++ b/src/nix/store-ls.md @@ -5,7 +5,7 @@ R""( * To list the contents of a store path in a binary cache: ```console - # nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + # nix store ls --store https://cache.nixos.org/ --long --recursive /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 dr-xr-xr-x 0 ./bin -r-xr-xr-x 38184 ./bin/hello dr-xr-xr-x 0 ./share @@ -15,7 +15,7 @@ R""( * To show information about a specific file in a binary cache: ```console - # nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello + # nix store ls --store https://cache.nixos.org/ --long /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello -r-xr-xr-x 38184 hello ``` diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 3997c98bf..d05c23fb7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -146,7 +146,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest(storePathsUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique<EvalState>(Strings(), store); + auto state = std::make_unique<EvalState>(SearchPath{}, store); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index 08757aebd..cce88c397 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -11,7 +11,7 @@ R""( * Upgrade Nix in a specific profile: ```console - # nix upgrade-nix -p ~alice/.local/state/nix/profiles/profile + # nix upgrade-nix --profile ~alice/.local/state/nix/profiles/profile ``` # Description diff --git a/src/nix/verify.md b/src/nix/verify.md index cc1122c02..e1d55eab4 100644 --- a/src/nix/verify.md +++ b/src/nix/verify.md @@ -12,7 +12,7 @@ R""( signatures: ```console - # nix store verify -r -n2 --no-contents $(type -p firefox) + # nix store verify --recursive --sigs-needed 2 --no-contents $(type -p firefox) ``` * Verify a store path in the binary cache `https://cache.nixos.org/`: diff --git a/tests/build.sh b/tests/build.sh index 697aff0f9..8ae20f0df 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -129,3 +129,7 @@ nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status (.drvPath | match(".*multiple-outputs-e.drv")) and (.outputs | keys == ["a_a", "b"])) ' + +# Make sure that `--stdin` works and does not apply any defaults +printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []' +printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")' diff --git a/tests/check.sh b/tests/check.sh index 645b90222..e13abf747 100644 --- a/tests/check.sh +++ b/tests/check.sh @@ -18,6 +18,9 @@ clearStore nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check +# Build failure exit codes (100, 104, etc.) are from +# doc/manual/src/command-ref/status-build-failure.md + # check for dangling temporary build directories # only retain if build fails and --keep-failed is specified, or... # ...build is non-deterministic and --check and --keep-failed are both specified diff --git a/tests/eval.sh b/tests/eval.sh index 066d8fc36..b81bb1e2c 100644 --- a/tests/eval.sh +++ b/tests/eval.sh @@ -35,3 +35,9 @@ nix-instantiate --eval -E 'assert 1 + 2 == 3; true' # Check that symlink cycles don't cause a hang. ln -sfn cycle.nix $TEST_ROOT/cycle.nix (! nix eval --file $TEST_ROOT/cycle.nix) + +# Check that relative symlinks are resolved correctly. +mkdir -p $TEST_ROOT/xyzzy $TEST_ROOT/foo +ln -sfn ../xyzzy $TEST_ROOT/foo/bar +printf 123 > $TEST_ROOT/xyzzy/default.nix +[[ $(nix eval --impure --expr "import $TEST_ROOT/foo/bar") = 123 ]] diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index a207f647c..a02d1ce7a 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -5,6 +5,12 @@ enableFeatures "fetch-closure" clearStore clearCacheCache +# Old daemons don't properly zero out the self-references when +# calculating the CA hashes, so this breaks `nix store +# make-content-addressed` which expects the client and the daemon to +# compute the same hash +requireDaemonNewerThan "2.16.0pre20230524" + # Initialize binary cache. nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out) caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') @@ -27,20 +33,43 @@ clearStore [ ! -e $nonCaPath ] [ -e $caPath ] +clearStore + +# The daemon will reject input addressed paths unless configured to trust the +# cache key or the user. This behavior should be covered by another test, so we +# skip this part when using the daemon. if [[ "$NIX_REMOTE" != "daemon" ]]; then - # In impure mode, we can use non-CA paths. - [[ $(nix eval --raw --no-require-sigs --impure --expr " + # If we want to return a non-CA path, we have to be explicit about it. + expectStderr 1 nix eval --raw --no-require-sigs --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + } + " | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false." + + # TODO: Should the closure be rejected, despite single user mode? + # [ ! -e $nonCaPath ] + + [ ! -e $caPath ] + + # We can use non-CA paths when we ask explicitly. + [[ $(nix eval --raw --no-require-sigs --expr " builtins.fetchClosure { fromStore = \"file://$cacheDir\"; fromPath = $nonCaPath; + inputAddressed = true; } ") = $nonCaPath ]] [ -e $nonCaPath ] + [ ! -e $caPath ] + fi +[ ! -e $caPath ] + # 'toPath' set to empty string should fail but print the expected path. expectStderr 1 nix eval -v --json --expr " builtins.fetchClosure { @@ -53,6 +82,10 @@ expectStderr 1 nix eval -v --json --expr " # If fromPath is CA, then toPath isn't needed. nix copy --to file://$cacheDir $caPath +clearStore + +[ ! -e $caPath ] + [[ $(nix eval -v --raw --expr " builtins.fetchClosure { fromStore = \"file://$cacheDir\"; @@ -60,6 +93,8 @@ nix copy --to file://$cacheDir $caPath } ") = $caPath ]] +[ -e $caPath ] + # Check that URL query parameters aren't allowed. clearStore narCache=$TEST_ROOT/nar-cache @@ -71,3 +106,45 @@ rm -rf $narCache } ") (! [ -e $narCache ]) + +# If toPath is specified but wrong, we check it (only) when the path is missing. +clearStore + +badPath=$(echo $caPath | sed -e 's!/store/................................-!/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-!') + +[ ! -e $badPath ] + +expectStderr 1 nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = $badPath; + } +" | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath.*while.*$badPath.*was expected" + +[ ! -e $badPath ] + +# We only check it when missing, as a performance optimization similar to what we do for fixed output derivations. So if it's already there, we don't check it. +# It would be nice for this to fail, but checking it would be too(?) slow. +[ -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $badPath; + toPath = $caPath; + } +") = $caPath ]] + + +# However, if the output address is unexpected, we can report it + + +expectStderr 1 nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + inputAddressed = true; + } +" | grepQuiet 'error.*The store object referred to by.*fromPath.* at .* is not input-addressed, but .*inputAddressed.* is set to .*true.*' + diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index e2ccb0e97..418b4f63f 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -105,6 +105,8 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [[ $(cat $path2/dir1/foo) = foo ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyRev") = "${rev2}-dirty" ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyShortRev") = "${rev2:0:7}-dirty" ]] # ... unless we're using an explicit ref or rev. path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") @@ -119,6 +121,10 @@ git -C $repo commit -m 'Bla3' -a path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") [[ $path2 = $path4 ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"rev\" (builtins.fetchGit $repo)") == "true" ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyRev\" (builtins.fetchGit $repo)") == "false" ]] +[[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyShortRev\" (builtins.fetchGit $repo)") == "false" ]] + status=0 nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" || status=$? [[ "$status" = "102" ]] diff --git a/tests/flakes/check.sh b/tests/flakes/check.sh index 865ca61b4..0433e5335 100644 --- a/tests/flakes/check.sh +++ b/tests/flakes/check.sh @@ -27,6 +27,18 @@ EOF cat > $flakeDir/flake.nix <<EOF { + outputs = { self, ... }: { + overlays.x86_64-linux.foo = final: prev: { + }; + }; +} +EOF + +checkRes=$(nix flake check $flakeDir 2>&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "error: overlay is not a function, but a set instead" + +cat > $flakeDir/flake.nix <<EOF +{ outputs = { self }: { nixosModules.foo = { a.b.c = 123; @@ -72,6 +84,8 @@ cat > $flakeDir/flake.nix <<EOF } EOF -checkRes=$(nix flake check --keep-going $flakeDir 2>&1 && fail "nix flake check should have failed" || true) +nix flake check $flakeDir + +checkRes=$(nix flake check --all-systems --keep-going $flakeDir 2>&1 && fail "nix flake check --all-systems should have failed" || true) echo "$checkRes" | grepQuiet "packages.system-1.default" echo "$checkRes" | grepQuiet "packages.system-2.default" diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index f2e216435..128f759ea 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -95,11 +95,16 @@ json=$(nix flake metadata flake1 --json | jq .) [[ $(echo "$json" | jq -r .lastModified) = $(git -C $flake1Dir log -n1 --format=%ct) ]] hash1=$(echo "$json" | jq -r .revision) +echo foo > $flake1Dir/foo +git -C $flake1Dir add $flake1Dir/foo +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] + echo -n '# foo' >> $flake1Dir/flake.nix flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) git -C $flake1Dir commit -a -m 'Foo' flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] [[ $hash1 != $hash2 ]] # Test 'nix build' on a flake. diff --git a/tests/gc.sh b/tests/gc.sh index 98d6cb032..ad09a8b39 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -50,20 +50,3 @@ if test -e $outPath/foobar; then false; fi # Check that the store is empty. rmdir $NIX_STORE_DIR/.links rmdir $NIX_STORE_DIR - -## Test `nix-collect-garbage -d` -# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're -# using them -if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then - clearProfiles - # Run two `nix-env` commands, should create two generations of - # the profile - nix-env -f ./user-envs.nix -i foo-1.0 - nix-env -f ./user-envs.nix -i foo-2.0pre1 - [[ $(nix-env --list-generations | wc -l) -eq 2 ]] - - # Clear the profile history. There should be only one generation - # left - nix-collect-garbage -d - [[ $(nix-env --list-generations | wc -l) -eq 1 ]] -fi diff --git a/tests/lang/eval-fail-bad-antiquote-1.nix b/tests/lang/eval-fail-bad-string-interpolation-1.nix index ffe9c983c..ffe9c983c 100644 --- a/tests/lang/eval-fail-bad-antiquote-1.nix +++ b/tests/lang/eval-fail-bad-string-interpolation-1.nix diff --git a/tests/lang/eval-fail-bad-antiquote-2.nix b/tests/lang/eval-fail-bad-string-interpolation-2.nix index 3745235ce..3745235ce 100644 --- a/tests/lang/eval-fail-bad-antiquote-2.nix +++ b/tests/lang/eval-fail-bad-string-interpolation-2.nix diff --git a/tests/lang/eval-fail-bad-antiquote-3.nix b/tests/lang/eval-fail-bad-string-interpolation-3.nix index 65b9d4f50..65b9d4f50 100644 --- a/tests/lang/eval-fail-bad-antiquote-3.nix +++ b/tests/lang/eval-fail-bad-string-interpolation-3.nix diff --git a/tests/lang/eval-fail-fromTOML-timestamps.nix b/tests/lang/eval-fail-fromTOML-timestamps.nix new file mode 100644 index 000000000..74cff9470 --- /dev/null +++ b/tests/lang/eval-fail-fromTOML-timestamps.nix @@ -0,0 +1,130 @@ +builtins.fromTOML '' + key = "value" + bare_key = "value" + bare-key = "value" + 1234 = "value" + + "127.0.0.1" = "value" + "character encoding" = "value" + "ʎǝʞ" = "value" + 'key2' = "value" + 'quoted "value"' = "value" + + name = "Orange" + + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + + # This is legal according to the spec, but cpptoml doesn't handle it. + #a.b.c = 1 + #a.d = 2 + + str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + + int1 = +99 + int2 = 42 + int3 = 0 + int4 = -17 + int5 = 1_000 + int6 = 5_349_221 + int7 = 1_2_3_4_5 + + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef + + oct1 = 0o01234567 + oct2 = 0o755 + + bin1 = 0b11010110 + + flt1 = +1.0 + flt2 = 3.1415 + flt3 = -0.01 + flt4 = 5e+22 + flt5 = 1e6 + flt6 = -2E-2 + flt7 = 6.626e-34 + flt8 = 9_224_617.445_991_228_313 + + bool1 = true + bool2 = false + + odt1 = 1979-05-27T07:32:00Z + odt2 = 1979-05-27T00:32:00-07:00 + odt3 = 1979-05-27T00:32:00.999999-07:00 + odt4 = 1979-05-27 07:32:00Z + ldt1 = 1979-05-27T07:32:00 + ldt2 = 1979-05-27T00:32:00.999999 + ld1 = 1979-05-27 + lt1 = 07:32:00 + lt2 = 00:32:00.999999 + + arr1 = [ 1, 2, 3 ] + arr2 = [ "red", "yellow", "green" ] + arr3 = [ [ 1, 2 ], [3, 4, 5] ] + arr4 = [ "all", 'strings', """are the same""", ''''type''''] + arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] + + arr7 = [ + 1, 2, 3 + ] + + arr8 = [ + 1, + 2, # this is ok + ] + + [table-1] + key1 = "some string" + key2 = 123 + + + [table-2] + key1 = "another string" + key2 = 456 + + [dog."tater.man"] + type.name = "pug" + + [a.b.c] + [ d.e.f ] + [ g . h . i ] + [ j . "ʞ" . 'l' ] + [x.y.z.w] + + name = { first = "Tom", last = "Preston-Werner" } + point = { x = 1, y = 2 } + animal = { type.name = "pug" } + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] + + [[products]] + name = "Nail" + sku = 284758393 + color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" +'' diff --git a/tests/lang/eval-okay-fromTOML-timestamps.exp b/tests/lang/eval-okay-fromTOML-timestamps.exp new file mode 100644 index 000000000..08b3c69a6 --- /dev/null +++ b/tests/lang/eval-okay-fromTOML-timestamps.exp @@ -0,0 +1 @@ +{ "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; ld1 = { _type = "timestamp"; value = "1979-05-27"; }; ldt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00"; }; ldt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999"; }; lt1 = { _type = "timestamp"; value = "07:32:00"; }; lt2 = { _type = "timestamp"; value = "00:32:00.999999"; }; name = "Orange"; oct1 = 342391; oct2 = 493; odt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; odt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00-07:00"; }; odt3 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999-07:00"; }; odt4 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } diff --git a/tests/lang/eval-okay-fromTOML-timestamps.flags b/tests/lang/eval-okay-fromTOML-timestamps.flags new file mode 100644 index 000000000..9ed39dc6b --- /dev/null +++ b/tests/lang/eval-okay-fromTOML-timestamps.flags @@ -0,0 +1 @@ +--extra-experimental-features parse-toml-timestamps diff --git a/tests/lang/eval-okay-fromTOML-timestamps.nix b/tests/lang/eval-okay-fromTOML-timestamps.nix new file mode 100644 index 000000000..74cff9470 --- /dev/null +++ b/tests/lang/eval-okay-fromTOML-timestamps.nix @@ -0,0 +1,130 @@ +builtins.fromTOML '' + key = "value" + bare_key = "value" + bare-key = "value" + 1234 = "value" + + "127.0.0.1" = "value" + "character encoding" = "value" + "ʎǝʞ" = "value" + 'key2' = "value" + 'quoted "value"' = "value" + + name = "Orange" + + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + + # This is legal according to the spec, but cpptoml doesn't handle it. + #a.b.c = 1 + #a.d = 2 + + str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + + int1 = +99 + int2 = 42 + int3 = 0 + int4 = -17 + int5 = 1_000 + int6 = 5_349_221 + int7 = 1_2_3_4_5 + + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef + + oct1 = 0o01234567 + oct2 = 0o755 + + bin1 = 0b11010110 + + flt1 = +1.0 + flt2 = 3.1415 + flt3 = -0.01 + flt4 = 5e+22 + flt5 = 1e6 + flt6 = -2E-2 + flt7 = 6.626e-34 + flt8 = 9_224_617.445_991_228_313 + + bool1 = true + bool2 = false + + odt1 = 1979-05-27T07:32:00Z + odt2 = 1979-05-27T00:32:00-07:00 + odt3 = 1979-05-27T00:32:00.999999-07:00 + odt4 = 1979-05-27 07:32:00Z + ldt1 = 1979-05-27T07:32:00 + ldt2 = 1979-05-27T00:32:00.999999 + ld1 = 1979-05-27 + lt1 = 07:32:00 + lt2 = 00:32:00.999999 + + arr1 = [ 1, 2, 3 ] + arr2 = [ "red", "yellow", "green" ] + arr3 = [ [ 1, 2 ], [3, 4, 5] ] + arr4 = [ "all", 'strings', """are the same""", ''''type''''] + arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] + + arr7 = [ + 1, 2, 3 + ] + + arr8 = [ + 1, + 2, # this is ok + ] + + [table-1] + key1 = "some string" + key2 = 123 + + + [table-2] + key1 = "another string" + key2 = 456 + + [dog."tater.man"] + type.name = "pug" + + [a.b.c] + [ d.e.f ] + [ g . h . i ] + [ j . "ʞ" . 'l' ] + [x.y.z.w] + + name = { first = "Tom", last = "Preston-Werner" } + point = { x = 1, y = 2 } + animal = { type.name = "pug" } + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] + + [[products]] + name = "Nail" + sku = 284758393 + color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" +'' diff --git a/tests/lang/eval-okay-path-antiquotation.exp b/tests/lang/eval-okay-path-string-interpolation.exp index 5b8ea0243..5b8ea0243 100644 --- a/tests/lang/eval-okay-path-antiquotation.exp +++ b/tests/lang/eval-okay-path-string-interpolation.exp diff --git a/tests/lang/eval-okay-path-antiquotation.nix b/tests/lang/eval-okay-path-string-interpolation.nix index 497d7c1c7..497d7c1c7 100644 --- a/tests/lang/eval-okay-path-antiquotation.nix +++ b/tests/lang/eval-okay-path-string-interpolation.nix diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/lang/eval-okay-replacestrings.exp index 72e8274d8..eac67c5fe 100644 --- a/tests/lang/eval-okay-replacestrings.exp +++ b/tests/lang/eval-okay-replacestrings.exp @@ -1 +1 @@ -[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" ] +[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" "fubar" ] diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/lang/eval-okay-replacestrings.nix index bd8031fc0..a803e6519 100644 --- a/tests/lang/eval-okay-replacestrings.nix +++ b/tests/lang/eval-okay-replacestrings.nix @@ -8,4 +8,5 @@ with builtins; (replaceStrings [""] ["X"] "abc") (replaceStrings [""] ["X"] "") (replaceStrings ["-"] ["_"] "a-b") + (replaceStrings ["oo" "XX"] ["u" (throw "unreachable")] "foobar") ] diff --git a/tests/linux-sandbox-cert-test.nix b/tests/linux-sandbox-cert-test.nix new file mode 100644 index 000000000..2fc083ea9 --- /dev/null +++ b/tests/linux-sandbox-cert-test.nix @@ -0,0 +1,30 @@ +{ mode }: + +with import ./config.nix; + +mkDerivation ( + { + name = "ssl-export"; + buildCommand = '' + # Add some indirection, otherwise grepping into the debug output finds the string. + report () { echo CERT_$1_IN_SANDBOX; } + + if [ -f /etc/ssl/certs/ca-certificates.crt ]; then + content=$(</etc/ssl/certs/ca-certificates.crt) + if [ "$content" == CERT_CONTENT ]; then + report present + fi + else + report missing + fi + + # Always fail, because we do not want to bother with fixed-output + # derivations being cached, and do not want to compute the right hash. + false; + ''; + } // { + fixed-output = { outputHash = "sha256:0000000000000000000000000000000000000000000000000000000000000000"; }; + normal = { }; + }.${mode} +) + diff --git a/tests/linux-sandbox.sh b/tests/linux-sandbox.sh index 5a2cf7abd..ff7d257bd 100644 --- a/tests/linux-sandbox.sh +++ b/tests/linux-sandbox.sh @@ -11,6 +11,8 @@ requireSandboxSupport # otherwise things get complicated (e.g. if it's in /bin, do we need # /lib as well?). if [[ ! $SHELL =~ /nix/store ]]; then skipTest "Shell is not from Nix store"; fi +# An alias to automatically bind-mount the $SHELL on nix-build invocations +nix-sandbox-build () { nix-build --no-out-link --sandbox-paths /nix/store "$@"; } chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 @@ -18,7 +20,7 @@ rm -rf $TEST_ROOT/store0 export NIX_STORE_DIR=/my/store export NIX_REMOTE=$TEST_ROOT/store0 -outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store) +outPath=$(nix-sandbox-build dependencies.nix) [[ $outPath =~ /my/store/.*-dependencies ]] @@ -29,14 +31,45 @@ nix store ls -R -l $outPath | grep foobar nix store cat $outPath/foobar | grep FOOBAR # Test --check without hash rewriting. -nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store +nix-sandbox-build dependencies.nix --check # Test that sandboxed builds with --check and -K can move .check directory to store -nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link +nix-sandbox-build check.nix -A nondeterministic -(! nix-build check.nix -A nondeterministic --sandbox-paths /nix/store --no-out-link --check -K 2> $TEST_ROOT/log) -if grepQuiet 'error: renaming' $TEST_ROOT/log; then false; fi +# `100 + 4` means non-determinstic, see doc/manual/src/command-ref/status-build-failure.md +expectStderr 104 nix-sandbox-build check.nix -A nondeterministic --check -K > $TEST_ROOT/log +grepQuietInverse 'error: renaming' $TEST_ROOT/log grepQuiet 'may not be deterministic' $TEST_ROOT/log # Test that sandboxed builds cannot write to /etc easily -(! nix-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' --no-out-link --sandbox-paths /nix/store) +# `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md +expectStderr 100 nix-sandbox-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' | + grepQuiet "/etc/test: Permission denied" + + +## Test mounting of SSL certificates into the sandbox +testCert () { + expectation=$1 # "missing" | "present" + mode=$2 # "normal" | "fixed-output" + certFile=$3 # a string that can be the path to a cert file + # `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md + [ "$mode" == fixed-output ] && ret=1 || ret=100 + expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | + grepQuiet "CERT_${expectation}_IN_SANDBOX" +} + +nocert=$TEST_ROOT/no-cert-file.pem +cert=$TEST_ROOT/some-cert-file.pem +echo -n "CERT_CONTENT" > $cert + +# No cert in sandbox when not a fixed-output derivation +testCert missing normal "$cert" + +# No cert in sandbox when ssl-cert-file is empty +testCert missing fixed-output "" + +# No cert in sandbox when ssl-cert-file is a nonexistent file +testCert missing fixed-output "$nocert" + +# Cert in sandbox when ssl-cert-file is set to an existing file +testCert present fixed-output "$cert" diff --git a/tests/local.mk b/tests/local.mk index 9cb81e1f0..2be1081f7 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -16,6 +16,7 @@ nix_tests = \ flakes/flake-in-submodule.sh \ ca/gc.sh \ gc.sh \ + nix-collect-garbage-d.sh \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ @@ -135,7 +136,9 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - toString-path.sh + test-libstoreconsumer.sh \ + toString-path.sh \ + read-only-store.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh @@ -153,6 +156,7 @@ test-deps += \ tests/common/vars-and-functions.sh \ tests/config.nix \ tests/ca/config.nix \ + tests/test-libstoreconsumer/test-libstoreconsumer \ tests/dyn-drv/config.nix ifeq ($(BUILD_SHARED_LIBS), 1) diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index dbb3114f1..b5d935004 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -8,6 +8,7 @@ rm -f $TEST_HOME/.nix-channels $TEST_HOME/.nix-profile nix-channel --add http://foo/bar xyzzy nix-channel --list | grepQuiet http://foo/bar nix-channel --remove xyzzy +[[ $(nix-channel --list-generations | wc -l) == 1 ]] [ -e $TEST_HOME/.nix-channels ] [ "$(cat $TEST_HOME/.nix-channels)" = '' ] @@ -38,6 +39,7 @@ ln -s dependencies.nix $TEST_ROOT/nixexprs/default.nix # Test the update action. nix-channel --add file://$TEST_ROOT/foo nix-channel --update +[[ $(nix-channel --list-generations | wc -l) == 2 ]] # Do a query. nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml diff --git a/tests/nix-collect-garbage-d.sh b/tests/nix-collect-garbage-d.sh new file mode 100644 index 000000000..bf30f8938 --- /dev/null +++ b/tests/nix-collect-garbage-d.sh @@ -0,0 +1,40 @@ +source common.sh + +clearStore + +## Test `nix-collect-garbage -d` + +# TODO make `nix-env` doesn't work with CA derivations, and make +# `ca/nix-collect-garbage-d.sh` wrapper. + +testCollectGarbageD () { + clearProfiles + # Run two `nix-env` commands, should create two generations of + # the profile + nix-env -f ./user-envs.nix -i foo-1.0 "$@" + nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@" + [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]] + + # Clear the profile history. There should be only one generation + # left + nix-collect-garbage -d + [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]] +} + +testCollectGarbageD + +# Run the same test, but forcing the profiles an arbitrary location. +rm ~/.nix-profile +ln -s $TEST_ROOT/blah ~/.nix-profile +testCollectGarbageD + +# Run the same test, but forcing the profiles at their legacy location under +# /nix/var/nix. +# +# Note that we *don't* use the default profile; `nix-collect-garbage` will +# need to check the legacy conditional unconditionally not just follow +# `~/.nix-profile` to pass this test. +# +# Regression test for #8294 +rm ~/.nix-profile +testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me" diff --git a/tests/nix-profile.sh b/tests/nix-profile.sh index 4ef5b484a..7c478a0cd 100644 --- a/tests/nix-profile.sh +++ b/tests/nix-profile.sh @@ -47,8 +47,9 @@ cp ./config.nix $flake1Dir/ # Test upgrading from nix-env. nix-env -f ./user-envs.nix -i foo-1.0 -nix profile list | grep '0 - - .*-foo-1.0' +nix profile list | grep -A2 'Index:.*0' | grep 'Store paths:.*foo-1.0' nix profile install $flake1Dir -L +nix profile list | grep -A4 'Index:.*1' | grep 'Locked flake URL:.*narHash' [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) @@ -157,17 +158,17 @@ error: An existing package already provides the following file: To remove the existing package: - nix profile remove path:${flake1Dir} + nix profile remove path:${flake1Dir}#packages.${system}.default The new package can also be installed next to the existing one by assigning a different priority. The conflicting packages have a priority of 5. To prioritise the new package: - nix profile install path:${flake2Dir} --priority 4 + nix profile install path:${flake2Dir}#packages.${system}.default --priority 4 To prioritise the existing package: - nix profile install path:${flake2Dir} --priority 6 + nix profile install path:${flake2Dir}#packages.${system}.default --priority 6 EOF ) [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] @@ -177,3 +178,10 @@ nix profile install $flake2Dir --priority 0 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]] # nix profile install $flake1Dir --priority 100 # [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] + +# Ensure that conflicts are handled properly even when the installables aren't +# flake references. +# Regression test for https://github.com/NixOS/nix/issues/8284 +clearProfiles +nix profile install $(nix build $flake1Dir --no-link --print-out-paths) +expect 1 nix profile install --impure --expr "(builtins.getFlake ''$flake2Dir'').packages.$system.default" diff --git a/tests/nixos/authorization.nix b/tests/nixos/authorization.nix index 7e8744dd9..fdeae06ed 100644 --- a/tests/nixos/authorization.nix +++ b/tests/nixos/authorization.nix @@ -75,5 +75,20 @@ su --login bob -c '(! nix-store --verify --repair 2>&1)' | tee diag 1>&2 grep -F "you are not privileged to repair paths" diag """) + + machine.succeed(""" + set -x + su --login mallory -c ' + nix-store --generate-binary-cache-key cache1.example.org sk1 pk1 + (! nix store sign --key-file sk1 ${pathFour} 2>&1)' | tee diag 1>&2 + grep -F "cannot open connection to remote store 'daemon'" diag + """) + + machine.succeed(""" + su --login bob -c ' + nix-store --generate-binary-cache-key cache1.example.org sk1 pk1 + nix store sign --key-file sk1 ${pathFour} + ' + """) ''; } diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index ee8b77100..16c477bf9 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -23,6 +23,12 @@ in { nix.settings.substituters = lib.mkForce [ ]; nix.settings.experimental-features = [ "nix-command" ]; services.getty.autologinUser = "root"; + programs.ssh.extraConfig = '' + Host * + ControlMaster auto + ControlPath ~/.ssh/master-%h:%r@%n:%p + ControlPersist 15m + ''; }; server = @@ -62,6 +68,10 @@ in { client.wait_for_text("done") server.succeed("nix-store --check-validity ${pkgA}") + # Check that ControlMaster is working + client.send_chars("nix copy --to ssh://server ${pkgA} >&2; echo done\n") + client.wait_for_text("done") + client.copy_from_host("key", "/root/.ssh/id_ed25519") client.succeed("chmod 600 /root/.ssh/id_ed25519") diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix new file mode 100644 index 000000000..1d43a5d04 --- /dev/null +++ b/tests/nixos/tarball-flakes.nix @@ -0,0 +1,84 @@ +{ lib, config, nixpkgs, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + root = pkgs.runCommand "nixpkgs-flake" {} + '' + mkdir -p $out/stable + + set -x + dir=nixpkgs-${nixpkgs.shortRev} + cp -prd ${nixpkgs} $dir + # Set the correct timestamp in the tarball. + find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + tar cfz $out/stable/${nixpkgs.rev}.tar.gz $dir --hard-dereference + + echo 'Redirect "/latest.tar.gz" "/stable/${nixpkgs.rev}.tar.gz"' > $out/.htaccess + + echo 'Header set Link "<http://localhost/stable/${nixpkgs.rev}.tar.gz?rev=${nixpkgs.rev}&revCount=1234>; rel=\"immutable\""' > $out/stable/.htaccess + ''; +in + +{ + name = "tarball-flakes"; + + nodes = + { + machine = + { config, pkgs, ... }: + { networking.firewall.allowedTCPPorts = [ 80 ]; + + services.httpd.enable = true; + services.httpd.adminAddr = "foo@example.org"; + services.httpd.extraConfig = '' + ErrorLog syslog:local6 + ''; + services.httpd.virtualHosts."localhost" = + { servedDirs = + [ { urlPath = "/"; + dir = root; + } + ]; + }; + + virtualisation.writableStore = true; + virtualisation.diskSize = 2048; + virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ]; + virtualisation.memorySize = 4096; + nix.settings.substituters = lib.mkForce [ ]; + nix.extraOptions = "experimental-features = nix-command flakes"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import json + + start_all() + + machine.wait_for_unit("httpd.service") + + out = machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz") + print(out) + info = json.loads(out) + + # Check that we got redirected to the immutable URL. + assert info["locked"]["url"] == "http://localhost/stable/${nixpkgs.rev}.tar.gz" + + # Check that we got the rev and revCount attributes. + assert info["revision"] == "${nixpkgs.rev}" + assert info["revCount"] == 1234 + + # Check that fetching with rev/revCount/narHash succeeds. + machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?rev=" + info["revision"]) + machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?revCount=" + str(info["revCount"])) + machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?narHash=" + info["locked"]["narHash"]) + + # Check that fetching fails if we provide incorrect attributes. + machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?rev=493300eb13ae6fb387fbd47bf54a85915acc31c0") + machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?revCount=789") + machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?narHash=sha256-tbudgBSg+bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw=") + ''; + +} diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc index 04b791021..e02fd68d5 100644 --- a/tests/plugins/plugintest.cc +++ b/tests/plugins/plugintest.cc @@ -21,4 +21,8 @@ static void prim_anotherNull (EvalState & state, const PosIdx pos, Value ** args v.mkBool(false); } -static RegisterPrimOp rp("anotherNull", 0, prim_anotherNull); +static RegisterPrimOp rp({ + .name = "anotherNull", + .arity = 0, + .fun = prim_anotherNull, +}); diff --git a/tests/read-only-store.sh b/tests/read-only-store.sh new file mode 100644 index 000000000..d63920c19 --- /dev/null +++ b/tests/read-only-store.sh @@ -0,0 +1,42 @@ +source common.sh + +enableFeatures "read-only-local-store" + +needLocalStore "cannot open store read-only when daemon has already opened it writeable" + +clearStore + +happy () { + # We can do a read-only query just fine with a read-only store + nix --store local?read-only=true path-info $dummyPath + + # We can "write" an already-present store-path a read-only store, because no IO is actually required + nix-store --store local?read-only=true --add dummy +} +## Testing read-only mode without forcing the underlying store to actually be read-only + +# Make sure the command fails when the store doesn't already have a database +expectStderr 1 nix-store --store local?read-only=true --add dummy | grepQuiet "database does not exist, and cannot be created in read-only mode" + +# Make sure the store actually has a current-database, with at least one store object +dummyPath=$(nix-store --add dummy) + +# Try again and make sure we fail when adding a item not already in the store +expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "attempt to write a readonly database" + +# Test a few operations that should work with the read-only store in its current state +happy + +## Testing read-only mode with an underlying store that is actually read-only + +# Ensure store is actually read-only +chmod -R -w $TEST_ROOT/store +chmod -R -w $TEST_ROOT/var + +# Make sure we fail on add operations on the read-only store +# This is only for adding files that are not *already* in the store +expectStderr 1 nix-store --add eval.nix | grepQuiet "error: opening lock file '$(readlink -e $TEST_ROOT)/var/nix/db/big-lock'" +expectStderr 1 nix-store --store local?read-only=true --add eval.nix | grepQuiet "Permission denied" + +# Test the same operations from before should again succeed +happy diff --git a/tests/recursive.sh b/tests/recursive.sh index ffeb44e50..0bf00f8fa 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -1,8 +1,5 @@ source common.sh -# FIXME -if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi - enableFeatures 'recursive-nix' restartDaemon diff --git a/tests/signing.sh b/tests/signing.sh index 9b673c609..942b51630 100644 --- a/tests/signing.sh +++ b/tests/signing.sh @@ -84,6 +84,10 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) # Copying to a diverted store should fail due to a lack of signatures by trusted keys. chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 + +# Fails or very flaky only on GHA + macOS: +# expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' +# but this works: (! nix copy --to $TEST_ROOT/store0 $outPath) # But succeed if we supply the public keys. diff --git a/tests/test-libstoreconsumer.sh b/tests/test-libstoreconsumer.sh new file mode 100644 index 000000000..8a77cf5a1 --- /dev/null +++ b/tests/test-libstoreconsumer.sh @@ -0,0 +1,6 @@ +source common.sh + +drv="$(nix-instantiate simple.nix)" +cat "$drv" +out="$(./test-libstoreconsumer/test-libstoreconsumer "$drv")" +cat "$out/hello" | grep -F "Hello World!" diff --git a/tests/test-libstoreconsumer/README.md b/tests/test-libstoreconsumer/README.md new file mode 100644 index 000000000..ded69850f --- /dev/null +++ b/tests/test-libstoreconsumer/README.md @@ -0,0 +1,6 @@ + +A very simple C++ consumer of the libstore library. + + - Keep it simple. Library consumers expect something simple. + - No build hook, or any other reinvocations. + - No more global state than necessary. diff --git a/tests/test-libstoreconsumer/local.mk b/tests/test-libstoreconsumer/local.mk new file mode 100644 index 000000000..edc140723 --- /dev/null +++ b/tests/test-libstoreconsumer/local.mk @@ -0,0 +1,15 @@ +programs += test-libstoreconsumer + +test-libstoreconsumer_DIR := $(d) + +# do not install +test-libstoreconsumer_INSTALL_DIR := + +test-libstoreconsumer_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +test-libstoreconsumer_CXXFLAGS += -I src/libutil -I src/libstore + +test-libstoreconsumer_LIBS = libstore libutil + +test-libstoreconsumer_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) diff --git a/tests/test-libstoreconsumer/main.cc b/tests/test-libstoreconsumer/main.cc new file mode 100644 index 000000000..31b6d8ef1 --- /dev/null +++ b/tests/test-libstoreconsumer/main.cc @@ -0,0 +1,45 @@ +#include "globals.hh" +#include "store-api.hh" +#include "build-result.hh" +#include <iostream> + +using namespace nix; + +int main (int argc, char **argv) +{ + try { + if (argc != 2) { + std::cerr << "Usage: " << argv[0] << " store/path/to/something.drv\n"; + return 1; + } + + std::string drvPath = argv[1]; + + initLibStore(); + + auto store = nix::openStore(); + + // build the derivation + + std::vector<DerivedPath> paths { + DerivedPath::Built { + .drvPath = store->parseStorePath(drvPath), + .outputs = OutputsSpec::Names{"out"} + } + }; + + const auto results = store->buildPathsWithResults(paths, bmNormal, store); + + for (const auto & result : results) { + for (const auto & [outputName, realisation] : result.builtOutputs) { + std::cout << store->printStorePath(realisation.outPath) << "\n"; + } + } + + return 0; + + } catch (const std::exception & e) { + std::cerr << "Error: " << e.what() << "\n"; + return 1; + } +} |