diff options
146 files changed, 2514 insertions, 1104 deletions
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index ec7ab4516..dd481160f 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -8,7 +8,7 @@ jobs: if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} # required to find all branches diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09436b7e3..d01ef4768 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,10 +14,10 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v16 + - uses: cachix/install-nix-action@v17 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/cachix-action@v10 if: needs.check_cachix.outputs.secret == 'true' @@ -46,11 +46,11 @@ jobs: outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3 with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v16 + - uses: cachix/install-nix-action@v17 - uses: cachix/cachix-action@v10 with: name: '${{ env.CACHIX_NAME }}' @@ -67,9 +67,9 @@ jobs: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v16 + - uses: cachix/install-nix-action@v17 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -83,10 +83,10 @@ jobs: needs.check_cachix.outputs.secret == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v16 + - uses: cachix/install-nix-action@v17 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix-instantiate --eval -E '(import ./default.nix).defaultPackage.${builtins.currentSystem}.version' | tr -d \")" >> $GITHUB_ENV - uses: cachix/cachix-action@v10 diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml index b97076bd7..53e69cb2d 100644 --- a/.github/workflows/hydra_status.yml +++ b/.github/workflows/hydra_status.yml @@ -9,7 +9,7 @@ jobs: if: github.repository_owner == 'NixOS' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3 with: fetch-depth: 0 - run: bash scripts/check-hydra-status.sh diff --git a/.gitignore b/.gitignore index 4b290425a..58e7377fb 100644 --- a/.gitignore +++ b/.gitignore @@ -90,6 +90,7 @@ perl/Makefile.config /misc/systemd/nix-daemon.service /misc/systemd/nix-daemon.socket +/misc/systemd/nix-daemon.conf /misc/upstart/nix-daemon.conf /src/resolve-system-dependencies/resolve-system-dependencies diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix index 84d90beb6..2d586fa1b 100644 --- a/doc/manual/generate-options.nix +++ b/doc/manual/generate-options.nix @@ -6,7 +6,8 @@ options: concatStrings (map (name: let option = options.${name}; in - " - `${name}` \n\n" + " - [`${name}`](#conf-${name})" + + "<p id=\"conf-${name}\"></p>\n\n" + concatStrings (map (s: " ${s}\n") (splitLines option.description)) + "\n\n" + (if option.documentDefault then " **Default:** " + ( diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index c4c60db15..b0d7fbf1a 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -110,7 +110,7 @@ default, set it to `-`. 7. A comma-separated list of *mandatory features*. A machine will only be used to build a derivation if all of the machine’s mandatory features appear in the derivation’s `requiredSystemFeatures` - attribute.. + attribute. 8. The (base64-encoded) public host key of the remote machine. If omitted, SSH will use its regular known-hosts file. Specifically, the field is calculated diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 4367654a2..e5fb50088 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -84,7 +84,9 @@ The installer will modify `/etc/bashrc`, and `/etc/zshrc` if they exist. The installer will first back up these files with a `.backup-before-nix` extension. The installer will also create `/etc/profile.d/nix.sh`. -You can uninstall Nix with the following commands: +## Uninstalling + +### Linux ```console sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels @@ -95,15 +97,95 @@ sudo systemctl stop nix-daemon.service sudo systemctl disable nix-daemon.socket sudo systemctl disable nix-daemon.service sudo systemctl daemon-reload - -# If you are on macOS, you will need to run: -sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist -sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist ``` There may also be references to Nix in `/etc/profile`, `/etc/bashrc`, and `/etc/zshrc` which you may remove. +### macOS + +1. Edit `/etc/zshrc` and `/etc/bashrc` to remove the lines sourcing + `nix-daemon.sh`, which should look like this: + + ```bash + # Nix + if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then + . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' + fi + # End Nix + ``` + + If these files haven't been altered since installing Nix you can simply put + the backups back in place: + + ```console + sudo mv /etc/zshrc.backup-before-nix /etc/zshrc + sudo mv /etc/bashrc.backup-before-nix /etc/bashrc + ``` + + This will stop shells from sourcing the file and bringing everything you + installed using Nix in scope. + +2. Stop and remove the Nix daemon services: + + ```console + sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist + sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist + sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist + sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist + ``` + + This stops the Nix daemon and prevents it from being started next time you + boot the system. + +3. Remove the `nixbld` group and the `_nixbuildN` users: + + ```console + sudo dscl . -delete /Groups/nixbld + for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done + ``` + + This will remove all the build users that no longer serve a purpose. + +4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store + volume on `/nix`, which looks like this, + `LABEL=Nix\040Store /nix apfs rw,nobrowse`. This will prevent automatic + mounting of the Nix Store volume. + +5. Edit `/etc/synthetic.conf` to remove the `nix` line. If this is the only + line in the file you can remove it entirely, `sudo rm /etc/synthetic.conf`. + This will prevent the creation of the empty `/nix` directory to provide a + mountpoint for the Nix Store volume. + +6. Remove the files Nix added to your system: + + ```console + sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels + ``` + + This gets rid of any data Nix may have created except for the store which is + removed next. + +7. Remove the Nix Store volume: + + ```console + sudo diskutil apfs deleteVolume /nix + ``` + + This will remove the Nix Store volume and everything that was added to the + store. + +> **Note** +> +> After you complete the steps here, you will still have an empty `/nix` +> directory. This is an expected sign of a successful uninstall. The empty +> `/nix` directory will disappear the next time you reboot. +> +> You do not have to reboot to finish uninstalling Nix. The uninstall is +> complete. macOS (Catalina+) directly controls root directories and its +> read-only root will prevent you from manually deleting the empty `/nix` +> mountpoint. + # macOS Installation <a name="sect-macos-installation-change-store-prefix"></a><a name="sect-macos-installation-encrypted-volume"></a><a name="sect-macos-installation-symlink"></a><a name="sect-macos-installation-recommended-notes"></a> <!-- Note: anchors above to catch permalinks to old explanations --> diff --git a/doc/manual/src/release-notes/rl-2.7.md b/doc/manual/src/release-notes/rl-2.7.md index dc92a9cde..2f3879422 100644 --- a/doc/manual/src/release-notes/rl-2.7.md +++ b/doc/manual/src/release-notes/rl-2.7.md @@ -1,4 +1,4 @@ -# Release X.Y (2022-03-07) +# Release 2.7 (2022-03-07) * Nix will now make some helpful suggestions when you mistype something on the command line. For instance, if you type `nix build diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c869b5e2f..6ebbe5eb4 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1 +1,46 @@ # Release X.Y (202?-??-??) + +* Various nix commands can now read expressions from stdin with `--file -`. + +* `nix store make-content-addressable` has been renamed to `nix store + make-content-addressed`. + +* New experimental builtin function `builtins.fetchClosure` that + copies a closure from a binary cache at evaluation time and rewrites + it to content-addressed form (if it isn't already). Like + `builtins.storePath`, this allows importing pre-built store paths; + the difference is that it doesn't require the user to configure + binary caches and trusted public keys. + + This function is only available if you enable the experimental + feature `fetch-closure`. + +* New experimental feature: *impure derivations*. These are + derivations that can produce a different result every time they're + built. Here is an example: + + ```nix + stdenv.mkDerivation { + name = "impure"; + __impure = true; # marks this derivation as impure + buildCommand = "date > $out"; + } + ``` + + Running `nix build` twice on this expression will build the + derivation twice, producing two different content-addressed store + paths. Like fixed-output derivations, impure derivations have access + to the network. Only fixed-output derivations and impure derivations + can depend on an impure derivation. + +* The `nixosModule` flake output attribute has been renamed consistent + with the `.default` renames in nix 2.7. + + * `nixosModule` → `nixosModules.default` + + As before, the old output will continue to work, but `nix flake check` will + issue a warning about it. + +* `nix run` is now stricter wrt what it accepts: + * Members of `apps` are now required to be apps (as defined in [the manual](https://nixos.org/manual/nix/stable/command-ref/new-cli/nix3-run.html#apps)) + * Member of `packages` or `legacyPackages` cannot be of type "app" when used by `nix run`. diff --git a/flake.lock b/flake.lock index 61eccb73c..cd79fa85e 100644 --- a/flake.lock +++ b/flake.lock @@ -18,11 +18,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1632864508, - "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=", + "lastModified": 1645296114, + "narHash": "sha256-y53N7TyIkXsjMpOG7RhvqJFGDacLs9HlyHeSTBioqYU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234", + "rev": "530a53dcbc9437363471167a5e4762c5fcfa34a1", "type": "github" }, "original": { diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh index 045053dee..9af695f5a 100644 --- a/misc/bash/completion.sh +++ b/misc/bash/completion.sh @@ -15,7 +15,7 @@ function _complete_nix { else COMPREPLY+=("$completion") fi - done < <(NIX_GET_COMPLETIONS=$cword "${words[@]/#\~/$HOME}" 2>/dev/null) + done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}" 2>/dev/null) __ltrim_colon_completions "$cur" } diff --git a/misc/systemd/local.mk b/misc/systemd/local.mk index 1fa037485..76121a0f9 100644 --- a/misc/systemd/local.mk +++ b/misc/systemd/local.mk @@ -1,7 +1,8 @@ ifdef HOST_LINUX $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) + $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/tmpfiles.d, 0644))) - clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service + clean-files += $(d)/nix-daemon.socket $(d)/nix-daemon.service $(d)/nix-daemon.conf endif diff --git a/misc/systemd/nix-daemon.conf.in b/misc/systemd/nix-daemon.conf.in new file mode 100644 index 000000000..e7b264234 --- /dev/null +++ b/misc/systemd/nix-daemon.conf.in @@ -0,0 +1 @@ +d @localstatedir@/nix/daemon-socket 0755 root root - - diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index 25655204d..24d894898 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -1,7 +1,9 @@ [Unit] Description=Nix Daemon +Documentation=man:nix-daemon https://nixos.org/manual RequiresMountsFor=@storedir@ RequiresMountsFor=@localstatedir@ +RequiresMountsFor=@localstatedir@/nix/db ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket [Service] diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh index 5e2f03429..e62705e94 100644 --- a/scripts/check-hydra-status.sh +++ b/scripts/check-hydra-status.sh @@ -14,7 +14,7 @@ curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master someBuildFailed=0 for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") + buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") finished=$(echo "$buildInfo" | jq -r '.finished') diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 5f2c93b7f..b79a9c23a 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -423,6 +423,18 @@ EOF fi done + if [ "$(uname -s)" = "Linux" ] && [ ! -e /run/systemd/system ]; then + warning <<EOF +We did not detect systemd on your system. With a multi-user install +without systemd you will have to manually configure your init system to +launch the Nix daemon after installation. +EOF + if ! ui_confirm "Do you want to proceed with a multi-user installation?"; then + failure <<EOF +You have aborted the installation. +EOF + fi + fi } setup_report() { @@ -739,7 +751,7 @@ install_from_extracted_nix() { cd "$EXTRACTED_NIX_PATH" _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ - cp -RLp ./store/* "$NIX_ROOT/store/" + cp -RPp ./store/* "$NIX_ROOT/store/" _sudo "to make the new store non-writable at $NIX_ROOT/store" \ chmod -R ugo-w "$NIX_ROOT/store/" diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index f4a2dfc5d..62397127a 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -9,6 +9,8 @@ readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket +readonly TMPFILES_SRC=/lib/tmpfiles.d/nix-daemon.conf +readonly TMPFILES_DEST=/etc/tmpfiles.d/nix-daemon.conf # Path for the systemd override unit file to contain the proxy settings readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf @@ -83,6 +85,13 @@ EOF poly_configure_nix_daemon_service() { if [ -e /run/systemd/system ]; then task "Setting up the nix-daemon systemd service" + + _sudo "to create the nix-daemon tmpfiles config" \ + ln -sfn /nix/var/nix/profiles/default/$TMPFILES_SRC $TMPFILES_DEST + + _sudo "to run systemd-tmpfiles once to pick that path up" \ + systemd-tmpfiles --create --prefix=/nix/var/nix + _sudo "to set up the nix-daemon service" \ systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC" diff --git a/scripts/install.in b/scripts/install.in index 38d1fb36f..af5f71080 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -82,7 +82,7 @@ if [ "$(uname -s)" != "Darwin" ]; then fi if command -v curl > /dev/null 2>&1; then - fetch() { curl -L "$1" -o "$2"; } + fetch() { curl --fail -L "$1" -o "$2"; } elif command -v wget > /dev/null 2>&1; then fetch() { wget "$1" -O "$2"; } else diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 8c9133c17..ff8ba2724 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -300,7 +300,7 @@ connected: std::set<Realisation> missingRealisations; StorePathSet missingPaths; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) { for (auto & outputName : wantedOutputs) { auto thisOutputHash = outputHashes.at(outputName); auto thisOutputId = DrvOutput{ thisOutputHash, outputName }; diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index dc8fa9e5a..a53b029b7 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -204,7 +204,8 @@ Strings editorFor(const Pos & pos) if (pos.line > 0 && ( editor.find("emacs") != std::string::npos || editor.find("nano") != std::string::npos || - editor.find("vim") != std::string::npos)) + editor.find("vim") != std::string::npos || + editor.find("kak") != std::string::npos)) args.push_back(fmt("+%d", pos.line)); args.push_back(pos.file); return args; diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 0f6125f11..84bbb5292 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -85,11 +85,12 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions { std::optional<Path> file; std::optional<std::string> expr; + bool readOnlyMode = false; // FIXME: move this; not all commands (e.g. 'nix run') use it. OperateOn operateOn = OperateOn::Output; - SourceExprCommand(); + SourceExprCommand(bool supportReadOnlyMode = false); std::vector<std::shared_ptr<Installable>> parseInstallables( ref<Store> store, std::vector<std::string> ss); @@ -128,13 +129,13 @@ struct InstallableCommand : virtual Args, SourceExprCommand { std::shared_ptr<Installable> installable; - InstallableCommand(); + InstallableCommand(bool supportReadOnlyMode = false); void prepare() override; std::optional<FlakeRef> getFlakeRefForCompletion() override { - return parseFlakeRef(_installable, absPath(".")); + return parseFlakeRefWithFragment(_installable, absPath(".")).first; } private: diff --git a/src/libexpr/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e50ff244c..5b6e82388 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -7,6 +7,7 @@ #include "registry.hh" #include "flake/flakeref.hh" #include "store-api.hh" +#include "command.hh" namespace nix { @@ -59,6 +60,9 @@ MixEvalArgs::MixEvalArgs() fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); + }}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(openStore(), prefix); }} }); diff --git a/src/libexpr/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 03fa226aa..03fa226aa 100644 --- a/src/libexpr/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index b7623d4ba..74c8a6df5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -1,4 +1,6 @@ +#include "globals.hh" #include "installables.hh" +#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" @@ -99,6 +101,14 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), parseFlakeRef(flakeRef, absPath("."), true)); + }}, + .completer = {[&](size_t n, std::string_view prefix) { + if (n == 0) { + if (auto flakeRef = getFlakeRefForCompletion()) + completeFlakeInputPath(getEvalState(), *flakeRef, prefix); + } else if (n == 1) { + completeFlakeRef(getEvalState()->store, prefix); + } }} }); @@ -129,12 +139,14 @@ MixFlakeOptions::MixFlakeOptions() }); } -SourceExprCommand::SourceExprCommand() +SourceExprCommand::SourceExprCommand(bool supportReadOnlyMode) { addFlag({ .longName = "file", .shortName = 'f', - .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.", + .description = + "Interpret installables as attribute paths relative to the Nix expression stored in *file*. " + "If *file* is the character -, then a Nix expression will be read from standard input.", .category = installablesCategory, .labels = {"file"}, .handler = {&file}, @@ -155,6 +167,17 @@ SourceExprCommand::SourceExprCommand() .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); + + if (supportReadOnlyMode) { + addFlag({ + .longName = "read-only", + .description = + "Do not instantiate each evaluated derivation. " + "This improves performance, but can cause errors when accessing " + "store paths of derivations during evaluation.", + .handler = {&readOnlyMode, true}, + }); + } } Strings SourceExprCommand::getDefaultFlakeAttrPaths() @@ -180,6 +203,8 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() void SourceExprCommand::completeInstallable(std::string_view prefix) { if (file) { + completionType = ctAttrs; + evalSettings.pureEval = false; auto state = getEvalState(); Expr *e = state->parseExprFromFile( @@ -208,13 +233,14 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) Value v2; state->autoCallFunction(*autoArgs, v1, v2); - completionType = ctAttrs; - if (v2.type() == nAttrs) { for (auto & i : *v2.attrs) { std::string name = i.name; if (name.find(searchWord) == 0) { - completions->add(i.name); + if (prefix_ == "") + completions->add(name); + else + completions->add(prefix_ + "." + name); } } } @@ -242,10 +268,11 @@ void completeFlakeRefWithFragment( if (hash == std::string::npos) { completeFlakeRef(evalState->store, prefix); } else { + completionType = ctAttrs; + auto fragment = prefix.substr(hash + 1); auto flakeRefS = std::string(prefix.substr(0, hash)); - // FIXME: do tilde expansion. - auto flakeRef = parseFlakeRef(flakeRefS, absPath(".")); + auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, std::make_shared<flake::LockedFlake>(lockFlake(*evalState, flakeRef, lockFlags))); @@ -257,8 +284,6 @@ void completeFlakeRefWithFragment( flake. */ attrPathPrefixes.push_back(""); - completionType = ctAttrs; - for (auto & attrPathPrefixS : attrPathPrefixes) { auto attrPathPrefix = parseAttrPath(*evalState, attrPathPrefixS); auto attrPathS = attrPathPrefixS + std::string(fragment); @@ -332,16 +357,16 @@ DerivedPath Installable::toDerivedPath() return std::move(buildables[0]); } -std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> +std::vector<ref<eval_cache::AttrCursor>> Installable::getCursors(EvalState & state) { auto evalCache = std::make_shared<nix::eval_cache::EvalCache>(std::nullopt, state, [&]() { return toValue(state).first; }); - return {{evalCache->getRoot(), ""}}; + return {evalCache->getRoot()}; } -std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string> +ref<eval_cache::AttrCursor> Installable::getCursor(EvalState & state) { auto cursors = getCursors(state); @@ -564,43 +589,21 @@ InstallableFlake::InstallableFlake( std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation() { - auto lockedFlake = getLockedFlake(); + auto attr = getCursor(*state); - auto cache = openEvalCache(*state, lockedFlake); - auto root = cache->getRoot(); - - Suggestions suggestions; - - for (auto & attrPath : getActualAttrPaths()) { - debug("trying flake output attribute '%s'", attrPath); - - auto attrOrSuggestions = root->findAlongAttrPath( - parseAttrPath(*state, attrPath), - true - ); - - if (!attrOrSuggestions) { - suggestions += attrOrSuggestions.getSuggestions(); - continue; - } + auto attrPath = attr->getAttrPathStr(); - auto attr = *attrOrSuggestions; + if (!attr->isDerivation()) + throw Error("flake output attribute '%s' is not a derivation", attrPath); - if (!attr->isDerivation()) - throw Error("flake output attribute '%s' is not a derivation", attrPath); + auto drvPath = attr->forceDerivation(); - auto drvPath = attr->forceDerivation(); - - auto drvInfo = DerivationInfo { - std::move(drvPath), - attr->getAttr(state->sOutputName)->getString() - }; - - return {attrPath, lockedFlake->flake.lockedRef, std::move(drvInfo)}; - } + auto drvInfo = DerivationInfo { + std::move(drvPath), + attr->getAttr(state->sOutputName)->getString() + }; - throw Error(suggestions, "flake '%s' does not provide attribute %s", - flakeRef, showAttrPaths(getActualAttrPaths())); + return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)}; } std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() @@ -612,33 +615,10 @@ std::vector<InstallableValue::DerivationInfo> InstallableFlake::toDerivations() std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state) { - auto lockedFlake = getLockedFlake(); - - auto vOutputs = getFlakeOutputs(state, *lockedFlake); - - auto emptyArgs = state.allocBindings(0); - - Suggestions suggestions; - - for (auto & attrPath : getActualAttrPaths()) { - try { - auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); - state.forceValue(*v, pos); - return {v, pos}; - } catch (AttrPathNotFound & e) { - suggestions += e.info().suggestions; - } - } - - throw Error( - suggestions, - "flake '%s' does not provide attribute %s", - flakeRef, - showAttrPaths(getActualAttrPaths()) - ); + return {&getCursor(state)->forceValue(), noPos}; } -std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> +std::vector<ref<eval_cache::AttrCursor>> InstallableFlake::getCursors(EvalState & state) { auto evalCache = openEvalCache(state, @@ -646,21 +626,55 @@ InstallableFlake::getCursors(EvalState & state) auto root = evalCache->getRoot(); - std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> res; + std::vector<ref<eval_cache::AttrCursor>> res; for (auto & attrPath : getActualAttrPaths()) { auto attr = root->findAlongAttrPath(parseAttrPath(state, attrPath)); - if (attr) res.push_back({*attr, attrPath}); + if (attr) res.push_back(ref(*attr)); } return res; } +ref<eval_cache::AttrCursor> InstallableFlake::getCursor(EvalState & state) +{ + auto lockedFlake = getLockedFlake(); + + auto cache = openEvalCache(state, lockedFlake); + auto root = cache->getRoot(); + + Suggestions suggestions; + + auto attrPaths = getActualAttrPaths(); + + for (auto & attrPath : attrPaths) { + debug("trying flake output attribute '%s'", attrPath); + + auto attrOrSuggestions = root->findAlongAttrPath( + parseAttrPath(state, attrPath), + true + ); + + if (!attrOrSuggestions) { + suggestions += attrOrSuggestions.getSuggestions(); + continue; + } + + return *attrOrSuggestions; + } + + throw Error( + suggestions, + "flake '%s' does not provide attribute %s", + flakeRef, + showAttrPaths(attrPaths)); +} + std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const { - flake::LockFlags lockFlagsApplyConfig = lockFlags; - lockFlagsApplyConfig.applyNixConfig = true; if (!_lockedFlake) { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + lockFlagsApplyConfig.applyNixConfig = true; _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; @@ -685,6 +699,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( { std::vector<std::shared_ptr<Installable>> result; + if (readOnlyMode) { + settings.readOnlyMode = true; + } + if (file || expr) { if (file && expr) throw UsageError("'--file' and '--expr' are exclusive"); @@ -695,7 +713,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( auto state = getEvalState(); auto vFile = state->allocValue(); - if (file) + if (file == "-") { + auto e = state->parseStdin(); + state->eval(e, *vFile); + } else if (file) state->evalFile(lookupFileArg(*state, *file), *vFile); else { auto e = state->parseExprFromString(*expr, absPath(".")); @@ -751,55 +772,20 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable( return installables.front(); } -BuiltPaths getBuiltPaths(ref<Store> evalStore, ref<Store> store, const DerivedPaths & hopefullyBuiltPaths) +BuiltPaths Installable::build( + ref<Store> evalStore, + ref<Store> store, + Realise mode, + const std::vector<std::shared_ptr<Installable>> & installables, + BuildMode bMode) { BuiltPaths res; - for (const auto & b : hopefullyBuiltPaths) - std::visit( - overloaded{ - [&](const DerivedPath::Opaque & bo) { - res.push_back(BuiltPath::Opaque{bo.path}); - }, - [&](const DerivedPath::Built & bfd) { - OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive - auto drvOutputs = drv.outputsAndOptPaths(*store); - for (auto & output : bfd.outputs) { - if (!outputHashes.count(output)) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store->printStorePath(bfd.drvPath), output); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { - auto outputId = - DrvOutput{outputHashes.at(output), output}; - auto realisation = - store->queryRealisation(outputId); - if (!realisation) - throw Error( - "cannot operate on an output of unbuilt " - "content-addressed derivation '%s'", - outputId.to_string()); - outputs.insert_or_assign( - output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - assert(drvOutputs.count(output)); - assert(drvOutputs.at(output).second); - outputs.insert_or_assign( - output, *drvOutputs.at(output).second); - } - } - res.push_back(BuiltPath::Built{bfd.drvPath, outputs}); - }, - }, - b.raw()); - + for (auto & [_, builtPath] : build2(evalStore, store, mode, installables, bMode)) + res.push_back(builtPath); return res; } -BuiltPaths Installable::build( +std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> Installable::build2( ref<Store> evalStore, ref<Store> store, Realise mode, @@ -810,39 +796,93 @@ BuiltPaths Installable::build( settings.readOnlyMode = true; std::vector<DerivedPath> pathsToBuild; + std::map<DerivedPath, std::vector<std::shared_ptr<Installable>>> backmap; for (auto & i : installables) { - auto b = i->toDerivedPaths(); - pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); + for (auto b : i->toDerivedPaths()) { + pathsToBuild.push_back(b); + backmap[b].push_back(i); + } } + std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> res; + switch (mode) { + case Realise::Nothing: case Realise::Derivation: printMissing(store, pathsToBuild, lvlError); - return getBuiltPaths(evalStore, store, pathsToBuild); + + for (auto & path : pathsToBuild) { + for (auto & installable : backmap[path]) { + std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + OutputPathMap outputs; + auto drv = evalStore->readDerivation(bfd.drvPath); + auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive + auto drvOutputs = drv.outputsAndOptPaths(*store); + for (auto & output : bfd.outputs) { + if (!outputHashes.count(output)) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + store->printStorePath(bfd.drvPath), output); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + DrvOutput outputId { outputHashes.at(output), output }; + auto realisation = store->queryRealisation(outputId); + if (!realisation) + throw Error( + "cannot operate on an output of unbuilt " + "content-addressed derivation '%s'", + outputId.to_string()); + outputs.insert_or_assign(output, realisation->outPath); + } else { + // If ca-derivations isn't enabled, assume that + // the output path is statically known. + assert(drvOutputs.count(output)); + assert(drvOutputs.at(output).second); + outputs.insert_or_assign( + output, *drvOutputs.at(output).second); + } + } + res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); + }, + [&](const DerivedPath::Opaque & bo) { + res.push_back({installable, BuiltPath::Opaque { bo.path }}); + }, + }, path.raw()); + } + } + + break; + case Realise::Outputs: { - BuiltPaths res; for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { if (!buildResult.success()) buildResult.rethrow(); - std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - std::map<std::string, StorePath> outputs; - for (auto & path : buildResult.builtOutputs) - outputs.emplace(path.first.outputName, path.second.outPath); - res.push_back(BuiltPath::Built { bfd.drvPath, outputs }); - }, - [&](const DerivedPath::Opaque & bo) { - res.push_back(BuiltPath::Opaque { bo.path }); - }, - }, buildResult.path.raw()); + + for (auto & installable : backmap[buildResult.path]) { + std::visit(overloaded { + [&](const DerivedPath::Built & bfd) { + std::map<std::string, StorePath> outputs; + for (auto & path : buildResult.builtOutputs) + outputs.emplace(path.first.outputName, path.second.outPath); + res.push_back({installable, BuiltPath::Built { bfd.drvPath, outputs }}); + }, + [&](const DerivedPath::Opaque & bo) { + res.push_back({installable, BuiltPath::Opaque { bo.path }}); + }, + }, buildResult.path.raw()); + } } - return res; + + break; } + default: assert(false); } + + return res; } BuiltPaths Installable::toBuiltPaths( @@ -930,7 +970,7 @@ InstallablesCommand::InstallablesCommand() void InstallablesCommand::prepare() { if (_installables.empty() && useDefaultInstallables()) - // FIXME: commands like "nix install" should not have a + // FIXME: commands like "nix profile install" should not have a // default, probably. _installables.push_back("."); installables = parseInstallables(getStore(), _installables); @@ -940,13 +980,14 @@ std::optional<FlakeRef> InstallablesCommand::getFlakeRefForCompletion() { if (_installables.empty()) { if (useDefaultInstallables()) - return parseFlakeRef(".", absPath(".")); + return parseFlakeRefWithFragment(".", absPath(".")).first; return {}; } - return parseFlakeRef(_installables.front(), absPath(".")); + return parseFlakeRefWithFragment(_installables.front(), absPath(".")).first; } -InstallableCommand::InstallableCommand() +InstallableCommand::InstallableCommand(bool supportReadOnlyMode) + : SourceExprCommand(supportReadOnlyMode) { expectArgs({ .label = "installable", diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index e172b71b0..b847f8939 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -80,10 +80,10 @@ struct Installable return {}; } - virtual std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> + virtual std::vector<ref<eval_cache::AttrCursor>> getCursors(EvalState & state); - std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string> + virtual ref<eval_cache::AttrCursor> getCursor(EvalState & state); virtual FlakeRef nixpkgsFlakeRef() const @@ -98,6 +98,13 @@ struct Installable const std::vector<std::shared_ptr<Installable>> & installables, BuildMode bMode = bmNormal); + static std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> build2( + ref<Store> evalStore, + ref<Store> store, + Realise mode, + const std::vector<std::shared_ptr<Installable>> & installables, + BuildMode bMode = bmNormal); + static std::set<StorePath> toStorePaths( ref<Store> evalStore, ref<Store> store, @@ -173,9 +180,15 @@ struct InstallableFlake : InstallableValue std::pair<Value *, Pos> toValue(EvalState & state) override; - std::vector<std::pair<std::shared_ptr<eval_cache::AttrCursor>, std::string>> + /* Get a cursor to every attrpath in getActualAttrPaths() that + exists. */ + std::vector<ref<eval_cache::AttrCursor>> getCursors(EvalState & state) override; + /* Get a cursor to the first attrpath in getActualAttrPaths() that + exists, or throw an exception with suggestions if none exists. */ + ref<eval_cache::AttrCursor> getCursor(EvalState & state) override; + std::shared_ptr<flake::LockedFlake> getLockedFlake() const; FlakeRef nixpkgsFlakeRef() const override; @@ -185,9 +198,4 @@ ref<eval_cache::EvalCache> openEvalCache( EvalState & state, std::shared_ptr<flake::LockedFlake> lockedFlake); -BuiltPaths getBuiltPaths( - ref<Store> evalStore, - ref<Store> store, - const DerivedPaths & hopefullyBuiltPaths); - } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 188223957..7d3fd01a4 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -21,6 +21,8 @@ struct AttrDb { std::atomic_bool failed{false}; + const Store & cfg; + struct State { SQLite db; @@ -33,8 +35,9 @@ struct AttrDb std::unique_ptr<Sync<State>> _state; - AttrDb(const Hash & fingerprint) - : _state(std::make_unique<Sync<State>>()) + AttrDb(const Store & cfg, const Hash & fingerprint) + : cfg(cfg) + , _state(std::make_unique<Sync<State>>()) { auto state(_state->lock()); @@ -254,10 +257,10 @@ struct AttrDb return {{rowId, attrs}}; } case AttrType::String: { - std::vector<std::pair<Path, std::string>> context; + NixStringContext context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";")) - context.push_back(decodeContext(s)); + context.push_back(decodeContext(cfg, s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: @@ -274,10 +277,10 @@ struct AttrDb } }; -static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint) +static std::shared_ptr<AttrDb> makeAttrDb(const Store & cfg, const Hash & fingerprint) { try { - return std::make_shared<AttrDb>(fingerprint); + return std::make_shared<AttrDb>(cfg, fingerprint); } catch (SQLiteError &) { ignoreException(); return nullptr; @@ -288,7 +291,7 @@ EvalCache::EvalCache( std::optional<std::reference_wrapper<const Hash>> useCache, EvalState & state, RootLoader rootLoader) - : db(useCache ? makeAttrDb(*useCache) : nullptr) + : db(useCache ? makeAttrDb(*state.store, *useCache) : nullptr) , state(state) , rootLoader(rootLoader) { @@ -303,9 +306,9 @@ Value * EvalCache::getRootValue() return *value; } -std::shared_ptr<AttrCursor> EvalCache::getRoot() +ref<AttrCursor> EvalCache::getRoot() { - return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt); + return make_ref<AttrCursor>(ref(shared_from_this()), std::nullopt); } AttrCursor::AttrCursor( @@ -546,7 +549,7 @@ string_t AttrCursor::getStringWithContext() if (auto s = std::get_if<string_t>(&cachedValue->second)) { bool valid = true; for (auto & c : s->second) { - if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) { + if (!root->state.store->isValidPath(c.first)) { valid = false; break; } @@ -563,7 +566,7 @@ string_t AttrCursor::getStringWithContext() auto & v = forceValue(); if (v.type() == nString) - return {v.string.s, v.getContext()}; + return {v.string.s, v.getContext(*root->state.store)}; else if (v.type() == nPath) return {v.path, {}}; else diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 40f1d4ffc..b0709ebc2 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -33,7 +33,7 @@ public: EvalState & state, RootLoader rootLoader); - std::shared_ptr<AttrCursor> getRoot(); + ref<AttrCursor> getRoot(); }; enum AttrType { @@ -52,7 +52,7 @@ struct misc_t {}; struct failed_t {}; typedef uint64_t AttrId; typedef std::pair<AttrId, Symbol> AttrKey; -typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t; +typedef std::pair<std::string, NixStringContext> string_t; typedef std::variant< std::vector<Symbol>, @@ -104,6 +104,8 @@ public: ref<AttrCursor> getAttr(std::string_view name); + /* Get an attribute along a chain of attrsets. Note that this does + not auto-call functors or functions. */ OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); std::string getString(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c65ef9738..b87e06ef5 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -96,20 +96,20 @@ RootValue allocRootValue(Value * v) } -void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v) +void Value::print(std::ostream & str, std::set<const void *> * seen) const { checkInterrupt(); - switch (v.internalType) { + switch (internalType) { case tInt: - str << v.integer; + str << integer; break; case tBool: - str << (v.boolean ? "true" : "false"); + str << (boolean ? "true" : "false"); break; case tString: str << "\""; - for (const char * i = v.string.s; *i; i++) + for (const char * i = string.s; *i; i++) if (*i == '\"' || *i == '\\') str << "\\" << *i; else if (*i == '\n') str << "\\n"; else if (*i == '\r') str << "\\r"; @@ -119,19 +119,19 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value & str << "\""; break; case tPath: - str << v.path; // !!! escaping? + str << path; // !!! escaping? break; case tNull: str << "null"; break; case tAttrs: { - if (!v.attrs->empty() && !seen.insert(v.attrs).second) - str << "<REPEAT>"; + if (seen && !attrs->empty() && !seen->insert(attrs).second) + str << "«repeated»"; else { str << "{ "; - for (auto & i : v.attrs->lexicographicOrder()) { + for (auto & i : attrs->lexicographicOrder()) { str << i->name << " = "; - printValue(str, seen, *i->value); + i->value->print(str, seen); str << "; "; } str << "}"; @@ -141,12 +141,12 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value & case tList1: case tList2: case tListN: - if (v.listSize() && !seen.insert(v.listElems()).second) - str << "<REPEAT>"; + if (seen && listSize() && !seen->insert(listElems()).second) + str << "«repeated»"; else { str << "[ "; - for (auto v2 : v.listItems()) { - printValue(str, seen, *v2); + for (auto v2 : listItems()) { + v2->print(str, seen); str << " "; } str << "]"; @@ -166,10 +166,10 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value & str << "<PRIMOP-APP>"; break; case tExternal: - str << *v.external; + str << *external; break; case tFloat: - str << v.fpoint; + str << fpoint; break; default: abort(); @@ -177,10 +177,16 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value & } -std::ostream & operator << (std::ostream & str, const Value & v) +void Value::print(std::ostream & str, bool showRepeated) const { std::set<const void *> seen; - printValue(str, seen, v); + print(str, showRepeated ? nullptr : &seen); +} + + +std::ostream & operator << (std::ostream & str, const Value & v) +{ + v.print(str, false); return str; } @@ -430,6 +436,7 @@ EvalState::EvalState( , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) , sContentAddressed(symbols.create("__contentAddressed")) + , sImpure(symbols.create("__impure")) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) @@ -501,23 +508,6 @@ EvalState::~EvalState() } -void EvalState::requireExperimentalFeatureOnEvaluation( - const ExperimentalFeature & feature, - const std::string_view fName, - const Pos & pos) -{ - if (!settings.isExperimentalFeatureEnabled(feature)) { - throw EvalError({ - .msg = hintfmt( - "Cannot call '%2%' because experimental Nix feature '%1%' is disabled. You can enable it via '--extra-experimental-features %1%'.", - feature, - fName - ), - .errPos = pos - }); - } -} - void EvalState::allowPath(const Path & path) { if (allowedPaths) @@ -1903,13 +1893,22 @@ std::string_view EvalState::forceString(Value & v, const Pos & pos) /* Decode a context string ‘!<name>!<path>’ into a pair <path, name>. */ -std::pair<std::string, std::string> decodeContext(std::string_view s) +NixStringContextElem decodeContext(const Store & store, std::string_view s) { if (s.at(0) == '!') { size_t index = s.find("!", 1); - return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))}; + return { + store.parseStorePath(s.substr(index + 1)), + std::string(s.substr(1, index - 1)), + }; } else - return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""}; + return { + store.parseStorePath( + s.at(0) == '/' + ? s + : s.substr(1)), + "", + }; } @@ -1921,13 +1920,13 @@ void copyContext(const Value & v, PathSet & context) } -std::vector<std::pair<Path, std::string>> Value::getContext() +NixStringContext Value::getContext(const Store & store) { - std::vector<std::pair<Path, std::string>> res; + NixStringContext res; assert(internalType == tString); if (string.context) for (const char * * p = string.context; *p; ++p) - res.push_back(decodeContext(*p)); + res.push_back(decodeContext(store, *p)); return res; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f1e00bae7..7ed376e8d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -78,7 +78,7 @@ public: sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, - sContentAddressed, + sContentAddressed, sImpure, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath, @@ -149,12 +149,6 @@ public: std::shared_ptr<Store> buildStore = nullptr); ~EvalState(); - void requireExperimentalFeatureOnEvaluation( - const ExperimentalFeature &, - const std::string_view fName, - const Pos & pos - ); - void addToSearchPath(const std::string & s); SearchPath getSearchPath() { return searchPath; } @@ -430,7 +424,7 @@ std::string showType(const Value & v); /* Decode a context string ‘!<name>!<path>’ into a pair <path, name>. */ -std::pair<std::string, std::string> decodeContext(std::string_view s); +NixStringContextElem decodeContext(const Store & store, std::string_view s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 6a1aca40d..22257c6b3 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -706,8 +706,6 @@ void callFlake(EvalState & state, static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos); - std::string flakeRefS(state.forceStringNoCtx(*args[0], pos)); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) @@ -723,7 +721,30 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va v); } -static RegisterPrimOp r2("__getFlake", 1, prim_getFlake); +static RegisterPrimOp r2({ + .name = "__getFlake", + .args = {"args"}, + .doc = R"( + Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + + ```nix + (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix + ``` + + Unless impure evaluation is allowed (`--impure`), the flake reference + must be "locked", e.g. contain a Git revision or content hash. An + example of an unlocked usage is: + + ```nix + (builtins.getFlake "github:edolstra/dwarffs").rev + ``` + + This function is only available if you enable the experimental feature + `flakes`. + )", + .fun = prim_getFlake, + .experimentalFeature = Xp::Flakes, +}); } diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 7630c5ff4..bb7e77b61 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,6 +1,7 @@ #include "get-drvs.hh" #include "util.hh" #include "eval-inline.hh" +#include "derivations.hh" #include "store-api.hh" #include "path-with-outputs.hh" @@ -102,7 +103,7 @@ StorePath DrvInfo::queryOutPath() const } -DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) +DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall) { if (outputs.empty()) { /* Get the ‘outputs’ list. */ @@ -112,20 +113,24 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) /* For each output... */ for (auto elem : i->value->listItems()) { - /* Evaluate the corresponding set. */ - std::string name(state->forceStringNoCtx(*elem, *i->pos)); - Bindings::iterator out = attrs->find(state->symbols.create(name)); - if (out == attrs->end()) continue; // FIXME: throw error? - state->forceAttrs(*out->value, *i->pos); - - /* And evaluate its ‘outPath’ attribute. */ - Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); - if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - PathSet context; - outputs.emplace(name, state->coerceToStorePath(*outPath->pos, *outPath->value, context)); + std::string output(state->forceStringNoCtx(*elem, *i->pos)); + + if (withPaths) { + /* Evaluate the corresponding set. */ + Bindings::iterator out = attrs->find(state->symbols.create(output)); + if (out == attrs->end()) continue; // FIXME: throw error? + state->forceAttrs(*out->value, *i->pos); + + /* And evaluate its ‘outPath’ attribute. */ + Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); + if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? + PathSet context; + outputs.emplace(output, state->coerceToStorePath(*outPath->pos, *outPath->value, context)); + } else + outputs.emplace(output, std::nullopt); } } else - outputs.emplace("out", queryOutPath()); + outputs.emplace("out", withPaths ? std::optional{queryOutPath()} : std::nullopt); } if (!onlyOutputsToInstall || !attrs) return outputs; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 3ca6f1fca..7cc1abef2 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -13,7 +13,7 @@ namespace nix { struct DrvInfo { public: - typedef std::map<std::string, StorePath> Outputs; + typedef std::map<std::string, std::optional<StorePath>> Outputs; private: EvalState * state; @@ -46,8 +46,9 @@ public: StorePath requireDrvPath() const; StorePath queryOutPath() const; std::string queryOutputName() const; - /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ - Outputs queryOutputs(bool onlyOutputsToInstall = false); + /** Return the unordered map of output names to (optional) output paths. + * The "outputs to install" are determined by `meta.outputsToInstall`. */ + Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false); StringSet queryMetaNames(); Value * queryMeta(const std::string & name); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index e276b0467..d574121b0 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -28,6 +28,13 @@ using namespace nix; namespace nix { +static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) +{ + return Pos(data->origin, data->file, loc.first_line, loc.first_column); +} + +#define CUR_POS makeCurPos(*yylloc, data) + // backup to recover from yyless(0) YYLTYPE prev_yylloc; @@ -37,7 +44,6 @@ static void initLoc(YYLTYPE * loc) loc->first_column = loc->last_column = 1; } - static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) { prev_yylloc = *loc; @@ -147,14 +153,20 @@ or { return OR_KW; } try { yylval->n = boost::lexical_cast<int64_t>(yytext); } catch (const boost::bad_lexical_cast &) { - throw ParseError("invalid integer '%1%'", yytext); + throw ParseError({ + .msg = hintfmt("invalid integer '%1%'", yytext), + .errPos = CUR_POS, + }); } return INT; } {FLOAT} { errno = 0; yylval->nf = strtod(yytext, 0); if (errno != 0) - throw ParseError("invalid float '%1%'", yytext); + throw ParseError({ + .msg = hintfmt("invalid float '%1%'", yytext), + .errPos = CUR_POS, + }); return FLOAT; } @@ -280,7 +292,10 @@ or { return OR_KW; } <INPATH_SLASH>{ANY} | <INPATH_SLASH><<EOF>> { - throw ParseError("path has a trailing slash"); + throw ParseError({ + .msg = hintfmt("path has a trailing slash"), + .errPos = CUR_POS, + }); } {SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 3124025aa..73817dbdd 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -43,8 +43,8 @@ StringMap EvalState::realiseContext(const PathSet & context) StringMap res; for (auto & i : context) { - auto [ctxS, outputName] = decodeContext(i); - auto ctx = store->parseStorePath(ctxS); + auto [ctx, outputName] = decodeContext(*store, i); + auto ctxS = store->printStorePath(ctx); if (!store->isValidPath(ctx)) throw InvalidPathError(store->printStorePath(ctx)); if (!outputName.empty() && ctx.isDerivation()) { @@ -694,7 +694,32 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info { .name = "__genericClosure", + .args = {"attrset"}, .arity = 1, + .doc = R"( + Take an *attrset* with values named `startSet` and `operator` in order to + return a *list of attrsets* by starting with the `startSet`, recursively + applying the `operator` function to each element. The *attrsets* in the + `startSet` and produced by the `operator` must each contain value named + `key` which are comparable to each other. The result is produced by + repeatedly calling the operator for each element encountered with a + unique key, terminating when no new elements are produced. For example, + + ``` + builtins.genericClosure { + startSet = [ {key = 5;} ]; + operator = item: [{ + key = if (item.key / 2 ) * 2 == item.key + then item.key / 2 + else 3 * item.key + 1; + }]; + } + ``` + evaluates to + ``` + [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] + ``` + )", .fun = prim_genericClosure, }); @@ -964,9 +989,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * PathSet context; bool contentAddressed = false; + bool isImpure = false; std::optional<std::string> outputHash; std::string outputHashAlgo; - auto ingestionMethod = FileIngestionMethod::Flat; + std::optional<FileIngestionMethod> ingestionMethod; StringSet outputs; outputs.insert("out"); @@ -1026,6 +1052,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * settings.requireExperimentalFeature(Xp::CaDerivations); } + else if (i->name == state.sImpure) { + isImpure = state.forceBool(*i->value, pos); + if (isImpure) + settings.requireExperimentalFeature(Xp::ImpureDerivations); + } + /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { @@ -1118,8 +1150,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Handle derivation outputs of the form ‘!<name>!<path>’. */ else if (path.at(0) == '!') { - auto ctx = decodeContext(path); - drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second); + auto ctx = decodeContext(*state.store, path); + drv.inputDrvs[ctx.first].insert(ctx.second); } /* Otherwise it's a source file. */ @@ -1158,31 +1190,44 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * .errPos = posDrvName }); - std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo); - Hash h = newHashAllowEmpty(*outputHash, ht); + auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); - auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); + auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); + auto outPath = state.store->makeFixedOutputPath(method, h, drvName); drv.env["out"] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", DerivationOutput { - .output = DerivationOutputCAFixed { - .hash = FixedOutputHash { - .method = ingestionMethod, - .hash = std::move(h), - }, + drv.outputs.insert_or_assign("out", + DerivationOutput::CAFixed { + .hash = FixedOutputHash { + .method = method, + .hash = std::move(h), }, - }); + }); } - else if (contentAddressed) { - HashType ht = parseHashType(outputHashAlgo); + else if (contentAddressed || isImpure) { + if (contentAddressed && isImpure) + throw EvalError({ + .msg = hintfmt("derivation cannot be both content-addressed and impure"), + .errPos = posDrvName + }); + + auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256); + auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); + for (auto & i : outputs) { drv.env[i] = hashPlaceholder(i); - drv.outputs.insert_or_assign(i, DerivationOutput { - .output = DerivationOutputCAFloating { - .method = ingestionMethod, - .hashType = ht, - }, - }); + if (isImpure) + drv.outputs.insert_or_assign(i, + DerivationOutput::Impure { + .method = method, + .hashType = ht, + }); + else + drv.outputs.insert_or_assign(i, + DerivationOutput::CAFloating { + .method = method, + .hashType = ht, + }); } } @@ -1196,44 +1241,29 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : outputs) { drv.env[i] = ""; drv.outputs.insert_or_assign(i, - DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = StorePath::dummy, - }, - }); + DerivationOutput::Deferred { }); } - // Regular, non-CA derivation should always return a single hash and not - // hash per output. auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); - std::visit(overloaded { - [&](Hash & h) { - for (auto & i : outputs) { - auto outPath = state.store->makeOutputPath(i, h, drvName); - drv.env[i] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign(i, - DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = std::move(outPath), - }, - }); - } - }, - [&](CaOutputHashes &) { - // Shouldn't happen as the toplevel derivation is not CA. - assert(false); - }, - [&](DeferredHash &) { - for (auto & i : outputs) { - drv.outputs.insert_or_assign(i, - DerivationOutput { - .output = DerivationOutputDeferred{}, - }); - } - }, - }, - hashModulo); - + switch (hashModulo.kind) { + case DrvHash::Kind::Regular: + for (auto & i : outputs) { + auto h = hashModulo.hashes.at(i); + auto outPath = state.store->makeOutputPath(i, h, drvName); + drv.env[i] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign( + i, + DerivationOutputInputAddressed { + .path = std::move(outPath), + }); + } + break; + ; + case DrvHash::Kind::Deferred: + for (auto & i : outputs) { + drv.outputs.insert_or_assign(i, DerivationOutputDeferred {}); + } + } } /* Write the resulting term into the Nix store directory. */ @@ -1244,12 +1274,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Optimisation, but required in read-only mode! because in that case we don't actually write store derivations, so we can't - read them later. - - However, we don't bother doing this for floating CA derivations because - their "hash modulo" is indeterminate until built. */ - if (drv.type() != DerivationType::CAFloating) { - auto h = hashDerivationModulo(*state.store, Derivation(drv), false); + read them later. */ + { + auto h = hashDerivationModulo(*state.store, drv, false); drvHashes.lock()->insert_or_assign(drvPath, h); } @@ -3801,7 +3828,7 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) .name = name, .args = {}, .arity = arity, - .fun = fun + .fun = fun, }); } @@ -3873,13 +3900,17 @@ void EvalState::createBaseEnv() if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) - addPrimOp({ - .fun = primOp.fun, - .arity = std::max(primOp.args.size(), primOp.arity), - .name = symbols.create(primOp.name), - .args = primOp.args, - .doc = primOp.doc, - }); + if (!primOp.experimentalFeature + || settings.isExperimentalFeatureEnabled(*primOp.experimentalFeature)) + { + addPrimOp({ + .fun = primOp.fun, + .arity = std::max(primOp.args.size(), primOp.arity), + .name = symbols.create(primOp.name), + .args = primOp.args, + .doc = primOp.doc, + }); + } /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 5b16e075f..905bd0366 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -16,6 +16,7 @@ struct RegisterPrimOp size_t arity = 0; const char * doc; PrimOpFun fun; + std::optional<ExperimentalFeature> experimentalFeature; }; typedef std::vector<Info> PrimOps; @@ -35,6 +36,7 @@ struct RegisterPrimOp /* These primops are disabled without enableNativeCode, but plugins may wish to use them in limited contexts without globally enabling them. */ + /* Load a ValueInitializer from a DSO and return whatever it initializes */ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 3701bd442..cc74c7f58 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "derivations.hh" #include "store-api.hh" namespace nix { @@ -82,8 +83,8 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, drv = std::string(p, 1); path = &drv; } else if (p.at(0) == '!') { - std::pair<std::string, std::string> ctx = decodeContext(p); - drv = ctx.first; + NixStringContextElem ctx = decodeContext(*state.store, p); + drv = state.store->printStorePath(ctx.first); output = ctx.second; path = &drv; } diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc new file mode 100644 index 000000000..821eba698 --- /dev/null +++ b/src/libexpr/primops/fetchClosure.cc @@ -0,0 +1,161 @@ +#include "primops.hh" +#include "store-api.hh" +#include "make-content-addressed.hh" +#include "url.hh" + +namespace nix { + +static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + + std::optional<std::string> fromStoreUrl; + std::optional<StorePath> fromPath; + bool toCA = false; + std::optional<StorePath> toPath; + + for (auto & attr : *args[0]->attrs) { + if (attr.name == "fromPath") { + PathSet context; + fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context); + } + + else if (attr.name == "toPath") { + state.forceValue(*attr.value, *attr.pos); + toCA = true; + if (attr.value->type() != nString || attr.value->string.s != std::string("")) { + PathSet context; + toPath = state.coerceToStorePath(*attr.pos, *attr.value, context); + } + } + + else if (attr.name == "fromStore") + fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos); + + else + throw Error({ + .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name), + .errPos = pos + }); + } + + if (!fromPath) + throw Error({ + .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"), + .errPos = pos + }); + + if (!fromStoreUrl) + throw Error({ + .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), + .errPos = pos + }); + + auto parsedURL = parseURL(*fromStoreUrl); + + if (parsedURL.scheme != "http" && + parsedURL.scheme != "https" && + !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file")) + throw Error({ + .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"), + .errPos = pos + }); + + if (!parsedURL.query.empty()) + throw Error({ + .msg = hintfmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), + .errPos = pos + }); + + auto fromStore = openStore(parsedURL.to_string()); + + if (toCA) { + if (!toPath || !state.store->isValidPath(*toPath)) { + auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); + auto i = remappings.find(*fromPath); + assert(i != remappings.end()); + if (toPath && *toPath != i->second) + throw Error({ + .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second), + state.store->printStorePath(*toPath)), + .errPos = pos + }); + if (!toPath) + throw Error({ + .msg = hintfmt( + "rewriting '%s' to content-addressed form yielded '%s'; " + "please set this in the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(*fromPath), + state.store->printStorePath(i->second)), + .errPos = pos + }); + } + } else { + if (!state.store->isValidPath(*fromPath)) + copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); + toPath = fromPath; + } + + /* In pure mode, require a CA path. */ + if (evalSettings.pureEval) { + auto info = state.store->queryPathInfo(*toPath); + if (!info->isContentAddressed(*state.store)) + throw Error({ + .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", + state.store->printStorePath(*toPath)), + .errPos = pos + }); + } + + auto toPathS = state.store->printStorePath(*toPath); + v.mkString(toPathS, {toPathS}); +} + +static RegisterPrimOp primop_fetchClosure({ + .name = "__fetchClosure", + .args = {"args"}, + .doc = R"( + Fetch a Nix store closure from a binary cache, rewriting it into + content-addressed form. For example, + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` + + fetches `/nix/store/r2jd...` from the specified binary cache, + and rewrites it into the content-addressed store path + `/nix/store/ldbh...`. + + If `fromPath` is already content-addressed, or if you are + allowing impure evaluation (`--impure`), then `toPath` may be + omitted. + + To find out the correct value for `toPath` given a `fromPath`, + you can use `nix store make-content-addressed`: + + ```console + # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 + rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' + ``` + + This function is similar to `builtins.storePath` in that it + allows you to use a previously built store path in a Nix + expression. However, it is more reproducible because it requires + specifying a binary cache from which the path can be fetched. + Also, requiring a content-addressed final store path avoids the + need for users to configure binary cache public keys. + + This function is only available if you enable the experimental + feature `fetch-closure`. + )", + .fun = prim_fetchClosure, + .experimentalFeature = Xp::FetchClosure, +}); + +} diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9c2da2178..42c98e312 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -145,7 +145,7 @@ static void fetchTree( if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) throw Error({ - .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), + .msg = hintfmt("attribute 'name' isn't supported in call to 'fetchTree'"), .errPos = pos }); @@ -329,7 +329,7 @@ static RegisterPrimOp primop_fetchTarball({ .fun = prim_fetchTarball, }); -static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) +static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) { fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 517da4c01..7b35abca2 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -84,7 +84,8 @@ void printValueAsJSON(EvalState & state, bool strict, .msg = hintfmt("cannot convert %1% to JSON", showType(v)), .errPos = v.determinePos(pos) }); - throw e.addTrace(pos, hintfmt("message for the trace")); + e.addTrace(pos, hintfmt("message for the trace")); + throw e; } } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index d0fa93e92..3d07c3198 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -57,6 +57,8 @@ struct ExprLambda; struct PrimOp; class Symbol; struct Pos; +class StorePath; +class Store; class EvalState; class XMLWriter; class JSONPlaceholder; @@ -64,6 +66,8 @@ class JSONPlaceholder; typedef int64_t NixInt; typedef double NixFloat; +typedef std::pair<StorePath, std::string> NixStringContextElem; +typedef std::vector<NixStringContextElem> NixStringContext; /* External values must descend from ExternalValueBase, so that * type-agnostic nix functions (e.g. showType) can be implemented @@ -115,10 +119,13 @@ private: InternalType internalType; friend std::string showType(const Value & v); - friend void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v); + + void print(std::ostream & str, std::set<const void *> * seen) const; public: + void print(std::ostream & str, bool showRepeated = false) const; + // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's // needed by callers into methods of this type @@ -368,7 +375,7 @@ public: non-trivial. */ bool isTrivial() const; - std::vector<std::pair<Path, std::string>> getContext(); + NixStringContext getContext(const Store &); auto listItems() { diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 976f40d3b..6957d2da4 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -238,9 +238,18 @@ std::optional<std::string> Input::getRef() const std::optional<Hash> Input::getRev() const { - if (auto s = maybeGetStrAttr(attrs, "rev")) - return Hash::parseAny(*s, htSHA1); - return {}; + std::optional<Hash> hash = {}; + + if (auto s = maybeGetStrAttr(attrs, "rev")) { + try { + hash = Hash::parseAnyPrefixed(*s); + } catch (BadHash &e) { + // Default to sha1 for backwards compatibility with existing flakes + hash = Hash::parseAny(*s, htSHA1); + } + } + + return hash; } std::optional<uint64_t> Input::getRevCount() const diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d75c5d3ae..34b1342a0 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -28,9 +28,7 @@ static std::string readHead(const Path & path) static bool isNotDotGitDirectory(const Path & path) { - static const std::regex gitDirRegex("^(?:.*/)?\\.git$"); - - return not std::regex_match(path, gitDirRegex); + return baseNameOf(path) != ".git"; } struct GitInputScheme : InputScheme @@ -189,8 +187,16 @@ struct GitInputScheme : InputScheme if (submodules) cacheType += "-submodules"; if (allRefs) cacheType += "-all-refs"; + auto checkHashType = [&](const std::optional<Hash> & hash) + { + if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); + }; + auto getLockedAttrs = [&]() { + checkHashType(input.getRev()); + return Attrs({ {"type", cacheType}, {"name", name}, @@ -285,9 +291,11 @@ struct GitInputScheme : InputScheme auto files = tokenizeString<std::set<std::string>>( runProgram("git", true, gitOpts), "\0"s); + Path actualPath(absPath(actualUrl)); + PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualUrl)); - std::string file(p, actualUrl.size() + 1); + assert(hasPrefix(p, actualPath)); + std::string file(p, actualPath.size() + 1); auto st = lstat(p); @@ -300,13 +308,13 @@ struct GitInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); // FIXME: maybe we should use the timestamp of the last // modified dirty file? input.attrs.insert_or_assign( "lastModified", - hasHead ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); + hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); return {std::move(storePath), input}; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index a1430f087..58b6e7c04 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -390,7 +390,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme ref_uri = line.substr(ref_index+5, line.length()-1); } else - ref_uri = fmt("refs/heads/%s", ref); + ref_uri = fmt("refs/(heads|tags)/%s", ref); auto file = store->toRealPath( downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); @@ -399,9 +399,11 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string line; std::string id; while(getline(is, line)) { - auto index = line.find(ref_uri); - if (index != std::string::npos) { - id = line.substr(0, index-1); + // Append $ to avoid partial name matches + std::regex pattern(fmt("%s$", ref_uri)); + + if (std::regex_search(line, pattern)) { + id = line.substr(0, line.find('\t')); break; } } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 8b82e9daa..1beb8b944 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -178,9 +178,11 @@ struct MercurialInputScheme : InputScheme auto files = tokenizeString<std::set<std::string>>( runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + Path actualPath(absPath(actualUrl)); + PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualUrl)); - std::string file(p, actualUrl.size() + 1); + assert(hasPrefix(p, actualPath)); + std::string file(p, actualPath.size() + 1); auto st = lstat(p); @@ -193,7 +195,7 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); return {std::move(storePath), input}; } @@ -201,8 +203,17 @@ struct MercurialInputScheme : InputScheme if (!input.getRef()) input.attrs.insert_or_assign("ref", "default"); + auto checkHashType = [&](const std::optional<Hash> & hash) + { + if (hash.has_value() && hash->type != htSHA1) + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); + }; + + auto getLockedAttrs = [&]() { + checkHashType(input.getRev()); + return Attrs({ {"type", "hg"}, {"name", name}, diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 59e228e97..f0ef97da5 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "store-api.hh" +#include "archive.hh" namespace nix::fetchers { @@ -80,8 +81,9 @@ struct PathInputScheme : InputScheme // nothing to do } - std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override + std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override { + Input input(_input); std::string absPath; auto path = getStrAttr(input.attrs, "path"); @@ -111,9 +113,15 @@ struct PathInputScheme : InputScheme if (storePath) store->addTempRoot(*storePath); - if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) + time_t mtime = 0; + if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { // FIXME: try to substitute storePath. - storePath = store->addToStore("source", absPath); + auto src = sinkToSource([&](Sink & sink) { + mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter); + }); + storePath = store->addToStoreFromDump(*src, "source"); + } + input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); return {std::move(*storePath), input}; } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 9603a8caa..ca538b3cb 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -2,6 +2,7 @@ #include "crypto.hh" #include "store-api.hh" +#include "log-store.hh" #include "pool.hh" @@ -28,7 +29,9 @@ struct BinaryCacheStoreConfig : virtual StoreConfig "other than -1 which we reserve to indicate Nix defaults should be used"}; }; -class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store +class BinaryCacheStore : public virtual BinaryCacheStoreConfig, + public virtual Store, + public virtual LogStore { private: diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index cb6d19b8e..24fb1f763 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -1,6 +1,7 @@ #pragma once #include "realisation.hh" +#include "derived-path.hh" #include <string> #include <chrono> @@ -30,6 +31,8 @@ struct BuildResult ResolvesToAlreadyValid, NoSubstituters, } status = MiscFailure; + + // FIXME: include entire ErrorInfo object. std::string errorMsg; std::string toString() const { diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index afed9bf16..53f212c1d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -204,10 +204,33 @@ void DerivationGoal::haveDerivation() { trace("have derivation"); - if (drv->type() == DerivationType::CAFloating) + parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + + if (!drv->type().hasKnownOutputPaths()) settings.requireExperimentalFeature(Xp::CaDerivations); - retrySubstitution = false; + if (!drv->type().isPure()) { + settings.requireExperimentalFeature(Xp::ImpureDerivations); + + for (auto & [outputName, output] : drv->outputs) { + auto randomPath = StorePath::random(outputPathName(drv->name, outputName)); + assert(!worker.store.isValidPath(randomPath)); + initialOutputs.insert({ + outputName, + InitialOutput { + .wanted = true, + .outputHash = impureOutputHash, + .known = InitialOutputStatus { + .path = randomPath, + .status = PathStatus::Absent + } + } + }); + } + + gaveUpOnSubstitution(); + return; + } for (auto & i : drv->outputsAndOptPaths(worker.store)) if (i.second.second) @@ -232,9 +255,6 @@ void DerivationGoal::haveDerivation() return; } - parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); - - /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ @@ -268,6 +288,8 @@ void DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); + assert(drv->type().isPure()); + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { done(BuildResult::TransientFailure, {}, Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", @@ -311,18 +333,27 @@ void DerivationGoal::outputsSubstitutionTried() gaveUpOnSubstitution(); } + /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ void DerivationGoal::gaveUpOnSubstitution() { - /* Make sure checkPathValidity() from now on checks all - outputs. */ - wantedOutputs.clear(); - /* The inputs must be built before we can build this goal. */ + inputDrvOutputs.clear(); if (useDerivation) - for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) + for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs) { + /* Ensure that pure, non-fixed-output derivations don't + depend on impure derivations. */ + if (drv->type().isPure() && !drv->type().isFixed()) { + auto inputDrv = worker.evalStore.readDerivation(i.first); + if (!inputDrv.type().isPure()) + throw Error("pure derivation '%s' depends on impure derivation '%s'", + worker.store.printStorePath(drvPath), + worker.store.printStorePath(i.first)); + } + addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + } /* Copy the input sources from the eval store to the build store. */ @@ -350,6 +381,8 @@ void DerivationGoal::gaveUpOnSubstitution() void DerivationGoal::repairClosure() { + assert(drv->type().isPure()); + /* If we're repairing, we now know that our own outputs are valid. Now check whether the other paths in the outputs closure are good. If not, then start derivation goals for the derivations @@ -426,7 +459,8 @@ void DerivationGoal::inputsRealised() return; } - if (retrySubstitution) { + if (retrySubstitution && !retriedSubstitution) { + retriedSubstitution = true; haveDerivation(); return; } @@ -440,19 +474,40 @@ void DerivationGoal::inputsRealised() if (useDerivation) { auto & fullDrv = *dynamic_cast<Derivation *>(drv.get()); - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && - ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type())) - || fullDrv.type() == DerivationType::DeferredInputAddressed)) { + auto drvType = fullDrv.type(); + bool resolveDrv = std::visit(overloaded { + [&](const DerivationType::InputAddressed & ia) { + /* must resolve if deferred. */ + return ia.deferred; + }, + [&](const DerivationType::ContentAddressed & ca) { + return !fullDrv.inputDrvs.empty() && ( + ca.fixed + /* Can optionally resolve if fixed, which is good + for avoiding unnecessary rebuilds. */ + ? settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + /* Must resolve if floating and there are any inputs + drvs. */ + : true); + }, + [&](const DerivationType::Impure &) { + return true; + } + }, drvType.raw()); + + if (resolveDrv && !fullDrv.inputDrvs.empty()) { + settings.requireExperimentalFeature(Xp::CaDerivations); + /* We are be able to resolve this derivation based on the - now-known results of dependencies. If so, we become a stub goal - aliasing that resolved derivation goal */ - std::optional attempt = fullDrv.tryResolve(worker.store); + now-known results of dependencies. If so, we become a + stub goal aliasing that resolved derivation goal. */ + std::optional attempt = fullDrv.tryResolve(worker.store, inputDrvOutputs); assert(attempt); Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); - auto msg = fmt("Resolved derivation: '%s' -> '%s'", + auto msg = fmt("resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), worker.store.printStorePath(pathResolved)); act = std::make_unique<Activity>(*logger, lvlInfo, actBuildWaiting, msg, @@ -473,21 +528,13 @@ void DerivationGoal::inputsRealised() /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - assert(worker.evalStore.isValidPath(drvPath)); - auto outputs = worker.evalStore.queryPartialDerivationOutputMap(depDrvPath); - for (auto & j : wantedDepOutputs) { - if (outputs.count(j) > 0) { - auto optRealizedInput = outputs.at(j); - if (!optRealizedInput) - throw Error( - "derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); - worker.store.computeFSClosure(*optRealizedInput, inputPaths); - } else + for (auto & j : wantedDepOutputs) + if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) + worker.store.computeFSClosure(*outPath, inputPaths); + else throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); - } } } @@ -501,7 +548,7 @@ void DerivationGoal::inputsRealised() /* Don't repeat fixed-output derivations since they're already verified by their output hash.*/ - nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1; + nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1; /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a @@ -908,7 +955,7 @@ void DerivationGoal::buildDone() st = dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure : + !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } @@ -919,60 +966,53 @@ void DerivationGoal::buildDone() void DerivationGoal::resolvedFinished() { + trace("resolved derivation finished"); + assert(resolvedDrvGoal); auto resolvedDrv = *resolvedDrvGoal->drv; + auto & resolvedResult = resolvedDrvGoal->buildResult; - auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); + DrvOutputs builtOutputs; - StorePathSet outputPaths; + if (resolvedResult.success()) { + auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); - // `wantedOutputs` might be empty, which means “all the outputs” - auto realWantedOutputs = wantedOutputs; - if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv.outputNames(); + StorePathSet outputPaths; - DrvOutputs builtOutputs; + // `wantedOutputs` might be empty, which means “all the outputs” + auto realWantedOutputs = wantedOutputs; + if (realWantedOutputs.empty()) + realWantedOutputs = resolvedDrv.outputNames(); + + for (auto & wantedOutput : realWantedOutputs) { + assert(initialOutputs.count(wantedOutput) != 0); + assert(resolvedHashes.count(wantedOutput) != 0); + auto realisation = resolvedResult.builtOutputs.at( + DrvOutput { resolvedHashes.at(wantedOutput), wantedOutput }); + if (drv->type().isPure()) { + auto newRealisation = realisation; + newRealisation.id = DrvOutput { initialOutputs.at(wantedOutput).outputHash, wantedOutput }; + newRealisation.signatures.clear(); + if (!drv->type().isFixed()) + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); + signRealisation(newRealisation); + worker.store.registerDrvOutput(newRealisation); + } + outputPaths.insert(realisation.outPath); + builtOutputs.emplace(realisation.id, realisation); + } - for (auto & wantedOutput : realWantedOutputs) { - assert(initialOutputs.count(wantedOutput) != 0); - assert(resolvedHashes.count(wantedOutput) != 0); - auto realisation = worker.store.queryRealisation( - DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput} + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths ); - // We've just built it, but maybe the build failed, in which case the - // realisation won't be there - if (realisation) { - auto newRealisation = *realisation; - newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; - newRealisation.signatures.clear(); - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); - signRealisation(newRealisation); - worker.store.registerDrvOutput(newRealisation); - outputPaths.insert(realisation->outPath); - builtOutputs.emplace(realisation->id, *realisation); - } else { - // If we don't have a realisation, then it must mean that something - // failed when building the resolved drv - assert(!buildResult.success()); - } } - runPostBuildHook( - worker.store, - *logger, - drvPath, - outputPaths - ); - - auto status = [&]() { - auto & resolvedResult = resolvedDrvGoal->buildResult; - switch (resolvedResult.status) { - case BuildResult::AlreadyValid: - return BuildResult::ResolvesToAlreadyValid; - default: - return resolvedResult.status; - } - }(); + auto status = resolvedResult.status; + if (status == BuildResult::AlreadyValid) + status = BuildResult::ResolvesToAlreadyValid; done(status, std::move(builtOutputs)); } @@ -1221,7 +1261,8 @@ void DerivationGoal::flushLine() std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap() { - if (!useDerivation || drv->type() != DerivationType::CAFloating) { + assert(drv->type().isPure()); + if (!useDerivation || drv->type().hasKnownOutputPaths()) { std::map<std::string, std::optional<StorePath>> res; for (auto & [name, output] : drv->outputs) res.insert_or_assign(name, output.path(worker.store, drv->name, name)); @@ -1233,7 +1274,8 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri OutputPathMap DerivationGoal::queryDerivationOutputMap() { - if (!useDerivation || drv->type() != DerivationType::CAFloating) { + assert(drv->type().isPure()); + if (!useDerivation || drv->type().hasKnownOutputPaths()) { OutputPathMap res; for (auto & [name, output] : drv->outputsAndOptPaths(worker.store)) res.insert_or_assign(name, *output.second); @@ -1246,6 +1288,8 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() { + if (!drv->type().isPure()) return { false, {} }; + bool checkHash = buildMode == bmRepair; auto wantedOutputsLeft = wantedOutputs; DrvOutputs validOutputs; @@ -1289,6 +1333,7 @@ std::pair<bool, DrvOutputs> DerivationGoal::checkPathValidity() if (info.wanted && info.known && info.known->isValid()) validOutputs.emplace(drvOutput, Realisation { drvOutput, info.known->path }); } + // If we requested all the outputs via the empty set, we are always fine. // If we requested specific elements, the loop above removes all the valid // ones, so any that are left must be invalid. @@ -1326,9 +1371,7 @@ void DerivationGoal::done( { buildResult.status = status; if (ex) - // FIXME: strip: "error: " - buildResult.errorMsg = ex->what(); - amDone(buildResult.success() ? ecSuccess : ecFailed, ex); + buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg)); if (buildResult.status == BuildResult::TimedOut) worker.timedOut = true; if (buildResult.status == BuildResult::PermanentFailure) @@ -1355,7 +1398,21 @@ void DerivationGoal::done( fs.open(traceBuiltOutputsFile, std::fstream::out); fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } + + amDone(buildResult.success() ? ecSuccess : ecFailed, ex); } +void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) +{ + Goal::waiteeDone(waitee, result); + + if (waitee->buildResult.success()) + if (auto bfd = std::get_if<DerivedPath::Built>(&waitee->buildResult.path)) + for (auto & [output, realisation] : waitee->buildResult.builtOutputs) + inputDrvOutputs.insert_or_assign( + { bfd->drvPath, output.outputName }, + realisation.outPath); +} + } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index ea2db89b2..2d8bfd592 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -57,12 +57,21 @@ struct DerivationGoal : public Goal them. */ StringSet wantedOutputs; + /* Mapping from input derivations + output names to actual store + paths. This is filled in by waiteeDone() as each dependency + finishes, before inputsRealised() is reached, */ + std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs; + /* Whether additional wanted outputs have been added. */ bool needRestart = false; /* Whether to retry substituting the outputs after building the - inputs. */ - bool retrySubstitution; + inputs. This is done in case of an incomplete closure. */ + bool retrySubstitution = false; + + /* Whether we've retried substitution, in which case we won't try + again. */ + bool retriedSubstitution = false; /* The derivation stored at drvPath. */ std::unique_ptr<Derivation> drv; @@ -220,6 +229,8 @@ struct DerivationGoal : public Goal DrvOutputs builtOutputs = {}, std::optional<Error> ex = {}); + void waiteeDone(GoalPtr waitee, ExitCode result) override; + StorePathSet exportReferences(const StorePathSet & storePaths); }; diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index e50292c1e..b7f7b5ab1 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -41,7 +41,7 @@ void DrvOutputSubstitutionGoal::tryNext() if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ - debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string()); + debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index d2420b107..58e805f55 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -28,7 +28,7 @@ void Goal::addWaitee(GoalPtr waitee) void Goal::waiteeDone(GoalPtr waitee, ExitCode result) { - assert(waitees.find(waitee) != waitees.end()); + assert(waitees.count(waitee)); waitees.erase(waitee); trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 07c752bb9..35121c5d9 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -40,21 +40,21 @@ struct Goal : public std::enable_shared_from_this<Goal> WeakGoals waiters; /* Number of goals we are/were waiting for that have failed. */ - unsigned int nrFailed; + size_t nrFailed = 0; /* Number of substitution goals we are/were waiting for that failed because there are no substituters. */ - unsigned int nrNoSubstituters; + size_t nrNoSubstituters = 0; /* Number of substitution goals we are/were waiting for that failed because they had unsubstitutable references. */ - unsigned int nrIncompleteClosure; + size_t nrIncompleteClosure = 0; /* Name of this goal for debugging purposes. */ std::string name; /* Whether the goal is finished. */ - ExitCode exitCode; + ExitCode exitCode = ecBusy; /* Build result. */ BuildResult buildResult; @@ -65,10 +65,7 @@ struct Goal : public std::enable_shared_from_this<Goal> Goal(Worker & worker, DerivedPath path) : worker(worker) , buildResult { .path = std::move(path) } - { - nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; - exitCode = ecBusy; - } + { } virtual ~Goal() { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a372728f5..4c91fa4fb 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = !(derivationIsImpure(derivationType)) && !noChroot; + useChroot = derivationType.isSandboxed() && !noChroot; } auto & localStore = getLocalStore(); @@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder() "nogroup:x:65534:\n", sandboxGid())); /* Create /etc/hosts with localhost entry. */ - if (!(derivationIsImpure(derivationType))) + if (derivationType.isSandboxed()) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -704,6 +704,9 @@ void LocalDerivationGoal::startBuilder() /* Run the builder. */ printMsg(lvlChatty, "executing builder '%1%'", drv->builder); + printMsg(lvlChatty, "using builder args '%1%'", concatStringsSep(" ", drv->args)); + for (auto & i : drv->env) + printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); /* Create the log file. */ Path logFile = openLogFile(); @@ -796,7 +799,7 @@ void LocalDerivationGoal::startBuilder() us. */ - if (!(derivationIsImpure(derivationType))) + if (derivationType.isSandboxed()) privateNetwork = true; userNamespaceSync.create(); @@ -1049,7 +1052,7 @@ void LocalDerivationGoal::initEnv() derivation, tell the builder, so that for instance `fetchurl' can skip checking the output. On older Nixes, this environment variable won't be set, so `fetchurl' will do the check. */ - if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1"; + if (derivationType.isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; /* *Only* if this is a fixed-output derivation, propagate the values of the environment variables specified in the @@ -1060,7 +1063,7 @@ void LocalDerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (derivationIsImpure(derivationType)) { + if (!derivationType.isSandboxed()) { for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) env[i] = getEnv(i).value_or(""); } @@ -1340,6 +1343,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo next->queryMissing(allowed, willBuild, willSubstitute, unknown, downloadSize, narSize); } + + virtual std::optional<std::string> getBuildLog(const StorePath & path) override + { return std::nullopt; } + + virtual void addBuildLog(const StorePath & path, std::string_view log) override + { unsupported("addBuildLog"); } }; @@ -1668,7 +1677,7 @@ void LocalDerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (derivationIsImpure(derivationType)) { + if (!derivationType.isSandboxed()) { // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the @@ -1912,7 +1921,7 @@ void LocalDerivationGoal::runChild() sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; - if (derivationIsImpure(derivationType)) + if (!derivationType.isSandboxed()) sandboxProfile += "(import \"sandbox-network.sb\")\n"; /* Add the output paths we'll use at build-time to the chroot */ @@ -2273,7 +2282,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() return res; }; - auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo { + auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo { auto & st = outputStats.at(outputName); if (outputHash.method == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ @@ -2340,7 +2349,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs() ValidPathInfo newInfo = std::visit(overloaded { - [&](const DerivationOutputInputAddressed & output) { + [&](const DerivationOutput::InputAddressed & output) { /* input-addressed case */ auto requiredFinalPath = output.path; /* Preemptively add rewrite rule for final hash, as that is @@ -2360,8 +2369,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs() return newInfo0; }, - [&](const DerivationOutputCAFixed & dof) { - auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating { + [&](const DerivationOutput::CAFixed & dof) { + auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { .method = dof.hash.method, .hashType = dof.hash.hash.type, }); @@ -2383,17 +2392,24 @@ DrvOutputs LocalDerivationGoal::registerOutputs() return newInfo0; }, - [&](DerivationOutputCAFloating & dof) { + [&](const DerivationOutput::CAFloating & dof) { return newInfoFromCA(dof); }, - [&](DerivationOutputDeferred) -> ValidPathInfo { + [&](const DerivationOutput::Deferred &) -> ValidPathInfo { // No derivation should reach that point without having been // rewritten first assert(false); }, - }, output.output); + [&](const DerivationOutput::Impure & doi) { + return newInfoFromCA(DerivationOutput::CAFloating { + .method = doi.method, + .hashType = doi.hashType, + }); + }, + + }, output.raw()); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ @@ -2603,11 +2619,14 @@ DrvOutputs LocalDerivationGoal::registerOutputs() }, .outPath = newInfo.path }; - if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) + && drv->type().isPure()) + { signRealisation(thisRealisation); worker.store.registerDrvOutput(thisRealisation); } - builtOutputs.emplace(thisRealisation.id, thisRealisation); + if (wantOutput(outputName, wantedOutputs)) + builtOutputs.emplace(thisRealisation.id, thisRealisation); } return builtOutputs; diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 31e6dbc9f..ca5218627 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -24,9 +24,16 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } -void PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status) +void PathSubstitutionGoal::done( + ExitCode result, + BuildResult::Status status, + std::optional<std::string> errorMsg) { buildResult.status = status; + if (errorMsg) { + debug(*errorMsg); + buildResult.errorMsg = *errorMsg; + } amDone(result); } @@ -67,12 +74,14 @@ void PathSubstitutionGoal::tryNext() if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal with it. */ - debug("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)); /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - done(substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters); + done( + substituterFailed ? ecFailed : ecNoSubstituters, + BuildResult::NoSubstituters, + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); if (substituterFailed) { worker.failedSubstitutions++; @@ -169,10 +178,10 @@ void PathSubstitutionGoal::referencesValid() trace("all references realised"); if (nrFailed > 0) { - debug("some references of path '%s' could not be realised", worker.store.printStorePath(storePath)); done( nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, - BuildResult::DependencyFailed); + BuildResult::DependencyFailed, + fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); return; } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 946f13841..a73f8e666 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -53,7 +53,10 @@ struct PathSubstitutionGoal : public Goal /* Content address for recomputing store path */ std::optional<ContentAddress> ca; - void done(ExitCode result, BuildResult::Status status); + void done( + ExitCode result, + BuildResult::Status status, + std::optional<std::string> errorMsg = {}); public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 25d015cb9..6f6ad57cb 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -47,9 +47,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw; } - /* The files below are special-cased to that they don't show up - * in user profiles, either because they are useless, or - * because they would cauase pointless collisions (e.g., each + /* The files below are special-cased to that they don't show + * up in user profiles, either because they are useless, or + * because they would cause pointless collisions (e.g., each * Python package brings its own * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.) */ @@ -57,7 +57,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, hasSuffix(srcFile, "/nix-support") || hasSuffix(srcFile, "/perllocal.pod") || hasSuffix(srcFile, "/info/dir") || - hasSuffix(srcFile, "/log")) + hasSuffix(srcFile, "/log") || + hasSuffix(srcFile, "/manifest.nix") || + hasSuffix(srcFile, "/manifest.json")) continue; else if (S_ISDIR(srcSt.st_mode)) { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e6760664c..de69b50ee 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -3,7 +3,9 @@ #include "worker-protocol.hh" #include "build-result.hh" #include "store-api.hh" +#include "store-cast.hh" #include "gc-store.hh" +#include "log-store.hh" #include "path-with-outputs.hh" #include "finally.hh" #include "archive.hh" @@ -558,6 +560,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); + auto drvType = drv.type(); + /* Content-addressed derivations are trustless because their output paths are verified by their content alone, so any derivation is free to try to produce such a path. @@ -590,12 +594,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store, derivations, we throw out the precomputed output paths and just store the hashes, so there aren't two competing sources of truth an attacker could exploit. */ - if (drv.type() == DerivationType::InputAddressed && !trusted) + if (!(drvType.isCA() || trusted)) throw Error("you are not privileged to build input-addressed derivations"); /* Make sure that the non-input-addressed derivations that got this far are in fact content-addressed if we don't trust them. */ - assert(derivationIsCA(drv.type()) || trusted); + assert(drvType.isCA() || trusted); /* Recompute the derivation path when we cannot trust the original. */ if (!trusted) { @@ -604,7 +608,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, original not-necessarily-resolved derivation to verify the drv derivation as adequate claim to the input-addressed output paths. */ - assert(derivationIsCA(drv.type())); + assert(drvType.isCA()); Derivation drv2; static_cast<BasicDerivation &>(drv2) = drv; @@ -645,7 +649,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, Path path = absPath(readString(from)); logger->startWork(); - auto & gcStore = requireGcStore(*store); + auto & gcStore = require<GcStore>(*store); gcStore.addIndirectRoot(path); logger->stopWork(); @@ -663,7 +667,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, case wopFindRoots: { logger->startWork(); - auto & gcStore = requireGcStore(*store); + auto & gcStore = require<GcStore>(*store); Roots roots = gcStore.findRoots(!trusted); logger->stopWork(); @@ -695,7 +699,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); if (options.ignoreLiveness) throw Error("you are not allowed to ignore liveness"); - auto & gcStore = requireGcStore(*store); + auto & gcStore = require<GcStore>(*store); gcStore.collectGarbage(options, results); logger->stopWork(); @@ -953,11 +957,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store, logger->startWork(); if (!trusted) throw Error("you are not privileged to add logs"); + auto & logStore = require<LogStore>(*store); { FramedSource source(from); StringSink sink; source.drainInto(sink); - store->addBuildLog(path, sink.s); + logStore.addBuildLog(path, sink.s); } logger->stopWork(); to << 1; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a49be0057..1c695de82 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -11,72 +11,114 @@ namespace nix { std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const { return std::visit(overloaded { - [](const DerivationOutputInputAddressed & doi) -> std::optional<StorePath> { + [](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> { return { doi.path }; }, - [&](const DerivationOutputCAFixed & dof) -> std::optional<StorePath> { + [&](const DerivationOutput::CAFixed & dof) -> std::optional<StorePath> { return { dof.path(store, drvName, outputName) }; }, - [](const DerivationOutputCAFloating & dof) -> std::optional<StorePath> { + [](const DerivationOutput::CAFloating & dof) -> std::optional<StorePath> { return std::nullopt; }, - [](const DerivationOutputDeferred &) -> std::optional<StorePath> { + [](const DerivationOutput::Deferred &) -> std::optional<StorePath> { return std::nullopt; }, - }, output); + [](const DerivationOutput::Impure &) -> std::optional<StorePath> { + return std::nullopt; + }, + }, raw()); } -StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const { +StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const +{ return store.makeFixedOutputPath( hash.method, hash.hash, outputPathName(drvName, outputName)); } -bool derivationIsCA(DerivationType dt) { - switch (dt) { - case DerivationType::InputAddressed: return false; - case DerivationType::CAFixed: return true; - case DerivationType::CAFloating: return true; - case DerivationType::DeferredInputAddressed: return false; - }; - // Since enums can have non-variant values, but making a `default:` would - // disable exhaustiveness warnings. - assert(false); +bool DerivationType::isCA() const +{ + /* Normally we do the full `std::visit` to make sure we have + exhaustively handled all variants, but so long as there is a + variant called `ContentAddressed`, it must be the only one for + which `isCA` is true for this to make sense!. */ + return std::visit(overloaded { + [](const InputAddressed & ia) { + return false; + }, + [](const ContentAddressed & ca) { + return true; + }, + [](const Impure &) { + return true; + }, + }, raw()); } -bool derivationIsFixed(DerivationType dt) { - switch (dt) { - case DerivationType::InputAddressed: return false; - case DerivationType::CAFixed: return true; - case DerivationType::CAFloating: return false; - case DerivationType::DeferredInputAddressed: return false; - }; - assert(false); +bool DerivationType::isFixed() const +{ + return std::visit(overloaded { + [](const InputAddressed & ia) { + return false; + }, + [](const ContentAddressed & ca) { + return ca.fixed; + }, + [](const Impure &) { + return false; + }, + }, raw()); } -bool derivationHasKnownOutputPaths(DerivationType dt) { - switch (dt) { - case DerivationType::InputAddressed: return true; - case DerivationType::CAFixed: return true; - case DerivationType::CAFloating: return false; - case DerivationType::DeferredInputAddressed: return false; - }; - assert(false); +bool DerivationType::hasKnownOutputPaths() const +{ + return std::visit(overloaded { + [](const InputAddressed & ia) { + return !ia.deferred; + }, + [](const ContentAddressed & ca) { + return ca.fixed; + }, + [](const Impure &) { + return false; + }, + }, raw()); } -bool derivationIsImpure(DerivationType dt) { - switch (dt) { - case DerivationType::InputAddressed: return false; - case DerivationType::CAFixed: return true; - case DerivationType::CAFloating: return false; - case DerivationType::DeferredInputAddressed: return false; - }; - assert(false); +bool DerivationType::isSandboxed() const +{ + return std::visit(overloaded { + [](const InputAddressed & ia) { + return true; + }, + [](const ContentAddressed & ca) { + return ca.sandboxed; + }, + [](const Impure &) { + return false; + }, + }, raw()); +} + + +bool DerivationType::isPure() const +{ + return std::visit(overloaded { + [](const InputAddressed & ia) { + return true; + }, + [](const ContentAddressed & ca) { + return true; + }, + [](const Impure &) { + return false; + }, + }, raw()); } @@ -177,37 +219,36 @@ static DerivationOutput parseDerivationOutput(const Store & store, hashAlgo = hashAlgo.substr(2); } const auto hashType = parseHashType(hashAlgo); - if (hash != "") { + if (hash == "impure") { + settings.requireExperimentalFeature(Xp::ImpureDerivations); + assert(pathS == ""); + return DerivationOutput::Impure { + .method = std::move(method), + .hashType = std::move(hashType), + }; + } else if (hash != "") { validatePath(pathS); - return DerivationOutput { - .output = DerivationOutputCAFixed { - .hash = FixedOutputHash { - .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed(hash, hashType), - }, + return DerivationOutput::CAFixed { + .hash = FixedOutputHash { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed(hash, hashType), }, }; } else { settings.requireExperimentalFeature(Xp::CaDerivations); assert(pathS == ""); - return DerivationOutput { - .output = DerivationOutputCAFloating { - .method = std::move(method), - .hashType = std::move(hashType), - }, + return DerivationOutput::CAFloating { + .method = std::move(method), + .hashType = std::move(hashType), }; } } else { if (pathS == "") { - return DerivationOutput { - .output = DerivationOutputDeferred { } - }; + return DerivationOutput::Deferred { }; } validatePath(pathS); - return DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = store.parseStorePath(pathS), - } + return DerivationOutput::InputAddressed { + .path = store.parseStorePath(pathS), }; } } @@ -335,27 +376,33 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); std::visit(overloaded { - [&](const DerivationOutputInputAddressed & doi) { + [&](const DerivationOutput::InputAddressed & doi) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path)); s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); }, - [&](const DerivationOutputCAFixed & dof) { + [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.hash.printMethodAlgo()); s += ','; printUnquotedString(s, dof.hash.hash.to_string(Base16, false)); }, - [&](const DerivationOutputCAFloating & dof) { + [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); s += ','; printUnquotedString(s, ""); }, - [&](const DerivationOutputDeferred &) { + [&](const DerivationOutput::Deferred &) { s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); + }, + [&](const DerivationOutputImpure & doi) { + // FIXME + s += ','; printUnquotedString(s, ""); + s += ','; printUnquotedString(s, makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); + s += ','; printUnquotedString(s, "impure"); } - }, i.second.output); + }, i.second.raw()); s += ')'; } @@ -419,49 +466,100 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName DerivationType BasicDerivation::type() const { - std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs; + std::set<std::string_view> + inputAddressedOutputs, + fixedCAOutputs, + floatingCAOutputs, + deferredIAOutputs, + impureOutputs; std::optional<HashType> floatingHashType; + for (auto & i : outputs) { std::visit(overloaded { - [&](const DerivationOutputInputAddressed &) { + [&](const DerivationOutput::InputAddressed &) { inputAddressedOutputs.insert(i.first); }, - [&](const DerivationOutputCAFixed &) { + [&](const DerivationOutput::CAFixed &) { fixedCAOutputs.insert(i.first); }, - [&](const DerivationOutputCAFloating & dof) { + [&](const DerivationOutput::CAFloating & dof) { floatingCAOutputs.insert(i.first); if (!floatingHashType) { floatingHashType = dof.hashType; } else { if (*floatingHashType != dof.hashType) - throw Error("All floating outputs must use the same hash type"); + throw Error("all floating outputs must use the same hash type"); } }, - [&](const DerivationOutputDeferred &) { - deferredIAOutputs.insert(i.first); + [&](const DerivationOutput::Deferred &) { + deferredIAOutputs.insert(i.first); }, - }, i.second.output); + [&](const DerivationOutput::Impure &) { + impureOutputs.insert(i.first); + }, + }, i.second.raw()); } - if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { - throw Error("Must have at least one output"); - } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { - return DerivationType::InputAddressed; - } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) { + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) + throw Error("must have at least one output"); + + if (!inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) + return DerivationType::InputAddressed { + .deferred = false, + }; + + if (inputAddressedOutputs.empty() + && !fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) + { if (fixedCAOutputs.size() > 1) // FIXME: Experimental feature? - throw Error("Only one fixed output is allowed for now"); + throw Error("only one fixed output is allowed for now"); if (*fixedCAOutputs.begin() != "out") - throw Error("Single fixed output must be named \"out\""); - return DerivationType::CAFixed; - } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) { - return DerivationType::CAFloating; - } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) { - return DerivationType::DeferredInputAddressed; - } else { - throw Error("Can't mix derivation output types"); + throw Error("single fixed output must be named \"out\""); + return DerivationType::ContentAddressed { + .sandboxed = false, + .fixed = true, + }; } + + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && !floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && impureOutputs.empty()) + return DerivationType::ContentAddressed { + .sandboxed = true, + .fixed = false, + }; + + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && !deferredIAOutputs.empty() + && impureOutputs.empty()) + return DerivationType::InputAddressed { + .deferred = true, + }; + + if (inputAddressedOutputs.empty() + && fixedCAOutputs.empty() + && floatingCAOutputs.empty() + && deferredIAOutputs.empty() + && !impureOutputs.empty()) + return DerivationType::Impure { }; + + throw Error("can't mix derivation output types"); } @@ -473,7 +571,7 @@ Sync<DrvHashes> drvHashes; /* Look up the derivation by value and memoize the `hashDerivationModulo` call. */ -static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath) +static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) { { auto hashes = drvHashes.lock(); @@ -508,88 +606,83 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & don't leak the provenance of fixed outputs, reducing pointless cache misses as the build itself won't know this. */ -DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) +DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs) { - bool isDeferred = false; + auto type = drv.type(); + /* Return a fixed hash for fixed-output derivations. */ - switch (drv.type()) { - case DerivationType::CAFixed: { + if (type.isFixed()) { std::map<std::string, Hash> outputHashes; for (const auto & i : drv.outputs) { - auto & dof = std::get<DerivationOutputCAFixed>(i.second.output); + auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw()); auto hash = hashString(htSHA256, "fixed:out:" + dof.hash.printMethodAlgo() + ":" + dof.hash.hash.to_string(Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } - return outputHashes; + return DrvHash { + .hashes = outputHashes, + .kind = DrvHash::Kind::Regular, + }; } - case DerivationType::CAFloating: - isDeferred = true; - break; - case DerivationType::InputAddressed: - break; - case DerivationType::DeferredInputAddressed: - break; + + if (!type.isPure()) { + std::map<std::string, Hash> outputHashes; + for (const auto & [outputName, _] : drv.outputs) + outputHashes.insert_or_assign(outputName, impureOutputHash); + return DrvHash { + .hashes = outputHashes, + .kind = DrvHash::Kind::Deferred, + }; } - /* For other derivations, replace the inputs paths with recursive - calls to this function. */ + auto kind = std::visit(overloaded { + [](const DerivationType::InputAddressed & ia) { + /* This might be a "pesimistically" deferred output, so we don't + "taint" the kind yet. */ + return DrvHash::Kind::Regular; + }, + [](const DerivationType::ContentAddressed & ca) { + return ca.fixed + ? DrvHash::Kind::Regular + : DrvHash::Kind::Deferred; + }, + [](const DerivationType::Impure &) -> DrvHash::Kind { + assert(false); + } + }, drv.type().raw()); + std::map<std::string, StringSet> inputs2; - for (auto & i : drv.inputDrvs) { - const auto & res = pathDerivationModulo(store, i.first); - std::visit(overloaded { - // Regular non-CA derivation, replace derivation - [&](const Hash & drvHash) { - inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second); - }, - [&](const DeferredHash & deferredHash) { - isDeferred = true; - inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second); - }, - // CA derivation's output hashes - [&](const CaOutputHashes & outputHashes) { - std::set<std::string> justOut = { "out" }; - for (auto & output : i.second) { - /* Put each one in with a single "out" output.. */ - const auto h = outputHashes.at(output); - inputs2.insert_or_assign( - h.to_string(Base16, false), - justOut); - } - }, - }, res); + for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { + // Avoid lambda capture restriction with standard / Clang + auto & inputOutputs = inputOutputs0; + const auto & res = pathDerivationModulo(store, drvPath); + if (res.kind == DrvHash::Kind::Deferred) + kind = DrvHash::Kind::Deferred; + for (auto & outputName : inputOutputs) { + const auto h = res.hashes.at(outputName); + inputs2[h.to_string(Base16, false)].insert(outputName); + } } auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); - if (isDeferred) - return DeferredHash { hash }; - else - return hash; + std::map<std::string, Hash> outputHashes; + for (const auto & [outputName, _] : drv.outputs) { + outputHashes.insert_or_assign(outputName, hash); + } + + return DrvHash { + .hashes = outputHashes, + .kind = kind, + }; } std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv) { - std::map<std::string, Hash> res; - std::visit(overloaded { - [&](const Hash & drvHash) { - for (auto & outputName : drv.outputNames()) { - res.insert({outputName, drvHash}); - } - }, - [&](const DeferredHash & deferredHash) { - for (auto & outputName : drv.outputNames()) { - res.insert({outputName, deferredHash.hash}); - } - }, - [&](const CaOutputHashes & outputHashes) { - res = outputHashes; - }, - }, hashDerivationModulo(store, drv, true)); - return res; + return hashDerivationModulo(store, drv, true).hashes; } @@ -616,7 +709,8 @@ StringSet BasicDerivation::outputNames() const return names; } -DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const +{ DerivationOutputsAndOptPaths outsAndOptPaths; for (auto output : outputs) outsAndOptPaths.insert(std::make_pair( @@ -627,7 +721,8 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & s return outsAndOptPaths; } -std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { +std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) +{ auto nameWithSuffix = drvPath.name(); constexpr std::string_view extension = ".drv"; assert(hasSuffix(nameWithSuffix, extension)); @@ -669,27 +764,32 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr for (auto & i : drv.outputs) { out << i.first; std::visit(overloaded { - [&](const DerivationOutputInputAddressed & doi) { + [&](const DerivationOutput::InputAddressed & doi) { out << store.printStorePath(doi.path) << "" << ""; }, - [&](const DerivationOutputCAFixed & dof) { + [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.hash.printMethodAlgo() << dof.hash.hash.to_string(Base16, false); }, - [&](const DerivationOutputCAFloating & dof) { + [&](const DerivationOutput::CAFloating & dof) { out << "" << (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)) << ""; }, - [&](const DerivationOutputDeferred &) { + [&](const DerivationOutput::Deferred &) { out << "" << "" << ""; }, - }, i.second.output); + [&](const DerivationOutput::Impure & doi) { + out << "" + << (makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)) + << "impure"; + }, + }, i.second.raw()); } worker_proto::write(store, out, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; @@ -714,21 +814,19 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath } -static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { - - debug("Rewriting the derivation"); - - for (auto &rewrite: rewrites) { +static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) +{ + for (auto & rewrite : rewrites) { debug("rewriting %s as %s", rewrite.first, rewrite.second); } drv.builder = rewriteStrings(drv.builder, rewrites); - for (auto & arg: drv.args) { + for (auto & arg : drv.args) { arg = rewriteStrings(arg, rewrites); } StringPairs newEnv; - for (auto & envVar: drv.env) { + for (auto & envVar : drv.env) { auto envName = rewriteStrings(envVar.first, rewrites); auto envValue = rewriteStrings(envVar.second, rewrites); newEnv.emplace(envName, envValue); @@ -737,43 +835,52 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative<DerivationOutputDeferred>(output.output)) { - Hash h = std::get<Hash>(hashModulo); + if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) { + auto & h = hashModulo.hashes.at(outputName); auto outPath = store.makeOutputPath(outputName, h, drv.name); drv.env[outputName] = store.printStorePath(outPath); - output = DerivationOutput { - .output = DerivationOutputInputAddressed { - .path = std::move(outPath), - }, + output = DerivationOutput::InputAddressed { + .path = std::move(outPath), }; } } } -std::optional<BasicDerivation> Derivation::tryResolve(Store & store) { +std::optional<BasicDerivation> Derivation::tryResolve(Store & store) const +{ + std::map<std::pair<StorePath, std::string>, StorePath> inputDrvOutputs; + + for (auto & input : inputDrvs) + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first)) + if (outputPath) + inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath); + + return tryResolve(store, inputDrvOutputs); +} + +std::optional<BasicDerivation> Derivation::tryResolve( + Store & store, + const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const +{ BasicDerivation resolved { *this }; // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & input : inputDrvs) { - auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first); - StringSet newOutputNames; - for (auto & outputName : input.second) { - auto actualPathOpt = inputDrvOutputs.at(outputName); - if (!actualPathOpt) { - warn("output %s of input %s missing, aborting the resolving", + for (auto & [inputDrv, inputOutputs] : inputDrvs) { + for (auto & outputName : inputOutputs) { + if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { + inputRewrites.emplace( + downstreamPlaceholder(store, inputDrv, outputName), + store.printStorePath(*actualPath)); + resolved.inputSrcs.insert(*actualPath); + } else { + warn("output '%s' of input '%s' missing, aborting the resolving", outputName, - store.printStorePath(input.first) - ); - return std::nullopt; + store.printStorePath(inputDrv)); + return {}; } - auto actualPath = *actualPathOpt; - inputRewrites.emplace( - downstreamPlaceholder(store, input.first, outputName), - store.printStorePath(actualPath)); - resolved.inputSrcs.insert(std::move(actualPath)); } } @@ -782,4 +889,6 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) { return resolved; } +const Hash impureOutputHash = hashString(htSHA256, "impure"); + } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 132de82b6..af198a767 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "hash.hh" #include "content-address.hh" +#include "repair-flag.hh" #include "sync.hh" #include <map> @@ -40,23 +41,47 @@ struct DerivationOutputCAFloating }; /* Input-addressed output which depends on a (CA) derivation whose hash isn't - * known atm + * known yet. */ struct DerivationOutputDeferred {}; -struct DerivationOutput +/* Impure output which is moved to a content-addressed location (like + CAFloating) but isn't registered as a realization. + */ +struct DerivationOutputImpure +{ + /* information used for expected hash computation */ + FileIngestionMethod method; + HashType hashType; +}; + +typedef std::variant< + DerivationOutputInputAddressed, + DerivationOutputCAFixed, + DerivationOutputCAFloating, + DerivationOutputDeferred, + DerivationOutputImpure +> _DerivationOutputRaw; + +struct DerivationOutput : _DerivationOutputRaw { - std::variant< - DerivationOutputInputAddressed, - DerivationOutputCAFixed, - DerivationOutputCAFloating, - DerivationOutputDeferred - > output; + using Raw = _DerivationOutputRaw; + using Raw::Raw; + + using InputAddressed = DerivationOutputInputAddressed; + using CAFixed = DerivationOutputCAFixed; + using CAFloating = DerivationOutputCAFloating; + using Deferred = DerivationOutputDeferred; + using Impure = DerivationOutputImpure; /* Note, when you use this function you should make sure that you're passing the right derivation name. When in doubt, you should use the safer interface provided by BasicDerivation::outputsAndOptPaths */ std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } }; typedef std::map<std::string, DerivationOutput> DerivationOutputs; @@ -72,30 +97,62 @@ typedef std::map<std::string, std::pair<DerivationOutput, std::optional<StorePat output IDs we are interested in. */ typedef std::map<StorePath, StringSet> DerivationInputs; -enum struct DerivationType : uint8_t { - InputAddressed, - DeferredInputAddressed, - CAFixed, - CAFloating, +struct DerivationType_InputAddressed { + bool deferred; }; -/* Do the outputs of the derivation have paths calculated from their content, - or from the derivation itself? */ -bool derivationIsCA(DerivationType); - -/* Is the content of the outputs fixed a-priori via a hash? Never true for - non-CA derivations. */ -bool derivationIsFixed(DerivationType); +struct DerivationType_ContentAddressed { + bool sandboxed; + bool fixed; +}; -/* Is the derivation impure and needs to access non-deterministic resources, or - pure and can be sandboxed? Note that whether or not we actually sandbox the - derivation is controlled separately. Never true for non-CA derivations. */ -bool derivationIsImpure(DerivationType); +struct DerivationType_Impure { +}; -/* Does the derivation knows its own output paths? - * Only true when there's no floating-ca derivation involved in the closure. - */ -bool derivationHasKnownOutputPaths(DerivationType); +typedef std::variant< + DerivationType_InputAddressed, + DerivationType_ContentAddressed, + DerivationType_Impure +> _DerivationTypeRaw; + +struct DerivationType : _DerivationTypeRaw { + using Raw = _DerivationTypeRaw; + using Raw::Raw; + using InputAddressed = DerivationType_InputAddressed; + using ContentAddressed = DerivationType_ContentAddressed; + using Impure = DerivationType_Impure; + + /* Do the outputs of the derivation have paths calculated from their content, + or from the derivation itself? */ + bool isCA() const; + + /* Is the content of the outputs fixed a-priori via a hash? Never true for + non-CA derivations. */ + bool isFixed() const; + + /* Whether the derivation is fully sandboxed. If false, the + sandbox is opened up, e.g. the derivation has access to the + network. Note that whether or not we actually sandbox the + derivation is controlled separately. Always true for non-CA + derivations. */ + bool isSandboxed() const; + + /* Whether the derivation is expected to produce the same result + every time, and therefore it only needs to be built once. This + is only false for derivations that have the attribute '__impure + = true'. */ + bool isPure() const; + + /* Does the derivation knows its own output paths? + Only true when there's no floating-ca derivation involved in the + closure, or if fixed output. + */ + bool hasKnownOutputPaths() const; + + inline const Raw & raw() const { + return static_cast<const Raw &>(*this); + } +}; struct BasicDerivation { @@ -140,7 +197,14 @@ struct Derivation : BasicDerivation added directly to input sources. 2. Input placeholders are replaced with realized input store paths. */ - std::optional<BasicDerivation> tryResolve(Store & store); + std::optional<BasicDerivation> tryResolve(Store & store) const; + + /* Like the above, but instead of querying the Nix database for + realisations, uses a given mapping from input derivation paths + + output names to actual output store paths. */ + std::optional<BasicDerivation> tryResolve( + Store & store, + const std::map<std::pair<StorePath, std::string>, StorePath> & inputDrvOutputs) const; Derivation() = default; Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } @@ -150,8 +214,6 @@ struct Derivation : BasicDerivation class Store; -enum RepairFlag : bool { NoRepair = false, Repair = true }; - /* Write a derivation to the Nix store, and return its path. */ StorePath writeDerivation(Store & store, const Derivation & drv, @@ -171,17 +233,27 @@ bool isDerivation(const std::string & fileName); the output name is "out". */ std::string outputPathName(std::string_view drvName, std::string_view outputName); -// known CA drv's output hashes, current just for fixed-output derivations -// whose output hashes are always known since they are fixed up-front. -typedef std::map<std::string, Hash> CaOutputHashes; -struct DeferredHash { Hash hash; }; +// The hashes modulo of a derivation. +// +// Each output is given a hash, although in practice only the content-addressed +// derivations (fixed-output or not) will have a different hash for each +// output. +struct DrvHash { + std::map<std::string, Hash> hashes; -typedef std::variant< - Hash, // regular DRV normalized hash - CaOutputHashes, // Fixed-output derivation hashes - DeferredHash // Deferred hashes for floating outputs drvs and their dependencies -> DrvHashModulo; + enum struct Kind : bool { + // Statically determined derivations. + // This hash will be directly used to compute the output paths + Regular, + // Floating-output derivations (and their reverse dependencies). + Deferred, + }; + + Kind kind; +}; + +void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept; /* Returns hashes with the details of fixed-output subderivations expunged. @@ -206,16 +278,18 @@ typedef std::variant< ATerm, after subderivations have been likewise expunged from that derivation. */ -DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); +DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs); /* Return a map associating each output to a hash that uniquely identifies its derivation (modulo the self-references). + + FIXME: what is the Hash in this map? */ -std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv); +std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation & drv); /* Memoisation of hashDerivationModulo(). */ -typedef std::map<StorePath, DrvHashModulo> DrvHashes; +typedef std::map<StorePath, DrvHash> DrvHashes; // FIXME: global, though at least thread-safe. extern Sync<DrvHashes> drvHashes; @@ -245,4 +319,6 @@ std::string hashPlaceholder(const std::string_view outputName); dependency which is a CA derivation. */ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath, std::string_view outputName); +extern const Hash impureOutputHash; + } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 194489580..319b1c790 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,4 +1,5 @@ #include "derived-path.hh" +#include "derivations.hh" #include "store-api.hh" #include <nlohmann/json.hpp> @@ -11,6 +12,21 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const { return res; } +nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const { + nlohmann::json res; + res["drvPath"] = store->printStorePath(drvPath); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath); + for (const auto& output : outputs) { + if (knownOutputs.at(output)) + res["outputs"][output] = store->printStorePath(knownOutputs.at(output).value()); + else + res["outputs"][output] = nullptr; + } + return res; +} + nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const { nlohmann::json res; res["drvPath"] = store->printStorePath(drvPath); @@ -35,16 +51,22 @@ StorePathSet BuiltPath::outPaths() const ); } -nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store) { +template<typename T> +nlohmann::json stuffToJSON(const std::vector<T> & ts, ref<Store> store) { auto res = nlohmann::json::array(); - for (const BuiltPath & buildable : buildables) { - std::visit([&res, store](const auto & buildable) { - res.push_back(buildable.toJSON(store)); - }, buildable.raw()); + for (const T & t : ts) { + std::visit([&res, store](const auto & t) { + res.push_back(t.toJSON(store)); + }, t.raw()); } return res; } +nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store) +{ return stuffToJSON<BuiltPath>(buildables, store); } +nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store) +{ return stuffToJSON<DerivedPath>(paths, store); } + std::string DerivedPath::Opaque::to_string(const Store & store) const { return store.printStorePath(path); diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 9d6ace069..24a0ae773 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -25,6 +25,9 @@ struct DerivedPathOpaque { nlohmann::json toJSON(ref<Store> store) const; std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); + + bool operator < (const DerivedPathOpaque & b) const + { return path < b.path; } }; /** @@ -45,6 +48,10 @@ struct DerivedPathBuilt { std::string to_string(const Store & store) const; static DerivedPathBuilt parse(const Store & store, std::string_view); + nlohmann::json toJSON(ref<Store> store) const; + + bool operator < (const DerivedPathBuilt & b) const + { return std::make_pair(drvPath, outputs) < std::make_pair(b.drvPath, b.outputs); } }; using _DerivedPathRaw = std::variant< @@ -119,5 +126,6 @@ typedef std::vector<DerivedPath> DerivedPaths; typedef std::vector<BuiltPath> BuiltPaths; nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store); +nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref<Store> store); } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index ca61e3937..40e7cf52c 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -31,7 +31,7 @@ struct FileTransferSettings : Config R"( The timeout (in seconds) for establishing connections in the binary cache substituter. It corresponds to `curl`’s - `--connect-timeout` option. + `--connect-timeout` option. A value of 0 means no limit. )"}; Setting<unsigned long> stalledDownloadTimeout{ @@ -123,8 +123,6 @@ public: template<typename... Args> FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args); - - virtual const char* sname() const override { return "FileTransferError"; } }; bool isUri(std::string_view s); diff --git a/src/libstore/gc-store.cc b/src/libstore/gc-store.cc deleted file mode 100644 index 3dbdec53b..000000000 --- a/src/libstore/gc-store.cc +++ /dev/null @@ -1,13 +0,0 @@ -#include "gc-store.hh" - -namespace nix { - -GcStore & requireGcStore(Store & store) -{ - auto * gcStore = dynamic_cast<GcStore *>(&store); - if (!gcStore) - throw UsageError("Garbage collection not supported by this store"); - return *gcStore; -} - -} diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 829f70dc4..b3cbbad74 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -61,6 +61,8 @@ struct GCResults struct GcStore : public virtual Store { + inline static std::string operationName = "Garbage collection"; + /* Add an indirect root, which is merely a symlink to `path' from /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed to be a symlink to a store path. The garbage collector will @@ -79,6 +81,4 @@ struct GcStore : public virtual Store virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; }; -GcStore & requireGcStore(Store & store); - } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 024da66c1..f65fb1b2e 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -678,7 +678,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) alive.insert(start); try { StorePathSet closure; - computeFSClosure(*path, closure); + computeFSClosure(*path, closure, + /* flipDirection */ false, gcKeepOutputs, gcKeepDerivations); for (auto & p : closure) alive.insert(p); } catch (InvalidPath &) { } @@ -841,7 +842,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (unlink(path.c_str()) == -1) throw SysError("deleting '%1%'", path); - results.bytesFreed += st.st_size; + /* Do not accound for deleted file here. Rely on deletePath() + accounting. */ } struct stat st; diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index fbd49dc2c..e6fb3201a 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -2,6 +2,7 @@ #include "store-api.hh" #include "gc-store.hh" +#include "log-store.hh" namespace nix { @@ -24,7 +25,10 @@ struct LocalFSStoreConfig : virtual StoreConfig "physical path to the Nix store"}; }; -class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store, virtual GcStore +class LocalFSStore : public virtual LocalFSStoreConfig, + public virtual Store, + public virtual GcStore, + public virtual LogStore { public: diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1ee71b1c0..d77fff963 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -695,31 +695,34 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat // combinations that are currently prohibited. drv.type(); - std::optional<Hash> h; + std::optional<DrvHash> hashesModulo; for (auto & i : drv.outputs) { std::visit(overloaded { - [&](const DerivationOutputInputAddressed & doia) { - if (!h) { + [&](const DerivationOutput::InputAddressed & doia) { + if (!hashesModulo) { // somewhat expensive so we do lazily - auto temp = hashDerivationModulo(*this, drv, true); - h = std::get<Hash>(temp); + hashesModulo = hashDerivationModulo(*this, drv, true); } - StorePath recomputed = makeOutputPath(i.first, *h, drvName); + StorePath recomputed = makeOutputPath(i.first, hashesModulo->hashes.at(i.first), drvName); if (doia.path != recomputed) throw Error("derivation '%s' has incorrect output '%s', should be '%s'", printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed)); envHasRightPath(doia.path, i.first); }, - [&](const DerivationOutputCAFixed & dof) { + [&](const DerivationOutput::CAFixed & dof) { StorePath path = makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName); envHasRightPath(path, i.first); }, - [&](const DerivationOutputCAFloating &) { + [&](const DerivationOutput::CAFloating &) { /* Nothing to check */ }, - [&](const DerivationOutputDeferred &) { + [&](const DerivationOutput::Deferred &) { + /* Nothing to check */ + }, + [&](const DerivationOutput::Impure &) { + /* Nothing to check */ }, - }, i.second.output); + }, i.second.raw()); } } diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh new file mode 100644 index 000000000..ff1b92e17 --- /dev/null +++ b/src/libstore/log-store.hh @@ -0,0 +1,21 @@ +#pragma once + +#include "store-api.hh" + + +namespace nix { + +struct LogStore : public virtual Store +{ + inline static std::string operationName = "Build log storage and retrieval"; + + /* Return the build log of the specified store path, if available, + or null otherwise. */ + virtual std::optional<std::string> getBuildLog(const StorePath & path) = 0; + + virtual void addBuildLog(const StorePath & path, std::string_view log) = 0; + + static LogStore & require(Store & store); +}; + +} diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc new file mode 100644 index 000000000..64d172918 --- /dev/null +++ b/src/libstore/make-content-addressed.cc @@ -0,0 +1,80 @@ +#include "make-content-addressed.hh" +#include "references.hh" + +namespace nix { + +std::map<StorePath, StorePath> makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths) +{ + StorePathSet closure; + srcStore.computeFSClosure(storePaths, closure); + + auto paths = srcStore.topoSortPaths(closure); + + std::reverse(paths.begin(), paths.end()); + + std::map<StorePath, StorePath> remappings; + + for (auto & path : paths) { + auto pathS = srcStore.printStorePath(path); + auto oldInfo = srcStore.queryPathInfo(path); + std::string oldHashPart(path.hashPart()); + + StringSink sink; + srcStore.narFromPath(path, sink); + + StringMap rewrites; + + StorePathSet references; + bool hasSelfReference = false; + for (auto & ref : oldInfo->references) { + if (ref == path) + hasSelfReference = true; + else { + auto i = remappings.find(ref); + auto replacement = i != remappings.end() ? i->second : ref; + // FIXME: warn about unremapped paths? + if (replacement != ref) + rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement)); + references.insert(std::move(replacement)); + } + } + + sink.s = rewriteStrings(sink.s, rewrites); + + HashModuloSink hashModuloSink(htSHA256, oldHashPart); + hashModuloSink(sink.s); + + auto narModuloHash = hashModuloSink.finish().first; + + auto dstPath = dstStore.makeFixedOutputPath( + FileIngestionMethod::Recursive, narModuloHash, path.name(), references, hasSelfReference); + + printInfo("rewriting '%s' to '%s'", pathS, srcStore.printStorePath(dstPath)); + + StringSink sink2; + RewritingSink rsink2(oldHashPart, std::string(dstPath.hashPart()), sink2); + rsink2(sink.s); + rsink2.flush(); + + ValidPathInfo info { dstPath, hashString(htSHA256, sink2.s) }; + info.references = std::move(references); + if (hasSelfReference) info.references.insert(info.path); + info.narSize = sink.s.size(); + info.ca = FixedOutputHash { + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, + }; + + StringSource source(sink2.s); + dstStore.addToStore(info, source); + + remappings.insert_or_assign(std::move(path), std::move(info.path)); + } + + return remappings; +} + +} diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh new file mode 100644 index 000000000..c4a66ed41 --- /dev/null +++ b/src/libstore/make-content-addressed.hh @@ -0,0 +1,12 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +std::map<StorePath, StorePath> makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePathSet & storePaths); + +} diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 6409874ff..2bbd7aa70 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -87,7 +87,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv) { auto out = drv.outputs.find("out"); if (out != drv.outputs.end()) { - if (auto v = std::get_if<DerivationOutputCAFixed>(&out->second.output)) + if (const auto * v = std::get_if<DerivationOutput::CAFixed>(&out->second.raw())) return v->hash; } return std::nullopt; @@ -277,15 +277,15 @@ std::map<DrvOutput, StorePath> drvOutputReferences( { std::set<Realisation> inputRealisations; - for (const auto& [inputDrv, outputNames] : drv.inputDrvs) { + for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { auto outputHashes = staticOutputHashes(store, store.readDerivation(inputDrv)); - for (const auto& outputName : outputNames) { + for (const auto & outputName : outputNames) { auto thisRealisation = store.queryRealisation( DrvOutput{outputHashes.at(outputName), outputName}); if (!thisRealisation) throw Error( - "output '%s' of derivation '%s' isn’t built", outputName, + "output '%s' of derivation '%s' isn't built", outputName, store.printStorePath(inputDrv)); inputRealisations.insert(*thisRealisation); } @@ -295,4 +295,5 @@ std::map<DrvOutput, StorePath> drvOutputReferences( return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); } + } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 8c65053e4..f2288a04e 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -93,7 +93,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const StringSet res; for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) res.insert(i); - if (!derivationHasKnownOutputPaths(drv.type())) + if (!drv.type().hasKnownOutputPaths()) res.insert("ca-derivations"); return res; } diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh index effcf099d..95bec21e8 100644 --- a/src/libstore/parsed-derivations.hh +++ b/src/libstore/parsed-derivations.hh @@ -1,5 +1,6 @@ #pragma once +#include "derivations.hh" #include "store-api.hh" #include <nlohmann/json_fwd.hpp> diff --git a/src/libstore/path.cc b/src/libstore/path.cc index e642abcd5..392db225e 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,5 +1,7 @@ #include "store-api.hh" +#include <sodium.h> + namespace nix { static void checkName(std::string_view path, std::string_view name) @@ -41,6 +43,13 @@ bool StorePath::isDerivation() const StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); +StorePath StorePath::random(std::string_view name) +{ + Hash hash(htSHA1); + randombytes_buf(hash.hash, hash.hashSize); + return StorePath(hash, name); +} + StorePath Store::parseStorePath(std::string_view path) const { auto p = canonPath(std::string(path)); diff --git a/src/libstore/path.hh b/src/libstore/path.hh index e65fee622..77fd0f8dc 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -58,6 +58,8 @@ public: } static StorePath dummy; + + static StorePath random(std::string_view name); }; typedef std::set<StorePath> StorePathSet; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 9f6f50593..8493be6fc 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -5,6 +5,7 @@ #include "store-api.hh" #include "gc-store.hh" +#include "log-store.hh" namespace nix { @@ -30,7 +31,10 @@ struct RemoteStoreConfig : virtual StoreConfig /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore +class RemoteStore : public virtual RemoteStoreConfig, + public virtual Store, + public virtual GcStore, + public virtual LogStore { public: diff --git a/src/libstore/repair-flag.hh b/src/libstore/repair-flag.hh new file mode 100644 index 000000000..a13cda312 --- /dev/null +++ b/src/libstore/repair-flag.hh @@ -0,0 +1,7 @@ +#pragma once + +namespace nix { + +enum RepairFlag : bool { NoRepair = false, Repair = true }; + +} diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index e6ecadd7f..1d82b4ab1 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -215,7 +215,6 @@ void handleSQLiteBusy(const SQLiteBusy & e) if (now > lastWarned + 10) { lastWarned = now; logWarning({ - .name = "Sqlite busy", .msg = hintfmt(e.what()) }); } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index bb03daef4..62daa838c 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -52,6 +52,10 @@ public: bool sameMachine() override { return false; } + // FIXME extend daemon protocol, move implementation to RemoteStore + std::optional<std::string> getBuildLog(const StorePath & path) override + { unsupported("getBuildLog"); } + private: struct Connection : RemoteStore::Connection diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 86fa6a211..59937be4d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,6 +1,7 @@ #include "crypto.hh" #include "fs-accessor.hh" #include "globals.hh" +#include "derivations.hh" #include "store-api.hh" #include "util.hh" #include "nar-info-disk-cache.hh" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e99a3f2cb..0c8a4db56 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -10,8 +10,8 @@ #include "sync.hh" #include "globals.hh" #include "config.hh" -#include "derivations.hh" #include "path-info.hh" +#include "repair-flag.hh" #include <atomic> #include <limits> @@ -62,6 +62,8 @@ MakeError(BadStorePath, Error); MakeError(InvalidStoreURI, Error); +struct BasicDerivation; +struct Derivation; class FSAccessor; class NarInfoDiskCache; class Store; @@ -605,14 +607,6 @@ public: */ StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths); - /* Return the build log of the specified store path, if available, - or null otherwise. */ - virtual std::optional<std::string> getBuildLog(const StorePath & path) - { return std::nullopt; } - - virtual void addBuildLog(const StorePath & path, std::string_view log) - { unsupported("addBuildLog"); } - /* Hack to allow long-running processes like hydra-queue-runner to occasionally flush their path info cache. */ void clearPathInfoCache() diff --git a/src/libstore/store-cast.hh b/src/libstore/store-cast.hh new file mode 100644 index 000000000..ff62fc359 --- /dev/null +++ b/src/libstore/store-cast.hh @@ -0,0 +1,16 @@ +#pragma once + +#include "store-api.hh" + +namespace nix { + +template<typename T> +T & require(Store & store) +{ + auto * castedStore = dynamic_cast<T *>(&store); + if (!castedStore) + throw UsageError("%s not supported by store '%s'", T::operationName, store.getUri()); + return *castedStore; +} + +} diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index eda004756..30b471af5 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -64,11 +64,12 @@ static void dumpContents(const Path & path, off_t size, } -static void dump(const Path & path, Sink & sink, PathFilter & filter) +static time_t dump(const Path & path, Sink & sink, PathFilter & filter) { checkInterrupt(); auto st = lstat(path); + time_t result = st.st_mtime; sink << "("; @@ -103,7 +104,10 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) for (auto & i : unhacked) if (filter(path + "/" + i.first)) { sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + "/" + i.second, sink, filter); + auto tmp_mtime = dump(path + "/" + i.second, sink, filter); + if (tmp_mtime > result) { + result = tmp_mtime; + } sink << ")"; } } @@ -114,13 +118,20 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) else throw Error("file '%1%' has an unsupported type", path); sink << ")"; + + return result; } -void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { sink << narVersionMagic1; - dump(path, sink, filter); + return dump(path, sink, filter); +} + +void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +{ + dumpPathAndGetMtime(path, sink, filter); } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index fca351605..79ce08df0 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -48,6 +48,10 @@ namespace nix { void dumpPath(const Path & path, Sink & sink, PathFilter & filter = defaultPathFilter); +/* Same as `void dumpPath()`, but returns the last modified date of the path */ +time_t dumpPathAndGetMtime(const Path & path, Sink & sink, + PathFilter & filter = defaultPathFilter); + void dumpString(std::string_view s, Sink & sink); /* FIXME: fix this API, it sucks. */ diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 69aa0d094..4b8c55686 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -127,11 +127,11 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) if (flag.handler.arity == ArityAny) break; throw UsageError("flag '%s' requires %d argument(s)", name, flag.handler.arity); } - if (flag.completer) - if (auto prefix = needsCompletion(*pos)) { - anyCompleted = true; + if (auto prefix = needsCompletion(*pos)) { + anyCompleted = true; + if (flag.completer) flag.completer(n, *prefix); - } + } args.push_back(*pos++); } if (!anyCompleted) @@ -146,6 +146,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) && hasPrefix(name, std::string(*prefix, 2))) completions->add("--" + name, flag->description); } + return false; } auto i = longFlags.find(std::string(*pos, 2)); if (i == longFlags.end()) return false; @@ -187,10 +188,12 @@ bool Args::processArgs(const Strings & args, bool finish) { std::vector<std::string> ss; for (const auto &[n, s] : enumerate(args)) { - ss.push_back(s); - if (exp.completer) - if (auto prefix = needsCompletion(s)) + if (auto prefix = needsCompletion(s)) { + ss.push_back(*prefix); + if (exp.completer) exp.completer(n, *prefix); + } else + ss.push_back(s); } exp.handler.fun(ss); expectedArgs.pop_front(); @@ -279,21 +282,22 @@ static void _completePath(std::string_view prefix, bool onlyDirs) { completionType = ctFilenames; glob_t globbuf; - int flags = GLOB_NOESCAPE | GLOB_TILDE; + int flags = GLOB_NOESCAPE; #ifdef GLOB_ONLYDIR if (onlyDirs) flags |= GLOB_ONLYDIR; #endif - if (glob((std::string(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) { + // using expandTilde here instead of GLOB_TILDE(_CHECK) so that ~<Tab> expands to /home/user/ + if (glob((expandTilde(prefix) + "*").c_str(), flags, nullptr, &globbuf) == 0) { for (size_t i = 0; i < globbuf.gl_pathc; ++i) { if (onlyDirs) { - auto st = lstat(globbuf.gl_pathv[i]); + auto st = stat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } completions->add(globbuf.gl_pathv[i]); } - globfree(&globbuf); } + globfree(&globbuf); } void completePath(size_t, std::string_view prefix) @@ -322,11 +326,6 @@ MultiCommand::MultiCommand(const Commands & commands_) .optional = true, .handler = {[=](std::string s) { assert(!command); - if (auto prefix = needsCompletion(s)) { - for (auto & [name, command] : commands) - if (hasPrefix(name, *prefix)) - completions->add(name); - } auto i = commands.find(s); if (i == commands.end()) { std::set<std::string> commandNames; @@ -337,6 +336,11 @@ MultiCommand::MultiCommand(const Commands & commands_) } command = {s, i->second()}; command->second->parent = this; + }}, + .completer = {[&](size_t, std::string_view prefix) { + for (auto & [name, command] : commands) + if (hasPrefix(name, prefix)) + completions->add(name); }} }); diff --git a/src/libutil/error.cc b/src/libutil/error.cc index b2dfb35b2..9172f67a6 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -9,10 +9,9 @@ namespace nix { const std::string nativeSystem = SYSTEM; -BaseError & BaseError::addTrace(std::optional<ErrPos> e, hintformat hint) +void BaseError::addTrace(std::optional<ErrPos> e, hintformat hint) { err.traces.push_front(Trace { .pos = e, .hint = hint }); - return *this; } // c++ std::exception descendants must have a 'const char* what()' function. @@ -22,12 +21,9 @@ const std::string & BaseError::calcWhat() const if (what_.has_value()) return *what_; else { - err.name = sname(); - std::ostringstream oss; showErrorInfo(oss, err, loggerSettings.showTrace); what_ = oss.str(); - return *what_; } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 93b789f0b..348018f57 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -109,7 +109,6 @@ struct Trace { struct ErrorInfo { Verbosity level; - std::string name; // FIXME: rename hintformat msg; std::optional<ErrPos> errPos; std::list<Trace> traces; @@ -162,8 +161,6 @@ public: : err(e) { } - virtual const char* sname() const { return "BaseError"; } - #ifdef EXCEPTION_NEEDS_THROW_SPEC ~BaseError() throw () { }; const char * what() const throw () { return calcWhat().c_str(); } @@ -175,12 +172,12 @@ public: const ErrorInfo & info() const { calcWhat(); return err; } template<typename... Args> - BaseError & addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args) + void addTrace(std::optional<ErrPos> e, const std::string & fs, const Args & ... args) { - return addTrace(e, hintfmt(fs, args...)); + addTrace(e, hintfmt(fs, args...)); } - BaseError & addTrace(std::optional<ErrPos> e, hintformat hint); + void addTrace(std::optional<ErrPos> e, hintformat hint); bool hasTrace() const { return !err.traces.empty(); } }; @@ -190,7 +187,6 @@ public: { \ public: \ using superClass::superClass; \ - virtual const char* sname() const override { return #newClass; } \ } MakeError(Error, BaseError); @@ -210,8 +206,6 @@ public: auto hf = hintfmt(args...); err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); } - - virtual const char* sname() const override { return "SysError"; } }; } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index b49f47e1d..e033a4116 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -7,10 +7,12 @@ namespace nix { std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = { { Xp::CaDerivations, "ca-derivations" }, + { Xp::ImpureDerivations, "impure-derivations" }, { Xp::Flakes, "flakes" }, { Xp::NixCommand, "nix-command" }, { Xp::RecursiveNix, "recursive-nix" }, { Xp::NoUrlLiterals, "no-url-literals" }, + { Xp::FetchClosure, "fetch-closure" }, }; const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 291a58e32..266e41a22 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -16,10 +16,12 @@ namespace nix { enum struct ExperimentalFeature { CaDerivations, + ImpureDerivations, Flakes, NixCommand, RecursiveNix, - NoUrlLiterals + NoUrlLiterals, + FetchClosure, }; /** @@ -47,10 +49,6 @@ public: ExperimentalFeature missingFeature; MissingExperimentalFeature(ExperimentalFeature); - virtual const char * sname() const override - { - return "MissingExperimentalFeature"; - } }; } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index a4d632161..d2fd0c15a 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -155,7 +155,7 @@ static std::pair<std::optional<HashType>, bool> getParsedTypeAndSRI(std::string_ { bool isSRI = false; - // Parse the has type before the separater, if there was one. + // Parse the hash type before the separator, if there was one. std::optional<HashType> optParsedType; { auto hashRaw = splitPrefixTo(rest, ':'); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 56b5938b3..00f70a572 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -93,13 +93,11 @@ public: std::string gitRev() const { - assert(type == htSHA1); return to_string(Base16, false); } std::string gitShortRev() const { - assert(type == htSHA1); return std::string(to_string(Base16, false), 0, 7); } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 6445b3f1b..8ff904583 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -357,7 +357,7 @@ Sink & operator << (Sink & sink, const Error & ex) sink << "Error" << info.level - << info.name + << "Error" // removed << info.msg.str() << 0 // FIXME: info.errPos << info.traces.size(); @@ -426,11 +426,10 @@ Error readError(Source & source) auto type = readString(source); assert(type == "Error"); auto level = (Verbosity) readInt(source); - auto name = readString(source); + auto name = readString(source); // removed auto msg = readString(source); ErrorInfo info { .level = level, - .name = name, .msg = hintformat(std::move(format("%s") % msg)), }; auto havePos = readNum<size_t>(source); diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 790bc943a..a7db58559 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -39,28 +39,30 @@ void TarArchive::check(int err, const std::string & reason) throw Error(reason, archive_error_string(this->archive)); } -TarArchive::TarArchive(Source & source, bool raw) - : source(&source), buffer(4096) +TarArchive::TarArchive(Source & source, bool raw) : buffer(4096) { - init(); - if (!raw) + this->archive = archive_read_new(); + this->source = &source; + + if (!raw) { + archive_read_support_filter_all(archive); archive_read_support_format_all(archive); - else + } else { + archive_read_support_filter_all(archive); archive_read_support_format_raw(archive); + archive_read_support_format_empty(archive); + } check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)"); } + TarArchive::TarArchive(const Path & path) { - init(); - archive_read_support_format_all(archive); - check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); -} + this->archive = archive_read_new(); -void TarArchive::init() -{ - archive = archive_read_new(); archive_read_support_filter_all(archive); + archive_read_support_format_all(archive); + check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); } void TarArchive::close() diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index f107a7e2e..4d9141fd4 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -17,13 +17,10 @@ struct TarArchive { // disable copy constructor TarArchive(const TarArchive &) = delete; - void init(); - void close(); ~TarArchive(); }; - void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index f20e2dc41..c3b233797 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -178,7 +178,7 @@ namespace nix { } TEST(parseURL, parseFileURLWithQueryAndFragment) { - auto s = "file:///none/of/your/business"; + auto s = "file:///none/of//your/business"; auto parsed = parseURL(s); ParsedURL expected { @@ -186,7 +186,7 @@ namespace nix { .base = "", .scheme = "file", .authority = "", - .path = "/none/of/your/business", + .path = "/none/of//your/business", .query = (StringMap) { }, .fragment = "", }; diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index da10a6bbc..d5e6a2736 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -18,7 +18,7 @@ const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncod const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; -const static std::string segmentRegex = "(?:" + pcharRegex + "+)"; +const static std::string segmentRegex = "(?:" + pcharRegex + "*)"; const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 9f13d5f02..656804007 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -71,13 +71,11 @@ void clearEnv() unsetenv(name.first.c_str()); } -void replaceEnv(std::map<std::string, std::string> newEnv) +void replaceEnv(const std::map<std::string, std::string> & newEnv) { clearEnv(); - for (auto newEnvVar : newEnv) - { + for (auto & newEnvVar : newEnv) setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); - } } @@ -200,6 +198,17 @@ std::string_view baseNameOf(std::string_view path) } +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + + bool isInDir(std::string_view path, std::string_view dir) { return path.substr(0, 1) == "/" @@ -215,6 +224,15 @@ bool isDirOrInDir(std::string_view path, std::string_view dir) } +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + struct stat lstat(const Path & path) { struct stat st; @@ -406,8 +424,29 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) throw SysError("getting status of '%1%'", path); } - if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) - bytesFreed += st.st_size; + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } if (S_ISDIR(st.st_mode)) { /* Make the directory accessible. */ @@ -1240,9 +1279,9 @@ template<class C> C tokenizeString(std::string_view s, std::string_view separato { C result; auto pos = s.find_first_not_of(separators, 0); - while (pos != std::string::npos) { + while (pos != std::string_view::npos) { auto end = s.find_first_of(separators, pos + 1); - if (end == std::string::npos) end = s.size(); + if (end == std::string_view::npos) end = s.size(); result.insert(result.end(), std::string(s, pos, end - pos)); pos = s.find_first_not_of(separators, end); } @@ -1452,6 +1491,7 @@ constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuv std::string base64Encode(std::string_view s) { std::string res; + res.reserve((s.size() + 2) / 3 * 4); int data = 0, nbits = 0; for (char c : s) { @@ -1483,6 +1523,9 @@ std::string base64Decode(std::string_view s) }(); std::string res; + // Some sequences are missing the padding consisting of up to two '='. + // vvv + res.reserve((s.size() + 2) / 4 * 3); unsigned int d = 0, bits = 0; for (char c : s) { @@ -1669,7 +1712,9 @@ void setStackSize(size_t stackSize) #endif } +#if __linux__ static AutoCloseFD fdSavedMountNamespace; +#endif void saveMountNamespace() { @@ -1688,8 +1733,13 @@ void restoreMountNamespace() { #if __linux__ try { + auto savedCwd = absPath("."); + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); + if (chdir(savedCwd.c_str()) == -1) { + throw SysError("restoring cwd"); + } } catch (Error & e) { debug(e.msg()); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 20591952d..a1d0e0e6b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -68,6 +68,9 @@ Path dirOf(const PathView path); following the final `/' (trailing slashes are removed). */ std::string_view baseNameOf(std::string_view path); +/* Perform tilde expansion on a path. */ +std::string expandTilde(std::string_view path); + /* Check whether 'path' is a descendant of 'dir'. Both paths must be canonicalized. */ bool isInDir(std::string_view path, std::string_view dir); @@ -77,6 +80,7 @@ bool isInDir(std::string_view path, std::string_view dir); bool isDirOrInDir(std::string_view path, std::string_view dir); /* Get status of `path'. */ +struct stat stat(const Path & path); struct stat lstat(const Path & path); /* Return true iff the given path exists. */ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 795a4f7bd..faa8c078f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -325,8 +325,7 @@ static void main_nix_build(int argc, char * * argv) state->printStats(); - auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths0) { - auto paths = toDerivedPaths(paths0); + auto buildPaths = [&](const std::vector<DerivedPath> & paths) { /* Note: we do this even when !printMissing to efficiently fetch binary cache data. */ uint64_t downloadSize, narSize; @@ -348,7 +347,7 @@ static void main_nix_build(int argc, char * * argv) auto & drvInfo = drvs.front(); auto drv = evalStore->derivationFromPath(drvInfo.requireDrvPath()); - std::vector<StorePathWithOutputs> pathsToBuild; + std::vector<DerivedPath> pathsToBuild; RealisedPath::Set pathsToCopy; /* Figure out what bash shell to use. If $NIX_BUILD_SHELL @@ -370,7 +369,10 @@ static void main_nix_build(int argc, char * * argv) throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation"); auto bashDrv = drv->requireDrvPath(); - pathsToBuild.push_back({bashDrv}); + pathsToBuild.push_back(DerivedPath::Built { + .drvPath = bashDrv, + .outputs = {}, + }); pathsToCopy.insert(bashDrv); shellDrv = bashDrv; @@ -382,17 +384,24 @@ static void main_nix_build(int argc, char * * argv) } // Build or fetch all dependencies of the derivation. - for (const auto & input : drv.inputDrvs) + for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) { + // To get around lambda capturing restrictions in the + // standard. + const auto & inputDrv = inputDrv0; if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const std::string & exclude) { - return !std::regex_search(store->printStorePath(input.first), std::regex(exclude)); + return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude)); })) { - pathsToBuild.push_back({input.first, input.second}); - pathsToCopy.insert(input.first); + pathsToBuild.push_back(DerivedPath::Built { + .drvPath = inputDrv, + .outputs = inputOutputs + }); + pathsToCopy.insert(inputDrv); } + } for (const auto & src : drv.inputSrcs) { - pathsToBuild.push_back({src}); + pathsToBuild.push_back(DerivedPath::Opaque{src}); pathsToCopy.insert(src); } @@ -543,7 +552,7 @@ static void main_nix_build(int argc, char * * argv) else { - std::vector<StorePathWithOutputs> pathsToBuild; + std::vector<DerivedPath> pathsToBuild; std::vector<std::pair<StorePath, std::string>> pathsToBuildOrdered; RealisedPath::Set drvsToCopy; @@ -556,7 +565,7 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back({drvPath, {outputName}}); + pathsToBuild.push_back(DerivedPath::Built{drvPath, {outputName}}); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 4b28ea6a4..af6f1c88c 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,4 +1,5 @@ #include "store-api.hh" +#include "store-cast.hh" #include "gc-store.hh" #include "profiles.hh" #include "shared.hh" @@ -81,7 +82,7 @@ static int main_nix_collect_garbage(int argc, char * * argv) // Run the actual garbage collector. if (!dryRun) { auto store = openStore(); - auto & gcStore = requireGcStore(*store); + auto & gcStore = require<GcStore>(*store); options.action = GCOptions::gcDeleteDead; GCResults results; PrintFreed freed(true, results); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 40c3c5d65..9a68899cd 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -128,7 +128,12 @@ static void getAllExprs(EvalState & state, if (hasSuffix(attrName, ".nix")) attrName = std::string(attrName, 0, attrName.size() - 4); if (!seen.insert(attrName).second) { - printError("warning: name collision in input Nix expressions, skipping '%1%'", path2); + std::string suggestionMessage = ""; + if (path2.find("channels") != std::string::npos && path.find("channels") != std::string::npos) { + suggestionMessage = fmt("\nsuggestion: remove '%s' from either the root channels or the user channels", attrName); + } + printError("warning: name collision in input Nix expressions, skipping '%1%'" + "%2%", path2, suggestionMessage); continue; } /* Load the expression on demand. */ @@ -918,12 +923,17 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin pkgObj.attr("pname", drvName.name); pkgObj.attr("version", drvName.version); pkgObj.attr("system", i.querySystem()); + pkgObj.attr("outputName", i.queryOutputName()); - if (printOutPath) { - DrvInfo::Outputs outputs = i.queryOutputs(); + { + DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); JSONObject outputObj = pkgObj.object("outputs"); - for (auto & j : outputs) - outputObj.attr(j.first, globals.state->store->printStorePath(j.second)); + for (auto & j : outputs) { + if (j.second) + outputObj.attr(j.first, globals.state->store->printStorePath(*j.second)); + else + outputObj.attr(j.first, nullptr); + } } if (printMeta) { @@ -1052,6 +1062,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Print the desired columns, or XML output. */ if (jsonOutput) { queryJSON(globals, elems, printOutPath, printMeta); + cout << '\n'; return; } @@ -1154,13 +1165,16 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) columns.push_back(drvPath ? store.printStorePath(*drvPath) : "-"); } + if (xmlOutput) + attrs["outputName"] = i.queryOutputName(); + if (printOutPath && !xmlOutput) { DrvInfo::Outputs outputs = i.queryOutputs(); std::string s; for (auto & j : outputs) { if (!s.empty()) s += ';'; if (j.first != "out") { s += j.first; s += "="; } - s += store.printStorePath(j.second); + s += store.printStorePath(*j.second); } columns.push_back(s); } @@ -1174,71 +1188,67 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } if (xmlOutput) { - if (printOutPath || printMeta) { - XMLOpenElement item(xml, "item", attrs); - if (printOutPath) { - DrvInfo::Outputs outputs = i.queryOutputs(); - for (auto & j : outputs) { - XMLAttrs attrs2; - attrs2["name"] = j.first; - attrs2["path"] = store.printStorePath(j.second); - xml.writeEmptyElement("output", attrs2); - } - } - if (printMeta) { - StringSet metaNames = i.queryMetaNames(); - for (auto & j : metaNames) { - XMLAttrs attrs2; - attrs2["name"] = j; - Value * v = i.queryMeta(j); - if (!v) - printError( - "derivation '%s' has invalid meta attribute '%s'", - i.queryName(), j); - else { - if (v->type() == nString) { - attrs2["type"] = "string"; - attrs2["value"] = v->string.s; - xml.writeEmptyElement("meta", attrs2); - } else if (v->type() == nInt) { - attrs2["type"] = "int"; - attrs2["value"] = (format("%1%") % v->integer).str(); - xml.writeEmptyElement("meta", attrs2); - } else if (v->type() == nFloat) { - attrs2["type"] = "float"; - attrs2["value"] = (format("%1%") % v->fpoint).str(); - xml.writeEmptyElement("meta", attrs2); - } else if (v->type() == nBool) { - attrs2["type"] = "bool"; - attrs2["value"] = v->boolean ? "true" : "false"; - xml.writeEmptyElement("meta", attrs2); - } else if (v->type() == nList) { - attrs2["type"] = "strings"; - XMLOpenElement m(xml, "meta", attrs2); - for (auto elem : v->listItems()) { - if (elem->type() != nString) continue; - XMLAttrs attrs3; - attrs3["value"] = elem->string.s; - xml.writeEmptyElement("string", attrs3); - } - } else if (v->type() == nAttrs) { - attrs2["type"] = "strings"; - XMLOpenElement m(xml, "meta", attrs2); - Bindings & attrs = *v->attrs; - for (auto &i : attrs) { - Attr & a(*attrs.find(i.name)); - if(a.value->type() != nString) continue; - XMLAttrs attrs3; - attrs3["type"] = i.name; - attrs3["value"] = a.value->string.s; - xml.writeEmptyElement("string", attrs3); + XMLOpenElement item(xml, "item", attrs); + DrvInfo::Outputs outputs = i.queryOutputs(printOutPath); + for (auto & j : outputs) { + XMLAttrs attrs2; + attrs2["name"] = j.first; + if (j.second) + attrs2["path"] = store.printStorePath(*j.second); + xml.writeEmptyElement("output", attrs2); + } + if (printMeta) { + StringSet metaNames = i.queryMetaNames(); + for (auto & j : metaNames) { + XMLAttrs attrs2; + attrs2["name"] = j; + Value * v = i.queryMeta(j); + if (!v) + printError( + "derivation '%s' has invalid meta attribute '%s'", + i.queryName(), j); + else { + if (v->type() == nString) { + attrs2["type"] = "string"; + attrs2["value"] = v->string.s; + xml.writeEmptyElement("meta", attrs2); + } else if (v->type() == nInt) { + attrs2["type"] = "int"; + attrs2["value"] = (format("%1%") % v->integer).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type() == nFloat) { + attrs2["type"] = "float"; + attrs2["value"] = (format("%1%") % v->fpoint).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type() == nBool) { + attrs2["type"] = "bool"; + attrs2["value"] = v->boolean ? "true" : "false"; + xml.writeEmptyElement("meta", attrs2); + } else if (v->type() == nList) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + for (auto elem : v->listItems()) { + if (elem->type() != nString) continue; + XMLAttrs attrs3; + attrs3["value"] = elem->string.s; + xml.writeEmptyElement("string", attrs3); } - } + } else if (v->type() == nAttrs) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + Bindings & attrs = *v->attrs; + for (auto &i : attrs) { + Attr & a(*attrs.find(i.name)); + if(a.value->type() != nString) continue; + XMLAttrs attrs3; + attrs3["type"] = i.name; + attrs3["value"] = a.value->string.s; + xml.writeEmptyElement("string", attrs3); + } } } } - } else - xml.writeEmptyElement("item", attrs); + } } else table.push_back(columns); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index af4f350ff..78692b9c6 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -56,7 +56,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, output paths, and optionally the derivation path, as well as the meta attributes. */ std::optional<StorePath> drvPath = keepDerivations ? i.queryDrvPath() : std::nullopt; - DrvInfo::Outputs outputs = i.queryOutputs(true); + DrvInfo::Outputs outputs = i.queryOutputs(true, true); StringSet metaNames = i.queryMetaNames(); auto attrs = state.buildBindings(7 + outputs.size()); @@ -76,15 +76,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, for (const auto & [m, j] : enumerate(outputs)) { (vOutputs.listElems()[m] = state.allocValue())->mkString(j.first); auto outputAttrs = state.buildBindings(2); - outputAttrs.alloc(state.sOutPath).mkString(state.store->printStorePath(j.second)); + outputAttrs.alloc(state.sOutPath).mkString(state.store->printStorePath(*j.second)); attrs.alloc(j.first).mkAttrs(outputAttrs); /* This is only necessary when installing store paths, e.g., `nix-env -i /nix/store/abcd...-foo'. */ - state.store->addTempRoot(j.second); - state.store->ensurePath(j.second); + state.store->addTempRoot(*j.second); + state.store->ensurePath(*j.second); - references.insert(j.second); + references.insert(*j.second); } // Copy the meta attributes. @@ -105,8 +105,10 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, /* Also write a copy of the list of user environment elements to the store; we need it for future modifications of the environment. */ + std::ostringstream str; + manifest.print(str, true); auto manifestFile = state.store->addTextToStore("env-manifest.nix", - fmt("%s", manifest), references); + str.str(), references); /* Get the environment builder expression. */ Value envBuilder; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 8ebaf9387..153b84137 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -3,7 +3,9 @@ #include "dotgraph.hh" #include "globals.hh" #include "build-result.hh" +#include "store-cast.hh" #include "gc-store.hh" +#include "log-store.hh" #include "local-store.hh" #include "monitor-fd.hh" #include "serve-protocol.hh" @@ -429,7 +431,7 @@ static void opQuery(Strings opFlags, Strings opArgs) store->computeFSClosure( args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); - auto & gcStore = requireGcStore(*store); + auto & gcStore = require<GcStore>(*store); Roots roots = gcStore.findRoots(false); for (auto & [target, links] : roots) if (referrers.find(target) != referrers.end()) @@ -474,13 +476,15 @@ static void opReadLog(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); + auto & logStore = require<LogStore>(*store); + RunPager pager; for (auto & i : opArgs) { - auto path = store->followLinksToStorePath(i); - auto log = store->getBuildLog(path); + auto path = logStore.followLinksToStorePath(i); + auto log = logStore.getBuildLog(path); if (!log) - throw Error("build log of derivation '%s' is not available", store->printStorePath(path)); + throw Error("build log of derivation '%s' is not available", logStore.printStorePath(path)); std::cout << *log; } } @@ -590,7 +594,7 @@ static void opGC(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); - auto & gcStore = requireGcStore(*store); + auto & gcStore = require<GcStore>(*store); if (printRoots) { Roots roots = gcStore.findRoots(false); @@ -629,7 +633,7 @@ static void opDelete(Strings opFlags, Strings opArgs) for (auto & i : opArgs) options.pathsToDelete.insert(store->followLinksToStorePath(i)); - auto & gcStore = requireGcStore(*store); + auto & gcStore = require<GcStore>(*store); GCResults results; PrintFreed freed(true, results); diff --git a/src/nix/app.cc b/src/nix/app.cc index 2563180fb..6b6b31a12 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -4,6 +4,7 @@ #include "eval-cache.hh" #include "names.hh" #include "command.hh" +#include "derivations.hh" namespace nix { @@ -60,17 +61,21 @@ std::string resolveString(Store & store, const std::string & toResolve, const Bu UnresolvedApp Installable::toApp(EvalState & state) { - auto [cursor, attrPath] = getCursor(state); + auto cursor = getCursor(state); + auto attrPath = cursor->getAttrPath(); auto type = cursor->getAttr("type")->getString(); + std::string expected = !attrPath.empty() && attrPath[0] == "apps" ? "app" : "derivation"; + if (type != expected) + throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected); + if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); - std::vector<StorePathWithOutputs> context2; for (auto & [path, name] : context) - context2.push_back({state.store->parseStorePath(path), {name}}); + context2.push_back({path, {name}}); return UnresolvedApp{App { .context = std::move(context2), @@ -100,7 +105,7 @@ UnresolvedApp Installable::toApp(EvalState & state) } else - throw Error("attribute '%s' has unsupported type '%s'", attrPath, type); + throw Error("attribute '%s' has unsupported type '%s'", cursor->getAttrPathStr(), type); } // FIXME: move to libcmd diff --git a/src/nix/build.cc b/src/nix/build.cc index 680db1c60..840c7ca38 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -52,15 +52,26 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile void run(ref<Store> store) override { + if (dryRun) { + std::vector<DerivedPath> pathsToBuild; + + for (auto & i : installables) { + auto b = i->toDerivedPaths(); + pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end()); + } + printMissing(store, pathsToBuild, lvlError); + if (json) + logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + return; + } + auto buildables = Installable::build( getEvalStore(), store, - dryRun ? Realise::Derivation : Realise::Outputs, + Realise::Outputs, installables, buildMode); if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump()); - if (dryRun) return; - if (outLink != "") if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) for (const auto & [_i, buildable] : enumerate(buildables)) { diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 7ed558dee..81fb8464a 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -9,7 +9,7 @@ using namespace nix; struct CmdBundle : InstallableCommand { - std::string bundler = "github:matthewbauer/nix-bundle"; + std::string bundler = "github:NixOS/bundlers"; std::optional<Path> outLink; CmdBundle() diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 8af5da9d0..7fc74d34e 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -196,21 +196,22 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore drv.inputSrcs.insert(std::move(getEnvShPath)); if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { for (auto & output : drv.outputs) { - output.second = { - .output = DerivationOutputDeferred{}, - }; + output.second = DerivationOutput::Deferred {}, drv.env[output.first] = hashPlaceholder(output.first); } } else { for (auto & output : drv.outputs) { - output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; + output.second = DerivationOutput::Deferred { }; drv.env[output.first] = ""; } - Hash h = std::get<0>(hashDerivationModulo(*evalStore, drv, true)); + auto hashesModulo = hashDerivationModulo(*evalStore, drv, true); for (auto & output : drv.outputs) { + Hash h = hashesModulo.hashes.at(output.first); auto outPath = store->makeOutputPath(output.first, h, drv.name); - output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; + output.second = DerivationOutput::InputAddressed { + .path = outPath, + }; drv.env[output.first] = store->printStorePath(outPath); } } diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 4f3003448..ea87e3d87 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -24,12 +24,12 @@ std::string formatProtocol(unsigned int proto) } bool checkPass(const std::string & msg) { - logger->log(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); + notice(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); return true; } bool checkFail(const std::string & msg) { - logger->log(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); + notice(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); return false; } diff --git a/src/nix/edit.md b/src/nix/edit.md index 80563d06b..89bd09abf 100644 --- a/src/nix/edit.md +++ b/src/nix/edit.md @@ -24,8 +24,8 @@ this attribute to the location of the definition of the `meta.description`, `version` or `name` derivation attributes. The editor to invoke is specified by the `EDITOR` environment -variable. It defaults to `cat`. If the editor is `emacs`, `nano` or -`vim`, it is passed the line number of the derivation using the -argument `+<lineno>`. +variable. It defaults to `cat`. If the editor is `emacs`, `nano`, +`vim` or `kak`, it is passed the line number of the derivation using +the argument `+<lineno>`. )"" diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 8cd04d5fe..733b93661 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -16,7 +16,7 @@ struct CmdEval : MixJSON, InstallableCommand std::optional<std::string> apply; std::optional<Path> writeTo; - CmdEval() + CmdEval() : InstallableCommand(true /* supportReadOnlyMode */) { addFlag({ .longName = "raw", diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 9830ce841..dbd157248 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -463,7 +463,7 @@ struct CmdFlakeCheck : FlakeCommand for (auto & attr : *v.attrs) { std::string name(attr.name); - if (name != "path" && name != "description") + if (name != "path" && name != "description" && name != "welcomeText") throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); } } catch (Error & e) { @@ -508,6 +508,7 @@ struct CmdFlakeCheck : FlakeCommand name == "defaultBundler" ? "bundlers.<system>.default" : name == "overlay" ? "overlays.default" : name == "devShell" ? "devShells.<system>.default" : + name == "nixosModule" ? "nixosModules.default" : ""; if (replacement != "") warn("flake output attribute '%s' is deprecated; use '%s' instead", name, replacement); @@ -714,7 +715,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand defaultTemplateAttrPathsPrefixes, lockFlags); - auto [cursor, attrPath] = installable.getCursor(*evalState); + auto cursor = installable.getCursor(*evalState); auto templateDirAttr = cursor->getAttr("path"); auto templateDir = templateDirAttr->getString(); diff --git a/src/nix/flake.md b/src/nix/flake.md index d59915eeb..7d179a6c4 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -177,8 +177,8 @@ Currently the `type` attribute can be one of the following: attribute `url`. In URL form, the schema must be `http://`, `https://` or `file://` - URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz`, - `.tar.bz2` or `.tar.zst`. + URLs and the extension must be `.zip`, `.tar`, `.tgz`, `.tar.gz`, + `.tar.xz`, `.tar.bz2` or `.tar.zst`. * `github`: A more efficient way to fetch repositories from GitHub. The following attributes are required: diff --git a/src/nix/log.cc b/src/nix/log.cc index fd3c1d787..72d02ef11 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "log-store.hh" #include "progress-bar.hh" using namespace nix; @@ -34,17 +35,24 @@ struct CmdLog : InstallableCommand RunPager pager; for (auto & sub : subs) { + auto * logSubP = dynamic_cast<LogStore *>(&*sub); + if (!logSubP) { + printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri()); + continue; + } + auto & logSub = *logSubP; + auto log = std::visit(overloaded { [&](const DerivedPath::Opaque & bo) { - return sub->getBuildLog(bo.path); + return logSub.getBuildLog(bo.path); }, [&](const DerivedPath::Built & bfd) { - return sub->getBuildLog(bfd.drvPath); + return logSub.getBuildLog(bfd.drvPath); }, }, b.raw()); if (!log) continue; stopProgressBar(); - printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); + printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); std::cout << *log; return; } diff --git a/src/nix/main.cc b/src/nix/main.cc index b923f2535..6198681e7 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -117,7 +117,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"hash-path", {"hash", "path"}}, {"ls-nar", {"nar", "ls"}}, {"ls-store", {"store", "ls"}}, - {"make-content-addressable", {"store", "make-content-addressable"}}, + {"make-content-addressable", {"store", "make-content-addressed"}}, {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, @@ -289,6 +289,7 @@ void mainWrapped(int argc, char * * argv) } if (argc == 2 && std::string(argv[1]) == "__dump-builtins") { + settings.experimentalFeatures = {Xp::Flakes, Xp::FetchClosure}; evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); auto res = nlohmann::json::object(); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc deleted file mode 100644 index 2e75a3b61..000000000 --- a/src/nix/make-content-addressable.cc +++ /dev/null @@ -1,102 +0,0 @@ -#include "command.hh" -#include "store-api.hh" -#include "references.hh" -#include "common-args.hh" -#include "json.hh" - -using namespace nix; - -struct CmdMakeContentAddressable : StorePathsCommand, MixJSON -{ - CmdMakeContentAddressable() - { - realiseMode = Realise::Outputs; - } - - std::string description() override - { - return "rewrite a path or closure to content-addressed form"; - } - - std::string doc() override - { - return - #include "make-content-addressable.md" - ; - } - - void run(ref<Store> store, StorePaths && storePaths) override - { - auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end())); - - std::reverse(paths.begin(), paths.end()); - - std::map<StorePath, StorePath> remappings; - - auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr; - auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr; - - for (auto & path : paths) { - auto pathS = store->printStorePath(path); - auto oldInfo = store->queryPathInfo(path); - std::string oldHashPart(path.hashPart()); - - StringSink sink; - store->narFromPath(path, sink); - - StringMap rewrites; - - StorePathSet references; - bool hasSelfReference = false; - for (auto & ref : oldInfo->references) { - if (ref == path) - hasSelfReference = true; - else { - auto i = remappings.find(ref); - auto replacement = i != remappings.end() ? i->second : ref; - // FIXME: warn about unremapped paths? - if (replacement != ref) - rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement)); - references.insert(std::move(replacement)); - } - } - - sink.s = rewriteStrings(sink.s, rewrites); - - HashModuloSink hashModuloSink(htSHA256, oldHashPart); - hashModuloSink(sink.s); - - auto narHash = hashModuloSink.finish().first; - - ValidPathInfo info { - store->makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, path.name(), references, hasSelfReference), - narHash, - }; - info.references = std::move(references); - if (hasSelfReference) info.references.insert(info.path); - info.narSize = sink.s.size(); - info.ca = FixedOutputHash { - .method = FileIngestionMethod::Recursive, - .hash = info.narHash, - }; - - if (!json) - notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path)); - - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink); - rsink2(sink.s); - rsink2.flush(); - }); - - store->addToStore(info, *source); - - if (json) - jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path)); - - remappings.insert_or_assign(std::move(path), std::move(info.path)); - } - } -}; - -static auto rCmdMakeContentAddressable = registerCommand2<CmdMakeContentAddressable>({"store", "make-content-addressable"}); diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc new file mode 100644 index 000000000..34860c38f --- /dev/null +++ b/src/nix/make-content-addressed.cc @@ -0,0 +1,55 @@ +#include "command.hh" +#include "store-api.hh" +#include "make-content-addressed.hh" +#include "common-args.hh" +#include "json.hh" + +using namespace nix; + +struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON +{ + CmdMakeContentAddressed() + { + realiseMode = Realise::Outputs; + } + + std::string description() override + { + return "rewrite a path or closure to content-addressed form"; + } + + std::string doc() override + { + return + #include "make-content-addressed.md" + ; + } + + void run(ref<Store> srcStore, StorePaths && storePaths) override + { + auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + + auto remappings = makeContentAddressed(*srcStore, *dstStore, + StorePathSet(storePaths.begin(), storePaths.end())); + + if (json) { + JSONObject jsonRoot(std::cout); + JSONObject jsonRewrites(jsonRoot.object("rewrites")); + for (auto & path : storePaths) { + auto i = remappings.find(path); + assert(i != remappings.end()); + jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second)); + } + } else { + for (auto & path : storePaths) { + auto i = remappings.find(path); + assert(i != remappings.end()); + notice("rewrote '%s' to '%s'", + srcStore->printStorePath(path), + srcStore->printStorePath(i->second)); + } + } + } +}; + +static auto rCmdMakeContentAddressed = registerCommand2<CmdMakeContentAddressed>({"store", "make-content-addressed"}); diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressed.md index 3dd847edc..215683e6d 100644 --- a/src/nix/make-content-addressable.md +++ b/src/nix/make-content-addressed.md @@ -5,7 +5,7 @@ R""( * Create a content-addressed representation of the closure of GNU Hello: ```console - # nix store make-content-addressable -r nixpkgs#hello + # nix store make-content-addressed nixpkgs#hello … rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10' ``` @@ -29,7 +29,7 @@ R""( system closure: ```console - # nix store make-content-addressable -r /run/current-system + # nix store make-content-addressed /run/current-system ``` # Description diff --git a/src/nix/profile.cc b/src/nix/profile.cc index a8ff9c78a..b151e48d6 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -62,22 +62,21 @@ struct ProfileElement return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); } - void updateStorePaths(ref<Store> evalStore, ref<Store> store, Installable & installable) + void updateStorePaths( + ref<Store> evalStore, + ref<Store> store, + const BuiltPaths & builtPaths) { // FIXME: respect meta.outputsToInstall storePaths.clear(); - for (auto & buildable : getBuiltPaths(evalStore, store, installable.toDerivedPaths())) { + for (auto & buildable : builtPaths) { std::visit(overloaded { [&](const BuiltPath::Opaque & bo) { storePaths.insert(bo.path); }, [&](const BuiltPath::Built & bfd) { - // TODO: Why are we querying if we know the output - // names already? Is it just to figure out what the - // default one is? - for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { + for (auto & output : bfd.outputs) storePaths.insert(output.second); - } }, }, buildable.raw()); } @@ -98,18 +97,30 @@ struct ProfileManifest auto json = nlohmann::json::parse(readFile(manifestPath)); auto version = json.value("version", 0); - if (version != 1) - throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + std::string sUrl; + std::string sOriginalUrl; + switch(version){ + case 1: + sUrl = "uri"; + sOriginalUrl = "originalUri"; + break; + case 2: + sUrl = "url"; + sOriginalUrl = "originalUrl"; + break; + default: + throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + } for (auto & e : json["elements"]) { ProfileElement element; for (auto & p : e["storePaths"]) element.storePaths.insert(state.store->parseStorePath((std::string) p)); element.active = e["active"]; - if (e.value("uri", "") != "") { + if (e.value(sUrl,"") != "") { element.source = ProfileElementSource{ - parseFlakeRef(e["originalUri"]), - parseFlakeRef(e["uri"]), + parseFlakeRef(e[sOriginalUrl]), + parseFlakeRef(e[sUrl]), e["attrPath"] }; } @@ -143,14 +154,14 @@ struct ProfileManifest obj["storePaths"] = paths; obj["active"] = element.active; if (element.source) { - obj["originalUri"] = element.source->originalRef.to_string(); - obj["uri"] = element.source->resolvedRef.to_string(); + obj["originalUrl"] = element.source->originalRef.to_string(); + obj["url"] = element.source->resolvedRef.to_string(); obj["attrPath"] = element.source->attrPath; } array.push_back(obj); } nlohmann::json json; - json["version"] = 1; + json["version"] = 2; json["elements"] = array; return json.dump(); } @@ -235,6 +246,16 @@ struct ProfileManifest } }; +static std::map<Installable *, BuiltPaths> +builtPathsPerInstallable( + const std::vector<std::pair<std::shared_ptr<Installable>, BuiltPath>> & builtPaths) +{ + std::map<Installable *, BuiltPaths> res; + for (auto & [installable, builtPath] : builtPaths) + res[installable.get()].push_back(builtPath); + return res; +} + struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { std::string description() override @@ -253,7 +274,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile { ProfileManifest manifest(*getEvalState(), *profile); - auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal); + auto builtPaths = builtPathsPerInstallable( + Installable::build2( + getEvalStore(), store, Realise::Outputs, installables, bmNormal)); for (auto & installable : installables) { ProfileElement element; @@ -268,7 +291,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile }; } - element.updateStorePaths(getEvalStore(), store, *installable); + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); manifest.elements.push_back(std::move(element)); } @@ -456,12 +479,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf warn ("Use 'nix profile list' to see the current profile."); } - auto builtPaths = Installable::build(getEvalStore(), store, Realise::Outputs, installables, bmNormal); + auto builtPaths = builtPathsPerInstallable( + Installable::build2( + getEvalStore(), store, Realise::Outputs, installables, bmNormal)); for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); auto & element = manifest.elements[indices.at(i)]; - element.updateStorePaths(getEvalStore(), store, *installable); + element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]); } updateProfile(manifest.build(store)); diff --git a/src/nix/profile.md b/src/nix/profile.md index 0a4ff2fa9..8dade051d 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -70,7 +70,7 @@ are installed in this version of the profile. It looks like this: { "active": true, "attrPath": "legacyPackages.x86_64-linux.zoom-us", - "originalUri": "flake:nixpkgs", + "originalUrl": "flake:nixpkgs", "storePaths": [ "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927" ], @@ -84,11 +84,11 @@ are installed in this version of the profile. It looks like this: Each object in the array `elements` denotes an installed package and has the following fields: -* `originalUri`: The [flake reference](./nix3-flake.md) specified by +* `originalUrl`: The [flake reference](./nix3-flake.md) specified by the user at the time of installation (e.g. `nixpkgs`). This is also the flake reference that will be used by `nix profile upgrade`. -* `uri`: The immutable flake reference to which `originalUri` +* `uri`: The immutable flake reference to which `originalUrl` resolved. * `attrPath`: The flake output attribute that provided this diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 3a51a13e6..1f9d4fb4e 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -25,6 +25,7 @@ extern "C" { #include "eval-inline.hh" #include "attr-path.hh" #include "store-api.hh" +#include "log-store.hh" #include "common-eval-args.hh" #include "get-drvs.hh" #include "derivations.hh" @@ -395,6 +396,7 @@ StorePath NixRepl::getDerivationPath(Value & v) { bool NixRepl::processLine(std::string line) { + line = trim(line); if (line == "") return true; _isInterrupted = false; @@ -526,9 +528,16 @@ bool NixRepl::processLine(std::string line) bool foundLog = false; RunPager pager; for (auto & sub : subs) { - auto log = sub->getBuildLog(drvPath); + auto * logSubP = dynamic_cast<LogStore *>(&*sub); + if (!logSubP) { + printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri()); + continue; + } + auto & logSub = *logSubP; + + auto log = logSub.getBuildLog(drvPath); if (log) { - printInfo("got build log for '%s' from '%s'", drvPathRaw, sub->getUri()); + printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.getUri()); logger->writeToStdout(*log); foundLog = true; break; diff --git a/src/nix/run.cc b/src/nix/run.cc index 033263c36..25a8fa8d3 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -38,9 +38,12 @@ void runProgramInStore(ref<Store> store, unshare(CLONE_NEWUSER) doesn't work in a multithreaded program (which "nix" is), so we exec() a single-threaded helper program (chrootHelper() below) to do the work. */ - auto store2 = store.dynamic_pointer_cast<LocalStore>(); + auto store2 = store.dynamic_pointer_cast<LocalFSStore>(); - if (store2 && store->storeDir != store2->getRealStoreDir()) { + if (!store2) + throw Error("store '%s' is not a local store so it does not support command execution", store->getUri()); + + if (store->storeDir != store2->getRealStoreDir()) { Strings helperArgs = { chrootHelperName, store->storeDir, store2->getRealStoreDir(), program }; for (auto & arg : args) helperArgs.push_back(arg); @@ -179,6 +182,7 @@ struct CmdRun : InstallableCommand { auto state = getEvalState(); + lockFlags.applyNixConfig = true; auto app = installable->toApp(*state).resolve(getEvalStore(), store); Strings allArgs{app.program}; diff --git a/src/nix/search.cc b/src/nix/search.cc index e9307342c..e96a85ea2 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -165,8 +165,8 @@ struct CmdSearch : InstallableCommand, MixJSON } }; - for (auto & [cursor, prefix] : installable->getCursors(*state)) - visit(*cursor, parseAttrPath(*state, prefix), true); + for (auto & cursor : installable->getCursors(*state)) + visit(*cursor, cursor->getAttrPath(), true); if (!json && !results) throw Error("no results for the given search term(s)!"); diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 61a02c9b3..fb46b4dbf 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -65,19 +65,23 @@ struct CmdShowDerivation : InstallablesCommand auto & outputName = _outputName; // work around clang bug auto outputObj { outputsObj.object(outputName) }; std::visit(overloaded { - [&](const DerivationOutputInputAddressed & doi) { + [&](const DerivationOutput::InputAddressed & doi) { outputObj.attr("path", store->printStorePath(doi.path)); }, - [&](const DerivationOutputCAFixed & dof) { + [&](const DerivationOutput::CAFixed & dof) { outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName))); outputObj.attr("hashAlgo", dof.hash.printMethodAlgo()); outputObj.attr("hash", dof.hash.hash.to_string(Base16, false)); }, - [&](const DerivationOutputCAFloating & dof) { + [&](const DerivationOutput::CAFloating & dof) { outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType)); }, - [&](const DerivationOutputDeferred &) {}, - }, output.output); + [&](const DerivationOutput::Deferred &) {}, + [&](const DerivationOutput::Impure & doi) { + outputObj.attr("hashAlgo", makeFileIngestionPrefix(doi.method) + printHashType(doi.hashType)); + outputObj.attr("impure", true); + }, + }, output.raw()); } } diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc index 079cd6b3e..2e288f743 100644 --- a/src/nix/store-copy-log.cc +++ b/src/nix/store-copy-log.cc @@ -1,6 +1,8 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" +#include "store-cast.hh" +#include "log-store.hh" #include "sync.hh" #include "thread-pool.hh" @@ -26,7 +28,10 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand void run(ref<Store> srcStore) override { + auto & srcLogStore = require<LogStore>(*srcStore); + auto dstStore = getDstStore(); + auto & dstLogStore = require<LogStore>(*dstStore); StorePathSet drvPaths; @@ -35,8 +40,8 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand drvPaths.insert(drvPath); for (auto & drvPath : drvPaths) { - if (auto log = srcStore->getBuildLog(drvPath)) - dstStore->addBuildLog(drvPath, *log); + if (auto log = srcLogStore.getBuildLog(drvPath)) + dstLogStore.addBuildLog(drvPath, *log); else throw Error("build log for '%s' is not available", srcStore->printStorePath(drvPath)); } diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc index aa7a8b12f..ca43f1530 100644 --- a/src/nix/store-delete.cc +++ b/src/nix/store-delete.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "store-cast.hh" #include "gc-store.hh" using namespace nix; @@ -33,7 +34,7 @@ struct CmdStoreDelete : StorePathsCommand void run(ref<Store> store, std::vector<StorePath> && storePaths) override { - auto & gcStore = requireGcStore(*store); + auto & gcStore = require<GcStore>(*store); for (auto & path : storePaths) options.pathsToDelete.insert(path); diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc index 21718dc0c..8b9b5d164 100644 --- a/src/nix/store-gc.cc +++ b/src/nix/store-gc.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "store-cast.hh" #include "gc-store.hh" using namespace nix; @@ -34,7 +35,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun void run(ref<Store> store) override { - auto & gcStore = requireGcStore(*store); + auto & gcStore = require<GcStore>(*store); options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; GCResults results; diff --git a/tests/binary-cache.sh b/tests/binary-cache.sh index 2368884f7..0361ac6a8 100644 --- a/tests/binary-cache.sh +++ b/tests/binary-cache.sh @@ -1,6 +1,6 @@ source common.sh -needLocalStore "“--no-require-sigs” can’t be used with the daemon" +needLocalStore "'--no-require-sigs' can’t be used with the daemon" # We can produce drvs directly into the binary cache clearStore diff --git a/tests/build-dry.sh b/tests/build-dry.sh index e72533e70..f0f38e9a0 100644 --- a/tests/build-dry.sh +++ b/tests/build-dry.sh @@ -50,3 +50,22 @@ nix build -f dependencies.nix -o $RESULT --dry-run nix build -f dependencies.nix -o $RESULT [[ -h $RESULT ]] + +################################################### +# Check the JSON output +clearStore +clearCache + +RES=$(nix build -f dependencies.nix --dry-run --json) + +if [[ -z "$NIX_TESTS_CA_BY_DEFAULT" ]]; then + echo "$RES" | jq '.[0] | [ + (.drvPath | test("'$NIX_STORE_DIR'.*\\.drv")), + (.outputs.out | test("'$NIX_STORE_DIR'")) + ] | all' +else + echo "$RES" | jq '.[0] | [ + (.drvPath | test("'$NIX_STORE_DIR'.*\\.drv")), + .outputs.out == null + ] | all' +fi diff --git a/tests/build.sh b/tests/build.sh index c77f620f7..13a0f42be 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -1,15 +1,27 @@ source common.sh -expectedJSONRegex='\[\{"drvPath":".*multiple-outputs-a.drv","outputs":\{"first":".*multiple-outputs-a-first","second":".*multiple-outputs-a-second"}},\{"drvPath":".*multiple-outputs-b.drv","outputs":\{"out":".*multiple-outputs-b"}}]' +clearStore + +# Make sure that 'nix build' only returns the outputs we asked for. +nix build -f multiple-outputs.nix --json a --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys | length == 1) and + (.outputs.first | match(".*multiple-outputs-a-first"))) +' + nix build -f multiple-outputs.nix --json a.all b.all --no-link | jq --exit-status ' (.[0] | (.drvPath | match(".*multiple-outputs-a.drv")) and + (.outputs | keys | length == 2) and (.outputs.first | match(".*multiple-outputs-a-first")) and (.outputs.second | match(".*multiple-outputs-a-second"))) and (.[1] | (.drvPath | match(".*multiple-outputs-b.drv")) and + (.outputs | keys | length == 1) and (.outputs.out | match(".*multiple-outputs-b"))) ' + testNormalization () { clearStore outPath=$(nix-build ./simple.nix --no-out-link) diff --git a/tests/ca/build-dry.sh b/tests/ca/build-dry.sh new file mode 100644 index 000000000..9a72075ec --- /dev/null +++ b/tests/ca/build-dry.sh @@ -0,0 +1,6 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. && source build-dry.sh + diff --git a/tests/ca/content-addressed.nix b/tests/ca/content-addressed.nix index d328fc92c..1be3eeb6e 100644 --- a/tests/ca/content-addressed.nix +++ b/tests/ca/content-addressed.nix @@ -64,8 +64,7 @@ rec { dependentFixedOutput = mkDerivation { name = "dependent-fixed-output"; outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - outputHash = "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA="; + outputHash = "sha512-7aJcmSuEuYP5tGKcmGY8bRr/lrCjJlOxP2mIUjO/vMQeg6gx/65IbzRWES8EKiPDOs9z+wF30lEfcwxM/cT4pw=="; buildCommand = '' cat ${dependentCA}/dep echo foo > $out diff --git a/tests/eval.nix b/tests/eval.nix new file mode 100644 index 000000000..befbd17a9 --- /dev/null +++ b/tests/eval.nix @@ -0,0 +1,5 @@ +{ + int = 123; + str = "foo"; + attr.foo = "bar"; +} diff --git a/tests/eval.sh b/tests/eval.sh new file mode 100644 index 000000000..2e5ceb969 --- /dev/null +++ b/tests/eval.sh @@ -0,0 +1,29 @@ +source common.sh + +clearStore + +testStdinHeredoc=$(nix eval -f - <<EOF +{ + bar = 3 + 1; + foo = 2 + 2; +} +EOF +) +[[ $testStdinHeredoc == '{ bar = 4; foo = 4; }' ]] + +nix eval --expr 'assert 1 + 2 == 3; true' + +[[ $(nix eval int -f "./eval.nix") == 123 ]] +[[ $(nix eval str -f "./eval.nix") == '"foo"' ]] +[[ $(nix eval str --raw -f "./eval.nix") == 'foo' ]] +[[ $(nix eval attr -f "./eval.nix") == '{ foo = "bar"; }' ]] +[[ $(nix eval attr --json -f "./eval.nix") == '{"foo":"bar"}' ]] +[[ $(nix eval int -f - < "./eval.nix") == 123 ]] + + +nix-instantiate --eval -E 'assert 1 + 2 == 3; true' +[[ $(nix-instantiate -A int --eval "./eval.nix") == 123 ]] +[[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo"' ]] +[[ $(nix-instantiate -A attr --eval "./eval.nix") == '{ foo = "bar"; }' ]] +[[ $(nix-instantiate -A attr --eval --json "./eval.nix") == '{"foo":"bar"}' ]] +[[ $(nix-instantiate -A int --eval - < "./eval.nix") == 123 ]] diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh new file mode 100644 index 000000000..96e4bb741 --- /dev/null +++ b/tests/fetchClosure.sh @@ -0,0 +1,70 @@ +source common.sh + +enableFeatures "fetch-closure" +needLocalStore "'--no-require-sigs' can’t be used with the daemon" + +clearStore +clearCacheCache + +# Initialize binary cache. +nonCaPath=$(nix build --json --file ./dependencies.nix | jq -r .[].outputs.out) +caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') +nix copy --to file://$cacheDir $nonCaPath + +# Test basic fetchClosure rewriting from non-CA to CA. +clearStore + +[ ! -e $nonCaPath ] +[ ! -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = $caPath; + } +") = $caPath ]] + +[ ! -e $nonCaPath ] +[ -e $caPath ] + +# In impure mode, we can use non-CA paths. +[[ $(nix eval --raw --no-require-sigs --impure --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + } +") = $nonCaPath ]] + +[ -e $nonCaPath ] + +# 'toPath' set to empty string should fail but print the expected path. +nix eval -v --json --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = \"\"; + } +" 2>&1 | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath" + +# If fromPath is CA, then toPath isn't needed. +nix copy --to file://$cacheDir $caPath + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + } +") = $caPath ]] + +# Check that URL query parameters aren't allowed. +clearStore +narCache=$TEST_ROOT/nar-cache +rm -rf $narCache +(! nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir?local-nar-cache=$narCache\"; + fromPath = $caPath; + } +") +(! [ -e $narCache ]) diff --git a/tests/fetchGit.sh b/tests/fetchGit.sh index ac23be5c0..9179e2071 100644 --- a/tests/fetchGit.sh +++ b/tests/fetchGit.sh @@ -7,7 +7,9 @@ fi clearStore -repo=$TEST_ROOT/git +# Intentionally not in a canonical form +# See https://github.com/NixOS/nix/issues/6195 +repo=$TEST_ROOT/./git export _NIX_FORCE_HTTP=1 diff --git a/tests/fetchMercurial.sh b/tests/fetchMercurial.sh index 726840664..5c64ffd26 100644 --- a/tests/fetchMercurial.sh +++ b/tests/fetchMercurial.sh @@ -7,7 +7,9 @@ fi clearStore -repo=$TEST_ROOT/hg +# Intentionally not in a canonical form +# See https://github.com/NixOS/nix/issues/6195 +repo=$TEST_ROOT/./hg rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix @@ -28,6 +30,12 @@ echo world > $repo/hello hg commit --cwd $repo -m 'Bla2' rev2=$(hg log --cwd $repo -r tip --template '{node}') +# Fetch an unclean branch. +echo unclean > $repo/hello +path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath") +[[ $(cat $path/hello) = unclean ]] +hg revert --cwd $repo --all + # Fetch the default branch. path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath") [[ $(cat $path/hello) = world ]] diff --git a/tests/fetchPath.sh b/tests/fetchPath.sh new file mode 100644 index 000000000..29be38ce2 --- /dev/null +++ b/tests/fetchPath.sh @@ -0,0 +1,6 @@ +source common.sh + +touch $TEST_ROOT/foo -t 202211111111 +# We only check whether 2022-11-1* **:**:** is the last modified date since +# `lastModified` is transformed into UTC in `builtins.fetchTarball`. +[[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]] diff --git a/tests/flakes-run.sh b/tests/flakes-run.sh new file mode 100644 index 000000000..88fc3e628 --- /dev/null +++ b/tests/flakes-run.sh @@ -0,0 +1,29 @@ +source common.sh + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local +cp ./shell-hello.nix ./config.nix $TEST_HOME +cd $TEST_HOME + +cat <<EOF > flake.nix +{ + outputs = {self}: { + packages.$system.pkgAsPkg = (import ./shell-hello.nix).hello; + packages.$system.appAsApp = self.packages.$system.appAsApp; + + apps.$system.pkgAsApp = self.packages.$system.pkgAsPkg; + apps.$system.appAsApp = { + type = "app"; + program = "\${(import ./shell-hello.nix).hello}/bin/hello"; + }; + }; +} +EOF +nix run --no-write-lock-file .#appAsApp +nix run --no-write-lock-file .#pkgAsPkg + +! nix run --no-write-lock-file .#pkgAsApp || fail "'nix run' shouldn’t accept an 'app' defined under 'packages'" +! nix run --no-write-lock-file .#appAsPkg || fail "elements of 'apps' should be of type 'app'" + +clearStore + diff --git a/tests/flakes.sh b/tests/flakes.sh index ea629ae70..46e6a7982 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -376,6 +376,9 @@ cat > $templatesDir/flake.nix <<EOF trivial = { path = ./trivial; description = "A trivial flake"; + welcomeText = '' + Welcome to my trivial flake + ''; }; default = trivial; }; diff --git a/tests/impure-derivations.nix b/tests/impure-derivations.nix new file mode 100644 index 000000000..98547e6c1 --- /dev/null +++ b/tests/impure-derivations.nix @@ -0,0 +1,63 @@ +with import ./config.nix; + +rec { + + impure = mkDerivation { + name = "impure"; + outputs = [ "out" "stuff" ]; + buildCommand = + '' + echo impure + x=$(< $TEST_ROOT/counter) + mkdir $out $stuff + echo $x > $out/n + ln -s $out/n $stuff/bla + printf $((x + 1)) > $TEST_ROOT/counter + ''; + __impure = true; + impureEnvVars = [ "TEST_ROOT" ]; + }; + + impureOnImpure = mkDerivation { + name = "impure-on-impure"; + buildCommand = + '' + echo impure-on-impure + x=$(< ${impure}/n) + mkdir $out + printf X$x > $out/n + ln -s ${impure.stuff} $out/symlink + ln -s $out $out/self + ''; + __impure = true; + }; + + # This is not allowed. + inputAddressed = mkDerivation { + name = "input-addressed"; + buildCommand = + '' + cat ${impure} > $out + ''; + }; + + contentAddressed = mkDerivation { + name = "content-addressed"; + buildCommand = + '' + echo content-addressed + x=$(< ${impureOnImpure}/n) + printf ''${x:0:1} > $out + ''; + outputHashMode = "recursive"; + outputHash = "sha256-eBYxcgkuWuiqs4cKNgKwkb3vY/HR0vVsJnqe8itJGcQ="; + }; + + inputAddressedAfterCA = mkDerivation { + name = "input-addressed-after-ca"; + buildCommand = + '' + cat ${contentAddressed} > $out + ''; + }; +} diff --git a/tests/impure-derivations.sh b/tests/impure-derivations.sh new file mode 100644 index 000000000..35ae3f5d3 --- /dev/null +++ b/tests/impure-derivations.sh @@ -0,0 +1,57 @@ +source common.sh + +requireDaemonNewerThan "2.8pre20220311" + +enableFeatures "ca-derivations ca-references impure-derivations" +restartDaemon + +set -o pipefail + +clearStore + +# Basic test of impure derivations: building one a second time should not use the previous result. +printf 0 > $TEST_ROOT/counter + +json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all) +path1=$(echo $json | jq -r .[].outputs.out) +path1_stuff=$(echo $json | jq -r .[].outputs.stuff) +[[ $(< $path1/n) = 0 ]] +[[ $(< $path1_stuff/bla) = 0 ]] + +[[ $(nix path-info --json $path1 | jq .[].ca) =~ fixed:r:sha256: ]] + +path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out) +[[ $(< $path2/n) = 1 ]] + +# Test impure derivations that depend on impure derivations. +path3=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out) +[[ $(< $path3/n) = X2 ]] + +path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out) +[[ $(< $path4/n) = X3 ]] + +# Test that (self-)references work. +[[ $(< $path4/symlink/bla) = 3 ]] +[[ $(< $path4/self/n) = X3 ]] + +# Input-addressed derivations cannot depend on impure derivations directly. +(! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' + +drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) +[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] +[[ $(nix show-derivation $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] + +# Fixed-output derivations *can* depend on impure derivations. +path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) +[[ $(< $path5) = X ]] +[[ $(< $TEST_ROOT/counter) = 5 ]] + +# And they should not be rebuilt. +path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) +[[ $(< $path5) = X ]] +[[ $(< $TEST_ROOT/counter) = 5 ]] + +# Input-addressed derivations can depend on fixed-output derivations that depend on impure derivations. +path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out) +[[ $(< $path6) = X ]] +[[ $(< $TEST_ROOT/counter) = 5 ]] diff --git a/tests/local.mk b/tests/local.mk index b42a5bccb..cb869f32e 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -1,5 +1,6 @@ nix_tests = \ flakes.sh \ + flakes-run.sh \ ca/gc.sh \ gc.sh \ remote-store.sh \ @@ -21,6 +22,7 @@ nix_tests = \ tarball.sh \ fetchGit.sh \ fetchurl.sh \ + fetchPath.sh \ simple.sh \ referrers.sh \ optimise-store.sh \ @@ -52,6 +54,7 @@ nix_tests = \ build-remote-content-addressed-floating.sh \ nar-access.sh \ pure-eval.sh \ + eval.sh \ ca/post-hook.sh \ repl.sh \ ca/repl.sh \ @@ -95,7 +98,9 @@ nix_tests = \ describe-stores.sh \ nix-profile.sh \ suggestions.sh \ - store-ping.sh + store-ping.sh \ + fetchClosure.sh \ + impure-derivations.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/logging.sh b/tests/logging.sh index c894ad3ff..1481b9b36 100644 --- a/tests/logging.sh +++ b/tests/logging.sh @@ -13,3 +13,14 @@ rm -rf $NIX_LOG_DIR (! nix-store -l $path) nix-build dependencies.nix --no-out-link --compress-build-log [ "$(nix-store -l $path)" = FOO ] + +# test whether empty logs work fine with `nix log`. +builder="$(mktemp)" +echo -e "#!/bin/sh\nmkdir \$out" > "$builder" +outp="$(nix-build -E \ + 'with import ./config.nix; mkDerivation { name = "fnord"; builder = '"$builder"'; }' \ + --out-link "$(mktemp -d)/result")" + +test -d "$outp" + +nix log "$outp" diff --git a/tests/sourcehut-flakes.nix b/tests/sourcehut-flakes.nix index d1d89d149..6a1930904 100644 --- a/tests/sourcehut-flakes.nix +++ b/tests/sourcehut-flakes.nix @@ -59,7 +59,7 @@ let echo 'ref: refs/heads/master' > $out/HEAD mkdir -p $out/info - echo '${nixpkgs.rev} refs/heads/master' > $out/info/refs + echo -e '${nixpkgs.rev}\trefs/heads/master' > $out/info/refs ''; in diff --git a/tests/user-envs.nix b/tests/user-envs.nix index 6ac896ed8..46f8b51dd 100644 --- a/tests/user-envs.nix +++ b/tests/user-envs.nix @@ -8,6 +8,8 @@ assert foo == "foo"; let + platforms = let x = "foobar"; in [ x x ]; + makeDrv = name: progName: (mkDerivation { name = assert progName != "fail"; name; inherit progName system; @@ -15,6 +17,7 @@ let } // { meta = { description = "A silly test package with some \${escaped anti-quotation} in it"; + inherit platforms; }; }); diff --git a/tests/user-envs.sh b/tests/user-envs.sh index 430688de1..d63fe780a 100644 --- a/tests/user-envs.sh +++ b/tests/user-envs.sh @@ -17,6 +17,16 @@ outPath10=$(nix-env -f ./user-envs.nix -qa --out-path --no-name '*' | grep foo-1 drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1.0) [ -n "$outPath10" -a -n "$drvPath10" ] +# Query with json +nix-env -f ./user-envs.nix -qa --json | jq -e '.[] | select(.name == "bar-0.1") | [ + .outputName == "out", + .outputs.out == null +] | all' +nix-env -f ./user-envs.nix -qa --json --out-path | jq -e '.[] | select(.name == "bar-0.1") | [ + .outputName == "out", + (.outputs.out | test("'$NIX_STORE_DIR'.*-0\\.1")) +] | all' + # Query descriptions. nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly rm -rf $HOME/.nix-defexpr |