diff options
88 files changed, 1912 insertions, 993 deletions
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml deleted file mode 100644 index 12c60c649..000000000 --- a/.github/workflows/backport.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Backport -on: - pull_request_target: - types: [closed, labeled] -permissions: - contents: read -jobs: - backport: - name: Backport Pull Request - permissions: - # for zeebe-io/backport-action - contents: write - pull-requests: write - if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - # required to find all branches - fetch-depth: 0 - - name: Create backport PRs - # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.4.0 - with: - # Config README: https://github.com/zeebe-io/backport-action#backport-action - github_token: ${{ secrets.GITHUB_TOKEN }} - github_workspace: ${{ github.workspace }} - pull_description: |- - Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}. - # should be kept in sync with `uses` - version: v0.0.5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 0c9c24dad..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: "CI" - -on: - pull_request: - push: - -permissions: read-all - -jobs: - - tests: - needs: [check_secrets] - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: cachix/install-nix-action@v23 - with: - # The sandbox would otherwise be disabled by default on Darwin - extra_nix_config: "sandbox = true" - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 - if: needs.check_secrets.outputs.cachix == 'true' - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - run: nix --experimental-features 'nix-command flakes' flake check -L - - check_secrets: - permissions: - contents: none - name: Check Cachix and Docker secrets present for installer tests - runs-on: ubuntu-latest - outputs: - cachix: ${{ steps.secret.outputs.cachix }} - docker: ${{ steps.secret.outputs.docker }} - steps: - - name: Check for secrets - id: secret - env: - _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} - _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} - run: | - echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}" - echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - - installer: - needs: [tests, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' - runs-on: ubuntu-latest - outputs: - installerURL: ${{ steps.prepare-installer.outputs.installerURL }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v23 - with: - install_url: https://releases.nixos.org/nix/nix-2.13.3/install - - uses: cachix/cachix-action@v12 - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - id: prepare-installer - run: scripts/prepare-installer-for-github-actions - - installer_test: - needs: [installer, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v23 - with: - install_url: '${{needs.installer.outputs.installerURL}}' - install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" - - run: sudo apt install fish zsh - if: matrix.os == 'ubuntu-latest' - - run: brew install fish - if: matrix.os == 'macos-latest' - - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec fish -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" - - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" - - docker_push_image: - needs: [check_secrets, tests] - if: >- - github.event_name == 'push' && - github.ref_name == 'master' && - needs.check_secrets.outputs.cachix == 'true' && - needs.check_secrets.outputs.docker == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: cachix/install-nix-action@v23 - with: - install_url: https://releases.nixos.org/nix/nix-2.13.3/install - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 - if: needs.check_secrets.outputs.cachix == 'true' - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - - run: docker load -i ./result/image.tar.gz - - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - - run: docker tag nix:$NIX_VERSION nixos/nix:master - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - run: docker push nixos/nix:$NIX_VERSION - - run: docker push nixos/nix:master diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml deleted file mode 100644 index 2fa89d72c..000000000 --- a/.github/workflows/hydra_status.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Hydra status - -permissions: read-all - -on: - schedule: - - cron: "12,42 * * * *" - workflow_dispatch: - -jobs: - check_hydra_status: - name: Check Hydra status - if: github.repository_owner == 'NixOS' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: bash scripts/check-hydra-status.sh diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml deleted file mode 100644 index d83cb4f18..000000000 --- a/.github/workflows/labels.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: "Label PR" - -on: - pull_request_target: - types: [edited, opened, synchronize, reopened] - -# WARNING: -# When extending this action, be aware that $GITHUB_TOKEN allows some write -# access to the GitHub API. This means that it should not evaluate user input in -# a way that allows code injection. - -permissions: - contents: read - pull-requests: write - -jobs: - labels: - runs-on: ubuntu-latest - if: github.repository_owner == 'NixOS' - steps: - - uses: actions/labeler@v4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - sync-labels: false diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 215609f79..50c67b8ae 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -29,7 +29,7 @@ man-pages += $(foreach subcommand, \ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. -# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine +# Set cores to 0 because otherwise nix config show resolves the cores based on the current machine dummy-env = env -i \ HOME=/dummy \ NIX_CONF_DIR=/dummy \ @@ -89,7 +89,7 @@ doc/manual/generated/in/nix.json: $(doc_nix) doc/manual/generated/in/conf-file.json: $(doc_nix) @mkdir -p doc/manual/generated/in - $(trace-gen) $(dummy-env) $(doc_nix) show-config --json --experimental-features nix-command > $@.tmp + $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ doc/manual/generated/in/contributing/experimental-feature-descriptions.md: doc/manual/generated/in/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) diff --git a/doc/manual/meson.build b/doc/manual/meson.build index cfb6be36f..e253a9bd8 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -197,6 +197,8 @@ endforeach nix3_manpages = [ 'nix3-build', 'nix3-bundle', + 'nix3-config', + 'nix3-config-show', 'nix3-copy', 'nix3-daemon', 'nix3-derivation-add', @@ -258,7 +260,6 @@ nix3_manpages = [ 'nix3-run', 'nix3-search', 'nix3-shell', - 'nix3-show-config', 'nix3-store-add-file', 'nix3-store-add-path', 'nix3-store-cat', diff --git a/doc/manual/rl-next/leading-period.md b/doc/manual/rl-next/leading-period.md new file mode 100644 index 000000000..7a2fd1f67 --- /dev/null +++ b/doc/manual/rl-next/leading-period.md @@ -0,0 +1,10 @@ +--- +synopsis: Store paths are allowed to start with `.` +issues: 912 +prs: [9867, 9091, 9095, 9120, 9121, 9122, 9130, 9219, 9224] +--- + +Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. +From now on, leading periods are officially, definitively supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + +Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md new file mode 100644 index 000000000..1e7545e73 --- /dev/null +++ b/doc/manual/rl-next/nix-config-show.md @@ -0,0 +1,7 @@ +--- +synopsis: rename 'nix show-config' to 'nix config show' +issues: 7672 +prs: 9477 +--- + +`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface. diff --git a/doc/manual/rl-next/nix-profile-names.md b/doc/manual/rl-next/nix-profile-names.md new file mode 100644 index 000000000..53dc53fa9 --- /dev/null +++ b/doc/manual/rl-next/nix-profile-names.md @@ -0,0 +1,9 @@ +--- +synopsis: "`nix profile` now allows referring to elements by human-readable name, and no longer accepts indices" +prs: 8678 +cls: [978, 980] +--- + +[`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) now uses names to refer to installed packages when running [`list`](@docroot@/command-ref/new-cli/nix3-profile-list.md), [`remove`](@docroot@/command-ref/new-cli/nix3-profile-remove.md) or [`upgrade`](@docroot@/command-ref/new-cli/nix3-profile-upgrade.md) as opposed to indices. Indices have been removed. Profile element names are generated when a package is installed and remain the same until the package is removed. + +**Warning**: The `manifest.nix` file used to record the contents of profiles has changed. Nix will automatically upgrade profiles to the new version when you modify the profile. After that, the profile can no longer be used by older versions of Nix. diff --git a/doc/manual/rl-next/upgrade-nix-override.md b/doc/manual/rl-next/upgrade-nix-override.md new file mode 100644 index 000000000..d3046ff13 --- /dev/null +++ b/doc/manual/rl-next/upgrade-nix-override.md @@ -0,0 +1,6 @@ +--- +synopsis: add --store-path argument to `nix upgrade-nix`, to manually specify the Nix to upgrade to +cls: 953 +--- + +`nix upgrade-nix` by default downloads a manifest to find the new Nix version to upgrade to, but now you can specify `--store-path` to upgrade Nix to an arbitrary version from the Nix store. diff --git a/doc/manual/rl-next/upgrade-nix-profile-compat.md b/doc/manual/rl-next/upgrade-nix-profile-compat.md new file mode 100644 index 000000000..df9879c6f --- /dev/null +++ b/doc/manual/rl-next/upgrade-nix-profile-compat.md @@ -0,0 +1,8 @@ +--- +synopsis: using `nix profile` on `/nix/var/nix/profiles/default` no longer breaks `nix upgrade-nix` +cls: 952 +--- + +On non-NixOS, Nix is conventionally installed into a `nix-env` style profile at /nix/var/nix/profiles/default. +Like any `nix-env` profile, using `nix profile` on it automatically migrates it to a `nix profile` style profile, which is incompatible with `nix-env`. +`nix upgrade-nix` previously relied solely on `nix-env` to do the upgrade, but now will work fine with either kind of profile. diff --git a/doc/manual/src/SUMMARY.md b/doc/manual/src/SUMMARY.md index 7747b9061..2437c0dc5 100644 --- a/doc/manual/src/SUMMARY.md +++ b/doc/manual/src/SUMMARY.md @@ -91,6 +91,8 @@ - [nix](command-ref/new-cli/nix.md) - [nix build](command-ref/new-cli/nix3-build.md) - [nix bundle](command-ref/new-cli/nix3-bundle.md) + - [nix config](command-ref/new-cli/nix3-config.md) + - [nix config show](command-ref/new-cli/nix3-config-show.md) - [nix copy](command-ref/new-cli/nix3-copy.md) - [nix daemon](command-ref/new-cli/nix3-daemon.md) - [nix derivation](command-ref/new-cli/nix3-derivation.md) diff --git a/doc/manual/src/command-ref/new-cli/nix3-config-show.md b/doc/manual/src/command-ref/new-cli/nix3-config-show.md new file mode 100644 index 000000000..a39cd13e9 --- /dev/null +++ b/doc/manual/src/command-ref/new-cli/nix3-config-show.md @@ -0,0 +1 @@ +{{#include @generated@/command-ref/new-cli/nix3-config-show.md}} diff --git a/doc/manual/src/command-ref/new-cli/nix3-config.md b/doc/manual/src/command-ref/new-cli/nix3-config.md new file mode 100644 index 000000000..ba824c7bc --- /dev/null +++ b/doc/manual/src/command-ref/new-cli/nix3-config.md @@ -0,0 +1 @@ +{{#include @generated@/command-ref/new-cli/nix3-config.md}} diff --git a/doc/manual/src/command-ref/new-cli/nix3-show-config.md b/doc/manual/src/command-ref/new-cli/nix3-show-config.md deleted file mode 100644 index 060fc065d..000000000 --- a/doc/manual/src/command-ref/new-cli/nix3-show-config.md +++ /dev/null @@ -1 +0,0 @@ -{{#include @generated@/command-ref/new-cli/nix3-show-config.md}} diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index 941723216..5a9e05fed 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -25,17 +25,17 @@ individual users can switch between different environments. `nix-env` takes exactly one *operation* flag which indicates the subcommand to be performed. The following operations are available: -- [`--install`](./nix-env/install.md) -- [`--upgrade`](./nix-env/upgrade.md) -- [`--uninstall`](./nix-env/uninstall.md) -- [`--set`](./nix-env/set.md) -- [`--set-flag`](./nix-env/set-flag.md) -- [`--query`](./nix-env/query.md) -- [`--switch-profile`](./nix-env/switch-profile.md) -- [`--list-generations`](./nix-env/list-generations.md) -- [`--delete-generations`](./nix-env/delete-generations.md) -- [`--switch-generation`](./nix-env/switch-generation.md) -- [`--rollback`](./nix-env/rollback.md) +- [`--install`](./nix-env/install.md) - add packages to user environment +- [`--upgrade`](./nix-env/upgrade.md) - upgrade packages in user environment +- [`--uninstall`](./nix-env/uninstall.md) - remove packages from user environment +- [`--set`](./nix-env/set.md) - set profile to contain a specified derivation +- [`--set-flag`](./nix-env/set-flag.md) - modify meta attributes of installed packages +- [`--query`](./nix-env/query.md) - display information about packages +- [`--switch-profile`](./nix-env/switch-profile.md) - set user environment to a given profile +- [`--list-generations`](./nix-env/list-generations.md) - list profile generations +- [`--delete-generations`](./nix-env/delete-generations.md) - delete profile generations +- [`--switch-generation`](./nix-env/switch-generation.md) - set user environment to a given profile generation +- [`--rollback`](./nix-env/rollback.md) - set user environment to previous generation These pages can be viewed offline: diff --git a/src/libcmd/cmd-profiles.cc b/src/libcmd/cmd-profiles.cc new file mode 100644 index 000000000..101064956 --- /dev/null +++ b/src/libcmd/cmd-profiles.cc @@ -0,0 +1,308 @@ +#include <set> + +#include "cmd-profiles.hh" +#include "built-path.hh" +#include "builtins/buildenv.hh" +#include "logging.hh" +#include "names.hh" +#include "store-api.hh" +#include "url-name.hh" + +namespace nix +{ + +DrvInfos queryInstalled(EvalState & state, const Path & userEnv) +{ + DrvInfos elems; + if (pathExists(userEnv + "/manifest.json")) + throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); + auto manifestFile = userEnv + "/manifest.nix"; + if (pathExists(manifestFile)) { + Value v; + state.evalFile(state.rootPath(CanonPath(manifestFile)), v); + Bindings & bindings(*state.allocBindings(0)); + getDerivations(state, v, "", bindings, elems, false); + } + return elems; +} + +std::string showVersions(const std::set<std::string> & versions) +{ + if (versions.empty()) return "∅"; + std::set<std::string> versions2; + for (auto & version : versions) + versions2.insert(version.empty() ? "ε" : version); + return concatStringsSep(", ", versions2); +} + +bool ProfileElementSource::operator<(const ProfileElementSource & other) const +{ + return std::tuple(originalRef.to_string(), attrPath, outputs) + < std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs); +} + +std::string ProfileElementSource::to_string() const +{ + return fmt("%s#%s%s", originalRef, attrPath, outputs.to_string()); +} + +std::string ProfileElement::identifier() const +{ + if (source) { + return source->to_string(); + } + StringSet names; + for (auto & path : storePaths) { + names.insert(DrvName(path.name()).name); + } + return concatStringsSep(", ", names); +} + +std::set<std::string> ProfileElement::toInstallables(Store & store) +{ + if (source) { + return {source->to_string()}; + } + StringSet rawPaths; + for (auto & path : storePaths) { + rawPaths.insert(store.printStorePath(path)); + } + return rawPaths; +} + +std::string ProfileElement::versions() const +{ + StringSet versions; + for (auto & path : storePaths) { + versions.insert(DrvName(path.name()).version); + } + return showVersions(versions); +} + +bool ProfileElement::operator<(const ProfileElement & other) const +{ + return std::tuple(identifier(), storePaths) < std::tuple(other.identifier(), other.storePaths); +} + +void ProfileElement::updateStorePaths( + ref<Store> evalStore, ref<Store> store, const BuiltPaths & builtPaths +) +{ + storePaths.clear(); + for (auto & buildable : builtPaths) { + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & bo) { storePaths.insert(bo.path); }, + [&](const BuiltPath::Built & bfd) { + for (auto & output : bfd.outputs) { + storePaths.insert(output.second); + } + }, + }, + buildable.raw() + ); + } +} + +ProfileManifest::ProfileManifest(EvalState & state, const Path & profile) +{ + auto manifestPath = profile + "/manifest.json"; + + if (pathExists(manifestPath)) { + auto json = nlohmann::json::parse(readFile(manifestPath)); + + auto version = json.value("version", 0); + std::string sUrl; + std::string sOriginalUrl; + switch (version) { + case 1: + sUrl = "uri"; + sOriginalUrl = "originalUri"; + break; + case 2: + [[fallthrough]]; + case 3: + sUrl = "url"; + sOriginalUrl = "originalUrl"; + break; + default: + throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + } + + auto elems = json["elements"]; + + for (auto & elem : elems.items()) { + auto & e = elem.value(); + ProfileElement element; + for (auto & p : e["storePaths"]) { + element.storePaths.insert(state.store->parseStorePath((std::string) p)); + } + element.active = e["active"]; + if (e.contains("priority")) { + element.priority = e["priority"]; + } + if (e.value(sUrl, "") != "") { + element.source = ProfileElementSource{ + parseFlakeRef(e[sOriginalUrl]), + parseFlakeRef(e[sUrl]), + e["attrPath"], + e["outputs"].get<ExtendedOutputsSpec>()}; + } + + // TODO(Qyriad): holy crap this chain of ternaries needs cleanup. + std::string name = + elems.is_object() + ? elem.key() + : element.source + ? getNameFromURL(parseURL(element.source->to_string())).value_or(element.identifier()) + : element.identifier(); + + addElement(name, std::move(element)); + } + } else if (pathExists(profile + "/manifest.nix")) { + // FIXME: needed because of pure mode; ugly. + state.allowPath(state.store->followLinksToStore(profile)); + state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix")); + + auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + + for (auto & drvInfo : drvInfos) { + ProfileElement element; + element.storePaths = {drvInfo.queryOutPath()}; + addElement(std::move(element)); + } + } +} + +void ProfileManifest::addElement(std::string_view nameCandidate, ProfileElement element) +{ + std::string finalName(nameCandidate); + + for (unsigned i = 1; elements.contains(finalName); ++i) { + finalName = nameCandidate + "-" + std::to_string(i); + } + + elements.insert_or_assign(finalName, std::move(element)); +} + +void ProfileManifest::addElement(ProfileElement element) +{ + auto name = + element.source + ? getNameFromURL(parseURL(element.source->to_string())) + : std::nullopt; + + auto finalName = name.value_or(element.identifier()); + addElement(finalName, std::move(element)); +} + +nlohmann::json ProfileManifest::toJSON(Store & store) const +{ + auto es = nlohmann::json::object(); + for (auto & [name, element] : elements) { + auto paths = nlohmann::json::array(); + for (auto & path : element.storePaths) { + paths.push_back(store.printStorePath(path)); + } + nlohmann::json obj; + obj["storePaths"] = paths; + obj["active"] = element.active; + obj["priority"] = element.priority; + if (element.source) { + obj["originalUrl"] = element.source->originalRef.to_string(); + obj["url"] = element.source->lockedRef.to_string(); + obj["attrPath"] = element.source->attrPath; + obj["outputs"] = element.source->outputs; + } + es[name] = obj; + } + nlohmann::json json; + json["version"] = 3; + json["elements"] = es; + return json; +} + +StorePath ProfileManifest::build(ref<Store> store) +{ + auto tempDir = createTempDir(); + + StorePathSet references; + + Packages pkgs; + for (auto & [name, element] : elements) { + for (auto & path : element.storePaths) { + if (element.active) { + pkgs.emplace_back(store->printStorePath(path), true, element.priority); + } + references.insert(path); + } + } + + buildProfile(tempDir, std::move(pkgs)); + + writeFile(tempDir + "/manifest.json", toJSON(*store).dump()); + + /* Add the symlink tree to the store. */ + StringSink sink; + dumpPath(tempDir, sink); + + auto narHash = hashString(htSHA256, sink.s); + + ValidPathInfo info{ + *store, + "profile", + FixedOutputInfo{ + .method = FileIngestionMethod::Recursive, + .hash = narHash, + .references = + { + .others = std::move(references), + // profiles never refer to themselves + .self = false, + }, + }, + narHash, + }; + info.narSize = sink.s.size(); + + StringSource source(sink.s); + store->addToStore(info, source); + + return std::move(info.path); +} + +void ProfileManifest::printDiff( + const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent +) +{ + auto prevElemIt = prev.elements.begin(); + auto curElemIt = cur.elements.begin(); + + bool changes = false; + + while (prevElemIt != prev.elements.end() || curElemIt != cur.elements.end()) { + if (curElemIt != cur.elements.end() && (prevElemIt == prev.elements.end() || prevElemIt->first > curElemIt->first)) { + logger->cout("%s%s: ∅ -> %s", indent, curElemIt->second.identifier(), curElemIt->second.versions()); + changes = true; + ++curElemIt; + } else if (prevElemIt != prev.elements.end() && (curElemIt == cur.elements.end() || prevElemIt->first < curElemIt->first)) { + logger->cout("%s%s: %s -> ∅", indent, prevElemIt->second.identifier(), prevElemIt->second.versions()); + changes = true; + ++prevElemIt; + } else { + auto v1 = prevElemIt->second.versions(); + auto v2 = curElemIt->second.versions(); + if (v1 != v2) { + logger->cout("%s%s: %s -> %s", indent, prevElemIt->second.identifier(), v1, v2); + changes = true; + } + ++prevElemIt; + ++curElemIt; + } + } + + if (!changes) { + logger->cout("%sNo changes.", indent); + } +} +} diff --git a/src/libcmd/cmd-profiles.hh b/src/libcmd/cmd-profiles.hh new file mode 100644 index 000000000..2185daa34 --- /dev/null +++ b/src/libcmd/cmd-profiles.hh @@ -0,0 +1,78 @@ +#pragma once +///@file + +#include "built-path.hh" +#include "eval.hh" +#include "flake/flakeref.hh" +#include "get-drvs.hh" +#include "types.hh" +#include "url.hh" +#include "url-name.hh" + +#include <string> +#include <set> + +#include <nlohmann/json.hpp> + +namespace nix +{ + +struct ProfileElementSource +{ + FlakeRef originalRef; + // FIXME: record original attrpath. + FlakeRef lockedRef; + std::string attrPath; + ExtendedOutputsSpec outputs; + + bool operator<(const ProfileElementSource & other) const; + + std::string to_string() const; +}; + +constexpr int DEFAULT_PRIORITY = 5; + +struct ProfileElement +{ + StorePathSet storePaths; + std::optional<ProfileElementSource> source; + bool active = true; + int priority = DEFAULT_PRIORITY; + + std::string identifier() const; + + /** + * Return a string representing an installable corresponding to the current + * element, either a flakeref or a plain store path + */ + std::set<std::string> toInstallables(Store & store); + + std::string versions() const; + + bool operator<(const ProfileElement & other) const; + + void updateStorePaths(ref<Store> evalStore, ref<Store> store, const BuiltPaths & builtPaths); +}; + +struct ProfileManifest +{ + std::map<std::string, ProfileElement> elements; + + ProfileManifest() { } + + ProfileManifest(EvalState & state, const Path & profile); + + nlohmann::json toJSON(Store & store) const; + + StorePath build(ref<Store> store); + + void addElement(std::string_view nameCandidate, ProfileElement element); + void addElement(ProfileElement element); + + static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent); +}; + +DrvInfos queryInstalled(EvalState & state, const Path & userEnv); +std::string showVersions(const std::set<std::string> & versions); + +} diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 120c832ac..6c6c81718 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -342,8 +342,6 @@ void completeFlakeRefWithFragment( const Strings & defaultFlakeAttrPaths, std::string_view prefix); -std::string showVersions(const std::set<std::string> & versions); - void printClosureDiff( ref<Store> store, const StorePath & beforePath, diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 6b1b44d84..5a0e61503 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -1,6 +1,7 @@ libcmd_sources = files( 'built-path.cc', 'command-installable-value.cc', + 'cmd-profiles.cc', 'command.cc', 'common-eval-args.cc', 'editor-for.cc', @@ -18,6 +19,7 @@ libcmd_sources = files( libcmd_headers = files( 'built-path.hh', 'command-installable-value.hh', + 'cmd-profiles.hh', 'command.hh', 'common-eval-args.hh', 'editor-for.hh', diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 248362c68..4278fab85 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -53,6 +53,9 @@ #if __APPLE__ #include <spawn.h> #include <sys/sysctl.h> + +/* This definition is undocumented but depended upon by all major browsers. */ +extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf); #endif #include <pwd.h> @@ -190,6 +193,7 @@ void LocalDerivationGoal::tryLocalBuild() throw Error("derivation '%s' has '__noChroot' set, " "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath)); #if __APPLE__ + additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); if (additionalSandboxProfile != "") throw Error("derivation '%s' specifies a sandbox profile, " "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath)); @@ -490,10 +494,6 @@ void LocalDerivationGoal::startBuilder() settings.thisSystem, concatStringsSep<StringSet>(", ", worker.store.systemFeatures)); -#if __APPLE__ - additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); -#endif - /* Create a temporary directory where the build will take place. */ tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); @@ -1716,7 +1716,7 @@ void LocalDerivationGoal::runChild() (which may run under a different uid and/or in a sandbox). */ std::string netrcData; try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl" && !derivationType->isSandboxed()) netrcData = readFile(settings.netrcFile); } catch (SysError &) { } @@ -2010,140 +2010,131 @@ void LocalDerivationGoal::runChild() std::string builder = "invalid"; - if (drv->isBuiltin()) { - ; - } #if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; - - if (useChroot) { - - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; - - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : pathsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useChroot) { - /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + PathSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : pathsInChroot) { + Path cur = i.first; while (cur.compare("/") != 0) { - ancestry.insert(cur); cur = dirOf(cur); + ancestry.insert(cur); } + } - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - pathsInChroot[p] = p; - } + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + Path cur = worker.store.storeDir; + while (cur.compare("/") != 0) { + ancestry.insert(cur); + cur = dirOf(cur); + } - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) { + auto p = worker.store.printStorePath(i); + pathsInChroot[p] = p; + } + + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ + if (settings.darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += + #include "sandbox-defaults.sb" + ; + if (!derivationType->isSandboxed()) sandboxProfile += - #include "sandbox-defaults.sb" + #include "sandbox-network.sb" ; - if (!derivationType->isSandboxed()) - sandboxProfile += - #include "sandbox-network.sb" - ; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : pathsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - std::string path = i.first; - auto optSt = maybeLstat(path.c_str()); - if (!optSt) { - if (i.second.optional) - continue; - throw SysError("getting attributes of required path '%s", path); - } - if (S_ISDIR(optSt->st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; + /* Add the output paths we'll use at build-time to the chroot */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; + sandboxProfile += ")\n"; - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += - #include "sandbox-minimal.sb" - ; + /* Our inputs (transitive dependencies and any impurities computed above) + + without file-write* allowed, access() incorrectly returns EPERM + */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & i : pathsInChroot) { + if (i.first != i.second.source) + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); + + std::string path = i.first; + struct stat st; + if (lstat(path.c_str(), &st)) { + if (i.second.optional && errno == ENOENT) + continue; + throw SysError("getting attributes of path '%s", path); + } + if (S_ISDIR(st.st_mode)) + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); + else + sandboxProfile += fmt("\t(literal \"%s\")\n", path); + } + sandboxProfile += ")\n"; - debug("Generated sandbox profile:"); - debug(sandboxProfile); + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += fmt("\t(literal \"%s\")\n", i); + } + sandboxProfile += ")\n"; - Path sandboxFile = tmpDir + "/.sandbox.sb"; + sandboxProfile += additionalSandboxProfile; + } else + sandboxProfile += + #include "sandbox-minimal.sb" + ; - writeFile(sandboxFile, sandboxProfile); + debug("Generated sandbox profile:"); + debug(sandboxProfile); - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); + bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ - Path globalTmpDir = canonPath(getEnvNonEmpty("TMPDIR").value_or("/tmp"), true); + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms + to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ + Path globalTmpDir = canonPath(getEnvNonEmpty("TMPDIR").value_or("/tmp"), true); - /* They don't like trailing slashes on subpath directives */ - if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); + /* They don't like trailing slashes on subpath directives */ + if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); - } else { - builder = drv->builder; - args.push_back(std::string(baseNameOf(drv->builder))); + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + Strings sandboxArgs; + sandboxArgs.push_back("_GLOBAL_TMP_DIR"); + sandboxArgs.push_back(globalTmpDir); + if (allowLocalNetworking) { + sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); + sandboxArgs.push_back("1"); + } + if (sandbox_init_with_parameters(sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), NULL)) { + writeFull(STDERR_FILENO, "failed to configure sandbox\n"); + _exit(1); } } + + builder = drv->builder; + args.push_back(std::string(baseNameOf(drv->builder))); #else - else { + if (!drv->isBuiltin()) { builder = drv->builder; args.push_back(std::string(baseNameOf(drv->builder))); } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 9dc742220..c9f5d6260 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -49,6 +49,7 @@ struct curlFileTransfer : public FileTransfer Activity act; bool done = false; // whether either the success or failure function has been called Callback<FileTransferResult> callback; + std::function<void(TransferItem &, std::string_view data)> dataCallback; CURL * req = 0; bool active = false; // whether the handle has been added to the multi object bool headersProcessed = false; @@ -82,23 +83,15 @@ struct curlFileTransfer : public FileTransfer TransferItem(curlFileTransfer & fileTransfer, const FileTransferRequest & request, - Callback<FileTransferResult> && callback) + Callback<FileTransferResult> && callback, + std::function<void(TransferItem &, std::string_view data)> dataCallback) : fileTransfer(fileTransfer) , request(request) , act(*logger, lvlTalkative, actFileTransfer, fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), {request.uri}, request.parentAct) , callback(std::move(callback)) - , finalSink([this](std::string_view data) { - auto httpStatus = getHTTPStatus(); - /* Only write data to the sink if this is a - successful response. */ - if (successfulStatuses.count(httpStatus) && this->request.dataCallback) { - writtenToSink += data.size(); - this->request.dataCallback(data); - } else - this->result.data.append(data); - }) + , dataCallback(std::move(dataCallback)) { requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) @@ -139,9 +132,6 @@ struct curlFileTransfer : public FileTransfer failEx(std::make_exception_ptr(std::forward<T>(e))); } - LambdaSink finalSink; - std::shared_ptr<FinishSink> decompressionSink; - std::exception_ptr writeException; std::optional<std::string> getHeader(const char * name) @@ -174,12 +164,13 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; result.bodySize += realSize; - if (!decompressionSink) { - decompressionSink = makeDecompressionSink(encoding, finalSink); + if (successfulStatuses.count(getHTTPStatus()) && this->dataCallback) { + writtenToSink += realSize; + dataCallback(*this, {(const char *) contents, realSize}); + } else { + this->result.data.append((const char *) contents, realSize); } - (*decompressionSink)({(char *) contents, realSize}); - return realSize; } catch (...) { writeException = std::current_exception(); @@ -342,14 +333,6 @@ struct curlFileTransfer : public FileTransfer debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); - if (decompressionSink) { - try { - decompressionSink->finish(); - } catch (...) { - writeException = std::current_exception(); - } - } - auto link = getHeader("link"); if (!link) { link = getHeader("x-amz-meta-link"); @@ -369,6 +352,14 @@ struct curlFileTransfer : public FileTransfer result.etag = std::move(*etag); } + // this has to happen here until we can return an actual future. + // wrapping user `callback`s instead is not possible because the + // Callback api expects std::functions, and copying Callbacks is + // not possible due the promises they hold. + if (code == CURLE_OK && !dataCallback) { + result.data = decompress(encoding, result.data); + } + if (writeException) failEx(writeException); @@ -455,7 +446,7 @@ struct curlFileTransfer : public FileTransfer ranged requests. */ if (err == Transient && attempt < request.tries - && (!this->request.dataCallback + && (!this->dataCallback || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { @@ -666,6 +657,13 @@ struct curlFileTransfer : public FileTransfer void enqueueFileTransfer(const FileTransferRequest & request, Callback<FileTransferResult> callback) override { + enqueueFileTransfer(request, std::move(callback), {}); + } + + void enqueueFileTransfer(const FileTransferRequest & request, + Callback<FileTransferResult> callback, + std::function<void(TransferItem &, std::string_view data)> dataCallback) + { /* Ugly hack to support s3:// URIs. */ if (request.uri.starts_with("s3://")) { // FIXME: do this on a worker thread @@ -694,7 +692,116 @@ struct curlFileTransfer : public FileTransfer return; } - enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback))); + enqueueItem(std::make_shared<TransferItem>( + *this, request, std::move(callback), std::move(dataCallback) + )); + } + + void download(FileTransferRequest && request, Sink & sink) override + { + /* Note: we can't call 'sink' via request.dataCallback, because + that would cause the sink to execute on the fileTransfer + thread. If 'sink' is a coroutine, this will fail. Also, if the + sink is expensive (e.g. one that does decompression and writing + to the Nix store), it would stall the download thread too much. + Therefore we use a buffer to communicate data between the + download thread and the calling thread. */ + + struct State { + bool quit = false; + std::exception_ptr exc; + std::string data; + std::condition_variable avail, request; + std::unique_ptr<FinishSink> decompressor; + }; + + auto _state = std::make_shared<Sync<State>>(); + + /* In case of an exception, wake up the download thread. FIXME: + abort the download request. */ + Finally finally([&]() { + auto state(_state->lock()); + state->quit = true; + state->request.notify_one(); + }); + + enqueueFileTransfer(request, + {[_state](std::future<FileTransferResult> fut) { + auto state(_state->lock()); + state->quit = true; + try { + fut.get(); + } catch (...) { + state->exc = std::current_exception(); + } + state->avail.notify_one(); + state->request.notify_one(); + }}, + [_state, &sink](TransferItem & transfer, std::string_view data) { + auto state(_state->lock()); + + if (state->quit) return; + + if (!state->decompressor) { + state->decompressor = makeDecompressionSink(transfer.encoding, sink); + } + + /* If the buffer is full, then go to sleep until the calling + thread wakes us up (i.e. when it has removed data from the + buffer). We don't wait forever to prevent stalling the + download thread. (Hopefully sleeping will throttle the + sender.) */ + if (state->data.size() > 1024 * 1024) { + debug("download buffer is full; going to sleep"); + state.wait_for(state->request, std::chrono::seconds(10)); + } + + /* Append data to the buffer and wake up the calling + thread. */ + state->data.append(data); + state->avail.notify_one(); + }); + + while (true) { + checkInterrupt(); + + std::string chunk; + FinishSink * sink = nullptr; + + /* Grab data if available, otherwise wait for the download + thread to wake us up. */ + { + auto state(_state->lock()); + + if (state->data.empty()) { + + if (state->quit) { + if (state->exc) std::rethrow_exception(state->exc); + if (state->decompressor) { + state->decompressor->finish(); + } + return; + } + + state.wait(state->avail); + + if (state->data.empty()) continue; + } + + chunk = std::move(state->data); + sink = state->decompressor.get(); + /* Reset state->data after the move, since we check data.empty() */ + state->data = ""; + + state->request.notify_one(); + } + + /* Flush the data to the sink and wake up the download thread + if it's blocked on a full buffer. We don't hold the state + lock while doing this to prevent blocking the download + thread if sink() takes a long time. */ + (*sink)(chunk); + } } }; @@ -743,105 +850,6 @@ FileTransferResult FileTransfer::upload(const FileTransferRequest & request) return enqueueFileTransfer(request).get(); } -void FileTransfer::download(FileTransferRequest && request, Sink & sink) -{ - /* Note: we can't call 'sink' via request.dataCallback, because - that would cause the sink to execute on the fileTransfer - thread. If 'sink' is a coroutine, this will fail. Also, if the - sink is expensive (e.g. one that does decompression and writing - to the Nix store), it would stall the download thread too much. - Therefore we use a buffer to communicate data between the - download thread and the calling thread. */ - - struct State { - bool quit = false; - std::exception_ptr exc; - std::string data; - std::condition_variable avail, request; - }; - - auto _state = std::make_shared<Sync<State>>(); - - /* In case of an exception, wake up the download thread. FIXME: - abort the download request. */ - Finally finally([&]() { - auto state(_state->lock()); - state->quit = true; - state->request.notify_one(); - }); - - request.dataCallback = [_state](std::string_view data) { - - auto state(_state->lock()); - - if (state->quit) return; - - /* If the buffer is full, then go to sleep until the calling - thread wakes us up (i.e. when it has removed data from the - buffer). We don't wait forever to prevent stalling the - download thread. (Hopefully sleeping will throttle the - sender.) */ - if (state->data.size() > 1024 * 1024) { - debug("download buffer is full; going to sleep"); - state.wait_for(state->request, std::chrono::seconds(10)); - } - - /* Append data to the buffer and wake up the calling - thread. */ - state->data.append(data); - state->avail.notify_one(); - }; - - enqueueFileTransfer(request, - {[_state](std::future<FileTransferResult> fut) { - auto state(_state->lock()); - state->quit = true; - try { - fut.get(); - } catch (...) { - state->exc = std::current_exception(); - } - state->avail.notify_one(); - state->request.notify_one(); - }}); - - while (true) { - checkInterrupt(); - - std::string chunk; - - /* Grab data if available, otherwise wait for the download - thread to wake us up. */ - { - auto state(_state->lock()); - - if (state->data.empty()) { - - if (state->quit) { - if (state->exc) std::rethrow_exception(state->exc); - return; - } - - state.wait(state->avail); - - if (state->data.empty()) continue; - } - - chunk = std::move(state->data); - /* Reset state->data after the move, since we check data.empty() */ - state->data = ""; - - state->request.notify_one(); - } - - /* Flush the data to the sink and wake up the download thread - if it's blocked on a full buffer. We don't hold the state - lock while doing this to prevent blocking the download - thread if sink() takes a long time. */ - sink(chunk); - } -} - template<typename... Args> FileTransferError::FileTransferError(FileTransfer::Error error, std::optional<std::string> response, const Args & ... args) : Error(args...), error(error), response(response) diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 6c11c14ee..e028d7f70 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -61,7 +61,6 @@ struct FileTransferRequest ActivityId parentAct; std::optional<std::string> data; std::string mimeType; - std::function<void(std::string_view data)> dataCallback; FileTransferRequest(std::string_view uri) : uri(uri), parentAct(getCurActivity()) { } @@ -115,7 +114,7 @@ struct FileTransfer * Download a file, writing its data to a sink. The sink will be * invoked on the thread of the caller. */ - void download(FileTransferRequest && request, Sink & sink); + virtual void download(FileTransferRequest && request, Sink & sink) = 0; enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 6bd73965d..078a63c83 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -7,6 +7,8 @@ libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) ifdef HOST_LINUX libstore_SOURCES += $(d)/platform/linux.cc +else ifdef HOST_DARWIN +libstore_SOURCES += $(d)/platform/darwin.cc else libstore_SOURCES += $(d)/platform/fallback.cc endif diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index ecae3054e..700c9b3dd 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -69,10 +69,10 @@ ref<Store> Machine::openStore() const Store::Params storeParams; if (storeUri.starts_with("ssh://")) { storeParams["max-connections"] = "1"; - storeParams["log-fd"] = "4"; } if (storeUri.starts_with("ssh://") || storeUri.starts_with("ssh-ng://")) { + storeParams["log-fd"] = "4"; if (sshKey != "") storeParams["ssh-key"] = sshKey; if (sshPublicHostKey != "") diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 94471dc29..5fde92dd0 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -162,6 +162,9 @@ libstore_headers = files( if host_machine.system() == 'linux' libstore_sources += files('platform/linux.cc') libstore_headers += files('platform/linux.hh') +elif host_machine.system() == 'darwin' + libstore_sources += files('platform/darwin.cc') + libstore_headers += files('platform/darwin.hh') else libstore_sources += files('platform/fallback.cc') libstore_headers += files('platform/fallback.hh') diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index a44e6a2eb..56c2cfc1d 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -3,6 +3,11 @@ namespace nix { -static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-_\?=][0-9a-zA-Z\+\-\._\?=]*)"; + +static constexpr std::string_view nameRegexStr = + // This uses a negative lookahead: (?!\.\.?(-|$)) + // - deny ".", "..", or those strings followed by '-' + // - when it's not those, start again at the start of the input and apply the next regex, which is [0-9a-zA-Z\+\-\._\?=]+ + R"((?!\.\.?(-|$))[0-9a-zA-Z\+\-\._\?=]+)"; } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 2f929b7b3..d029e986b 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -11,9 +11,20 @@ static void checkName(std::string_view path, std::string_view name) if (name.size() > StorePath::MaxPathLen) throw BadStorePath("store path '%s' has a name longer than %d characters", path, StorePath::MaxPathLen); - if (name[0] == '.') - throw BadStorePath("store path '%s' starts with illegal character '.'", path); // See nameRegexStr for the definition + if (name[0] == '.') { + // check against "." and "..", followed by end or dash + if (name.size() == 1) + throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + if (name[1] == '-') + throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, "."); + if (name[1] == '.') { + if (name.size() == 2) + throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + if (name[2] == '-') + throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, ".."); + } + } for (auto c : name) if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') diff --git a/src/libstore/platform.cc b/src/libstore/platform.cc index 9c389ef55..acdedab99 100644 --- a/src/libstore/platform.cc +++ b/src/libstore/platform.cc @@ -2,6 +2,8 @@ #if __linux__ #include "platform/linux.hh" +#elif __APPLE__ +#include "platform/darwin.hh" #else #include "platform/fallback.hh" #endif @@ -11,6 +13,8 @@ std::shared_ptr<LocalStore> LocalStore::makeLocalStore(const Params & params) { #if __linux__ return std::shared_ptr<LocalStore>(new LinuxLocalStore(params)); +#elif __APPLE__ + return std::shared_ptr<LocalStore>(new DarwinLocalStore(params)); #else return std::shared_ptr<LocalStore>(new FallbackLocalStore(params)); #endif diff --git a/src/libstore/platform/darwin.cc b/src/libstore/platform/darwin.cc new file mode 100644 index 000000000..bbb81784c --- /dev/null +++ b/src/libstore/platform/darwin.cc @@ -0,0 +1,223 @@ +#include "gc-store.hh" +#include "signals.hh" +#include "platform/darwin.hh" +#include "regex.hh" + +#include <sys/proc_info.h> +#include <sys/sysctl.h> +#include <libproc.h> + +#include <regex> + +namespace nix { + +void DarwinLocalStore::findPlatformRoots(UncheckedRoots & unchecked) +{ + auto storePathRegex = regex::storePathRegex(storeDir); + + std::vector<int> pids; + int pidBufSize = 1; + + while (pidBufSize > pids.size() * sizeof(int)) { + // Reserve some extra size so we don't fail too much + pids.resize((pidBufSize + pidBufSize / 8) / sizeof(int)); + pidBufSize = proc_listpids(PROC_ALL_PIDS, 0, pids.data(), pids.size() * sizeof(int)); + + if (pidBufSize <= 0) { + throw SysError("Listing PIDs"); + } + } + + pids.resize(pidBufSize / sizeof(int)); + + for (auto pid : pids) { + // It doesn't make sense to ask about the kernel + if (pid == 0) { + continue; + } + + try { + // Process cwd/root directory + struct proc_vnodepathinfo vnodeInfo; + if (proc_pidinfo(pid, PROC_PIDVNODEPATHINFO, 0, &vnodeInfo, sizeof(vnodeInfo)) <= 0) { + throw SysError("Getting pid %1% working directory", pid); + } + + unchecked[std::string(vnodeInfo.pvi_cdir.vip_path)].emplace(fmt("{libproc/%d/cwd}", pid) + ); + unchecked[std::string(vnodeInfo.pvi_rdir.vip_path)].emplace( + fmt("{libproc/%d/rootdir}", pid) + ); + + // File descriptors + std::vector<struct proc_fdinfo> fds; + int fdBufSize = 1; + while (fdBufSize > fds.size() * sizeof(struct proc_fdinfo)) { + // Reserve some extra size so we don't fail too much + fds.resize((fdBufSize + fdBufSize / 8) / sizeof(struct proc_fdinfo)); + fdBufSize = proc_pidinfo( + pid, PROC_PIDLISTFDS, 0, fds.data(), fds.size() * sizeof(struct proc_fdinfo) + ); + + if (fdBufSize <= 0) { + throw SysError("Listing pid %1% file descriptors", pid); + } + } + fds.resize(fdBufSize / sizeof(struct proc_fdinfo)); + + for (auto fd : fds) { + // By definition, only a vnode is on the filesystem + if (fd.proc_fdtype != PROX_FDTYPE_VNODE) { + continue; + } + + struct vnode_fdinfowithpath fdInfo; + if (proc_pidfdinfo( + pid, fd.proc_fd, PROC_PIDFDVNODEPATHINFO, &fdInfo, sizeof(fdInfo) + ) + <= 0) + { + // They probably just closed this fd, no need to cancel looking at ranges and + // arguments + if (errno == EBADF) { + continue; + } + throw SysError("Getting pid %1% fd %2% path", pid, fd.proc_fd); + } + + unchecked[std::string(fdInfo.pvip.vip_path)].emplace( + fmt("{libproc/%d/fd/%d}", pid, fd.proc_fd) + ); + } + + // Regions (e.g. mmapped files, executables, shared libraries) + uint64_t nextAddr = 0; + while (true) { + // Seriously, what are you doing XNU? + // There's 3 flavors of PROC_PIDREGIONPATHINFO: + // * PROC_PIDREGIONPATHINFO includes all regions + // * PROC_PIDREGIONPATHINFO2 includes regions backed by a vnode + // * PROC_PIDREGIONPATHINFO3 includes regions backed by a vnode on a specified + // filesystem Only PROC_PIDREGIONPATHINFO is documented. Unfortunately, using it + // would make finding gcroots take about 100x as long and tests would fail from + // timeout. According to the Frida source code, PROC_PIDREGIONPATHINFO2 has been + // available since XNU 2782.1.97 in OS X 10.10 + // + // 22 means PROC_PIDREGIONPATHINFO2 + struct proc_regionwithpathinfo regionInfo; + if (proc_pidinfo(pid, 22, nextAddr, ®ionInfo, sizeof(regionInfo)) <= 0) { + // PROC_PIDREGIONPATHINFO signals we're done with an error, + // so we're expected to hit this once per process + if (errno == ESRCH || errno == EINVAL) { + break; + } + throw SysError("Getting pid %1% region path", pid); + } + + unchecked[std::string(regionInfo.prp_vip.vip_path)].emplace( + fmt("{libproc/%d/region}", pid) + ); + + nextAddr = regionInfo.prp_prinfo.pri_address + regionInfo.prp_prinfo.pri_size; + } + + // Arguments and environment variables + // We can't read environment variables of binaries with entitlements unless + // nix has the `com.apple.private.read-environment-variables` entitlement or SIP is off + // We can read arguments for all applications though. + + // Yes, it's a sysctl, the proc_info and sysctl APIs are mostly similar, + // but both have exclusive capabilities + int sysctlName[3] = {CTL_KERN, KERN_PROCARGS2, pid}; + size_t argsSize = 0; + if (sysctl(sysctlName, 3, nullptr, &argsSize, nullptr, 0) < 0) { + throw SysError("Reading pid %1% arguments", pid); + } + + std::vector<char> args(argsSize); + if (sysctl(sysctlName, 3, args.data(), &argsSize, nullptr, 0) < 0) { + throw SysError("Reading pid %1% arguments", pid); + } + + if (argsSize < args.size()) { + args.resize(argsSize); + } + + // We have these perfectly nice arguments, but have to ignore them because + // otherwise we'd see arguments to nix-store commands and + // `nix-store --delete /nix/store/whatever` would always fail + // First 4 bytes are an int of argc. + if (args.size() < sizeof(int)) { + continue; + } + auto argc = reinterpret_cast<int *>(args.data())[0]; + + auto argsIter = args.begin(); + std::advance(argsIter, sizeof(int)); + // Executable then argc args, each separated by some number of null bytes + for (int i = 0; argsIter != args.end() && i < argc + 1; i++) { + argsIter = std::find(argsIter, args.end(), '\0'); + argsIter = std::find_if(argsIter, args.end(), [](char ch) { return ch != '\0'; }); + } + + if (argsIter != args.end()) { + auto env_end = std::sregex_iterator{}; + for (auto i = std::sregex_iterator{argsIter, args.end(), storePathRegex}; + i != env_end; + ++i) + { + unchecked[i->str()].emplace(fmt("{libproc/%d/environ}", pid)); + } + }; + + // Per-thread working directories + struct proc_taskallinfo taskAllInfo; + if (proc_pidinfo(pid, PROC_PIDTASKALLINFO, 0, &taskAllInfo, sizeof(taskAllInfo)) <= 0) { + throw SysError("Reading pid %1% tasks", pid); + } + + // If the process doesn't have the per-thread cwd flag then we already have the + // process-wide cwd from PROC_PIDVNODEPATHINFO + if (taskAllInfo.pbsd.pbi_flags & PROC_FLAG_THCWD) { + std::vector<uint64_t> tids(taskAllInfo.ptinfo.pti_threadnum); + int tidBufSize = proc_pidinfo( + pid, PROC_PIDLISTTHREADS, 0, tids.data(), tids.size() * sizeof(uint64_t) + ); + if (tidBufSize <= 0) { + throw SysError("Listing pid %1% threads", pid); + } + + for (auto tid : tids) { + struct proc_threadwithpathinfo threadPathInfo; + if (proc_pidinfo( + pid, + PROC_PIDTHREADPATHINFO, + tid, + &threadPathInfo, + sizeof(threadPathInfo) + ) + <= 0) + { + throw SysError("Reading pid %1% thread %2% cwd", pid, tid); + } + + unchecked[std::string(threadPathInfo.pvip.vip_path)].emplace( + fmt("{libproc/%d/thread/%d/cwd}", pid, tid) + ); + } + } + } catch (SysError & e) { + // ENOENT/ESRCH: Process no longer exists (proc_info) + // EINVAL: Process no longer exists (sysctl) + // EACCESS/EPERM: We don't have permission to read this field (proc_info) + // EIO: Kernel failed to read from target process memory during KERN_PROCARGS2 (sysctl) + if (errno == ENOENT || errno == ESRCH || errno == EINVAL || errno == EACCES + || errno == EPERM || errno == EIO) + { + continue; + } + throw; + } + } +} +} diff --git a/src/libstore/platform/darwin.hh b/src/libstore/platform/darwin.hh new file mode 100644 index 000000000..b7170aa05 --- /dev/null +++ b/src/libstore/platform/darwin.hh @@ -0,0 +1,35 @@ +#pragma once +///@file + +#include "gc-store.hh" +#include "local-store.hh" + +namespace nix { + +/** + * Darwin-specific implementation of LocalStore + */ +class DarwinLocalStore : public LocalStore +{ +public: + DarwinLocalStore(const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) + , Store(params) + , LocalFSStore(params) + , LocalStore(params) + { + } + DarwinLocalStore(const std::string scheme, std::string path, const Params & params) + : DarwinLocalStore(params) + { + throw UnimplementedError("DarwinLocalStore"); + } + +private: + + void findPlatformRoots(UncheckedRoots & unchecked) override; +}; + +} diff --git a/src/libstore/platform/linux.cc b/src/libstore/platform/linux.cc index 9be3e47da..a34608894 100644 --- a/src/libstore/platform/linux.cc +++ b/src/libstore/platform/linux.cc @@ -1,6 +1,7 @@ #include "gc-store.hh" #include "signals.hh" #include "platform/linux.hh" +#include "regex.hh" #include <regex> @@ -26,12 +27,6 @@ static void readProcLink(const std::string & file, UncheckedRoots & roots) } } -static std::string quoteRegexChars(const std::string & raw) -{ - static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])"); - return std::regex_replace(raw, specialRegex, R"(\$&)"); -} - static void readFileRoots(const char * path, UncheckedRoots & roots) { try { @@ -50,8 +45,7 @@ void LinuxLocalStore::findPlatformRoots(UncheckedRoots & unchecked) struct dirent * ent; auto digitsRegex = std::regex(R"(^\d+$)"); auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); - auto storePathRegex = - std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); + auto storePathRegex = regex::storePathRegex(storeDir); while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (std::regex_match(ent->d_name, digitsRegex)) { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 4a6aad449..80d10eb0f 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -32,6 +32,10 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { public: + // Hack for getting remote build log output. + // Intentionally not in `SSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) @@ -47,7 +51,8 @@ public: sshPublicHostKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - compress) + compress, + logFD) { } diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 7aac026c9..2d2c24afa 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -88,8 +88,6 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string addCommonSSHOpts(args); if (socketPath != "") args.insert(args.end(), {"-S", socketPath}); - if (verbosity >= lvlChatty) - args.push_back("-v"); } args.push_back(command); @@ -154,8 +152,6 @@ Path SSHMaster::startMaster() throw SysError("duping over stdout"); Strings args = { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath }; - if (verbosity >= lvlChatty) - args.push_back("-v"); addCommonSSHOpts(args); execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 69e89263b..8c9940c86 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1085,8 +1085,6 @@ void copyStorePath( auto info = srcStore.queryPathInfo(storePath); - uint64_t total = 0; - // recompute store path on the chance dstStore does it differently if (info->ca && info->references.empty()) { auto info2 = make_ref<ValidPathInfo>(*info); @@ -1105,7 +1103,7 @@ void copyStorePath( } auto source = sinkToSource([&](Sink & sink) { - LambdaSink progressSink([&](std::string_view data) { + LambdaSink progressSink([&, total = 0ULL](std::string_view data) mutable { total += data.size(); act.progress(total, info->narSize); }); @@ -1218,9 +1216,6 @@ std::map<StorePath, StorePath> copyPaths( return storePathForDst; }; - // total is accessed by each copy, which are each handled in separate threads - std::atomic<uint64_t> total = 0; - for (auto & missingPath : sortedMissing) { auto info = srcStore.queryPathInfo(missingPath); @@ -1241,7 +1236,7 @@ std::map<StorePath, StorePath> copyPaths( {storePathS, srcUri, dstUri}); PushActivity pact(act.id); - LambdaSink progressSink([&](std::string_view data) { + LambdaSink progressSink([&, total = 0ULL](std::string_view data) mutable { total += data.size(); act.progress(total, info->narSize); }); diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 11bf97ee7..8caa0532a 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -22,6 +22,7 @@ libutil_sources = files( 'position.cc', 'print-elided.cc', 'references.cc', + 'regex.cc', 'serialise.cc', 'shlex.cc', 'signals.cc', @@ -30,6 +31,7 @@ libutil_sources = files( 'tarfile.cc', 'thread-pool.cc', 'url.cc', + 'url-name.cc', 'util.cc', 'xml-writer.cc', ) @@ -77,6 +79,7 @@ libutil_headers = files( 'ref.hh', 'references.hh', 'regex-combinators.hh', + 'regex.hh', 'repair-flag.hh', 'serialise.hh', 'shlex.hh', @@ -90,6 +93,7 @@ libutil_headers = files( 'topo-sort.hh', 'types.hh', 'url-parts.hh', + 'url-name.hh', 'url.hh', 'util.hh', 'variant-wrapper.hh', diff --git a/src/libutil/regex.cc b/src/libutil/regex.cc new file mode 100644 index 000000000..a9e6c6bee --- /dev/null +++ b/src/libutil/regex.cc @@ -0,0 +1,16 @@ +#include <string> +#include <regex> + +namespace nix::regex { +std::string quoteRegexChars(const std::string & raw) +{ + static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])"); + return std::regex_replace(raw, specialRegex, R"(\$&)"); +} + +std::regex storePathRegex(const std::string & storeDir) +{ + return std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); +} + +} diff --git a/src/libutil/regex.hh b/src/libutil/regex.hh new file mode 100644 index 000000000..744a7d54a --- /dev/null +++ b/src/libutil/regex.hh @@ -0,0 +1,11 @@ +#pragma once +///@file + +#include <string> +#include <regex> + +namespace nix::regex { +std::string quoteRegexChars(const std::string & raw); + +std::regex storePathRegex(const std::string & storeDir); +} diff --git a/src/libutil/url-name.cc b/src/libutil/url-name.cc new file mode 100644 index 000000000..7c526752c --- /dev/null +++ b/src/libutil/url-name.cc @@ -0,0 +1,59 @@ +#include <iostream> +#include <regex> + +#include "url-name.hh" + +namespace nix { + +static std::string const attributeNamePattern("[a-zA-Z0-9_-]+"); +static std::regex const lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?"); +static std::string const pathSegmentPattern("[a-zA-Z0-9_-]+"); +static std::regex const lastPathSegmentRegex(".*/(" + pathSegmentPattern +")"); +static std::regex const secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?"); +static std::regex const gitProviderRegex("github|gitlab|sourcehut"); +static std::regex const gitSchemeRegex("git($|\\+.*)"); +static std::regex const defaultOutputRegex(".*\\.default($|\\^.*)"); + +std::optional<std::string> getNameFromURL(ParsedURL const & url) +{ + std::smatch match; + + /* If there is a dir= argument, use its value */ + if (url.query.count("dir") > 0) { + return url.query.at("dir"); + } + + /* If the fragment isn't a "default" and contains two attribute elements, use the last one */ + if (std::regex_match(url.fragment, match, lastAttributeRegex)) { + return match.str(1); + } + + /* If this is a github/gitlab/sourcehut flake, use the repo name */ + if ( + std::regex_match(url.scheme, gitProviderRegex) + && std::regex_match(url.path, match, secondPathSegmentRegex) + ) { + return match.str(1); + } + + /* If it is a regular git flake, use the directory name */ + if ( + std::regex_match(url.scheme, gitSchemeRegex) + && std::regex_match(url.path, match, lastPathSegmentRegex) + ) { + return match.str(1); + } + + /* If everything failed but there is a non-default fragment, use it in full */ + if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex)) + return url.fragment; + + /* If there is no fragment, take the last element of the path */ + if (std::regex_match(url.path, match, lastPathSegmentRegex)) + return match.str(1); + + /* If even that didn't work, the URL does not contain enough info to determine a useful name */ + return {}; +} + +} diff --git a/src/libutil/url-name.hh b/src/libutil/url-name.hh new file mode 100644 index 000000000..3a3f88e76 --- /dev/null +++ b/src/libutil/url-name.hh @@ -0,0 +1,26 @@ +#pragma once +///@file url-name.hh, for some hueristic-ish URL parsing. + +#include <string> +#include <optional> + +#include "url.hh" +#include "url-parts.hh" +#include "util.hh" +#include "split.hh" + +namespace nix { + +/** + * Try to extract a reasonably unique and meaningful, human-readable + * name of a flake output from a parsed URL. + * When nullopt is returned, the callsite should use information available + * to it outside of the URL to determine a useful name. + * This is a heuristic approach intended for user interfaces. + * @return nullopt if the extracted name is not useful to identify a + * flake output, for example because it is empty or "default". + * Otherwise returns the extracted name. + */ +std::optional<std::string> getNameFromURL(ParsedURL const & url); + +} diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 6255c1d02..6efcc7e50 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -19,6 +19,7 @@ const static std::string userRegex = "(?:(?:" + unreservedRegex + "|" + pctEncod const static std::string authorityRegex = "(?:" + userRegex + "@)?" + hostRegex + "(?::[0-9]+)?"; const static std::string pcharRegex = "(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + "|[:@])"; const static std::string queryRegex = "(?:" + pcharRegex + "|[/? \"])*"; +const static std::string fragmentRegex = "(?:" + pcharRegex + "|[/? \"^])*"; const static std::string segmentRegex = "(?:" + pcharRegex + "*)"; const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index a8f7d39fd..afccc4245 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -16,7 +16,7 @@ ParsedURL parseURL(const std::string & url) "((" + schemeRegex + "):" + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", + + "(?:#(" + fragmentRegex + "))?", std::regex::ECMAScript); std::smatch match; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index dc724db3e..bc2dd1802 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1385,13 +1385,31 @@ std::string replaceStrings( } -std::string rewriteStrings(std::string s, const StringMap & rewrites) +Rewriter::Rewriter(std::map<std::string, std::string> rewrites) + : rewrites(std::move(rewrites)) { - for (auto & i : rewrites) { - if (i.first == i.second) continue; - size_t j = 0; - while ((j = s.find(i.first, j)) != std::string::npos) - s.replace(j, i.first.size(), i.second); + for (const auto & [k, v] : this->rewrites) { + assert(!k.empty()); + initials.push_back(k[0]); + } + std::ranges::sort(initials); + auto [firstDupe, end] = std::ranges::unique(initials); + initials.erase(firstDupe, end); +} + +std::string Rewriter::operator()(std::string s) +{ + size_t j = 0; + while ((j = s.find_first_of(initials, j)) != std::string::npos) { + size_t skip = 1; + for (auto & [from, to] : rewrites) { + if (s.compare(j, from.size(), from) == 0) { + s.replace(j, from.size(), to); + skip = to.size(); + break; + } + } + j += skip; } return s; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 9c2385e84..ac4aa1d3a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -604,7 +604,33 @@ std::string replaceStrings( std::string_view to); -std::string rewriteStrings(std::string s, const StringMap & rewrites); +/** + * Rewrites a string given a map of replacements, applying the replacements in + * sorted order, only once, considering only the strings appearing in the input + * string in performing replacement. + * + * - Replacements are not performed on intermediate strings. That is, for an input + * `"abb"` with replacements `{"ab" -> "ba"}`, the result is `"bab"`. + * - Transitive replacements are not performed. For example, for the input `"abcde"` + * with replacements `{"a" -> "b", "b" -> "c", "e" -> "b"}`, the result is + * `"bccdb"`. + */ +class Rewriter +{ +private: + std::string initials; + std::map<std::string, std::string> rewrites; + +public: + explicit Rewriter(std::map<std::string, std::string> rewrites); + + std::string operator()(std::string s); +}; + +inline std::string rewriteStrings(std::string s, const StringMap & rewrites) +{ + return Rewriter(rewrites)(s); +} /** diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 757938914..f0131a458 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -16,22 +16,6 @@ namespace nix { -DrvInfos queryInstalled(EvalState & state, const Path & userEnv) -{ - DrvInfos elems; - if (pathExists(userEnv + "/manifest.json")) - throw Error("profile '%s' is incompatible with 'nix-env'; please use 'nix profile' instead", userEnv); - auto manifestFile = userEnv + "/manifest.nix"; - if (pathExists(manifestFile)) { - Value v; - state.evalFile(state.rootPath(CanonPath(manifestFile)), v); - Bindings & bindings(*state.allocBindings(0)); - getDerivations(state, v, "", bindings, elems, false); - } - return elems; -} - - bool createUserEnv(EvalState & state, DrvInfos & elems, const Path & profile, bool keepDerivations, const std::string & lockToken) diff --git a/src/nix/build.md b/src/nix/build.md index 0fbb39cc3..2435c1ef6 100644 --- a/src/nix/build.md +++ b/src/nix/build.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Build the default package from the flake in the current directory: diff --git a/src/nix/bundle.md b/src/nix/bundle.md index 89458aaaa..156f040ba 100644 --- a/src/nix/bundle.md +++ b/src/nix/bundle.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Bundle Hello: diff --git a/src/nix/show-config.cc b/src/nix/config.cc index 3530584f9..5b280d11d 100644 --- a/src/nix/show-config.cc +++ b/src/nix/config.cc @@ -7,11 +7,31 @@ using namespace nix; -struct CmdShowConfig : Command, MixJSON +struct CmdConfig : virtual NixMultiCommand +{ + CmdConfig() : MultiCommand(RegisterCommand::getCommandsFor({"config"})) + { } + + std::string description() override + { + return "manipulate the Nix configuration"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix config' requires a sub-command."); + command->second->run(); + } +}; + +struct CmdConfigShow : Command, MixJSON { std::optional<std::string> name; - CmdShowConfig() { + CmdConfigShow() { expectArgs({ .label = {"name"}, .optional = true, @@ -56,4 +76,5 @@ struct CmdShowConfig : Command, MixJSON } }; -static auto rShowConfig = registerCommand<CmdShowConfig>("show-config"); +static auto rCmdConfig = registerCommand<CmdConfig>("config"); +static auto rShowConfig = registerCommand2<CmdConfigShow>({"config", "show"}); diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md index f116681ab..ba75b402a 100644 --- a/src/nix/derivation-add.md +++ b/src/nix/derivation-add.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Description This command reads from standard input a JSON representation of a diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md index 1296e2885..f644a429d 100644 --- a/src/nix/derivation-show.md +++ b/src/nix/derivation-show.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Show the [store derivation] that results from evaluating the Hello diff --git a/src/nix/develop.md b/src/nix/develop.md index c49b39669..e39048bb1 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Start a shell with the build environment of the default package of diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index c7c37b66f..736cbf55d 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "cmd-profiles.hh" #include "shared.hh" #include "store-api.hh" #include "common-args.hh" @@ -43,15 +44,6 @@ GroupedPaths getClosureInfo(ref<Store> store, const StorePath & toplevel) return groupedPaths; } -std::string showVersions(const std::set<std::string> & versions) -{ - if (versions.empty()) return "∅"; - std::set<std::string> versions2; - for (auto & version : versions) - versions2.insert(version.empty() ? "ε" : version); - return concatStringsSep(", ", versions2); -} - void printClosureDiff( ref<Store> store, const StorePath & beforePath, diff --git a/src/nix/diff-closures.md b/src/nix/diff-closures.md index 0294c0d8d..5f81aa4e2 100644 --- a/src/nix/diff-closures.md +++ b/src/nix/diff-closures.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Show what got added and removed between two versions of the NixOS diff --git a/src/nix/edit.md b/src/nix/edit.md index 89bd09abf..d120f3205 100644 --- a/src/nix/edit.md +++ b/src/nix/edit.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Open the Nix expression of the GNU Hello package: diff --git a/src/nix/eval.md b/src/nix/eval.md index d1daaf755..eb9f753c5 100644 --- a/src/nix/eval.md +++ b/src/nix/eval.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Evaluate a Nix expression given on the command line: diff --git a/src/nix/fmt.md b/src/nix/fmt.md index 1c78bb36f..8b90f33ef 100644 --- a/src/nix/fmt.md +++ b/src/nix/fmt.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples With [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt): diff --git a/src/nix/log.md b/src/nix/log.md index 01e9801df..81bfefd96 100644 --- a/src/nix/log.md +++ b/src/nix/log.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Get the build log of GNU Hello: diff --git a/src/nix/main.cc b/src/nix/main.cc index 6bc46eba3..64755d445 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -151,6 +151,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, {"show-derivation", {"derivation", "show"}}, + {"show-config", {"config", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, diff --git a/src/nix/meson.build b/src/nix/meson.build index cb8f73174..e41399b5d 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -59,7 +59,7 @@ nix_sources = files( 'repl.cc', 'run.cc', 'search.cc', - 'show-config.cc', + 'config.cc', 'sigs.cc', 'store-copy-log.cc', 'store-delete.cc', diff --git a/src/nix/nix.md b/src/nix/nix.md index e0f459d6b..0d588cd01 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -59,9 +59,13 @@ These are command line arguments that represent something that can be realised i The following types of installable are supported by most commands: - [Flake output attribute](#flake-output-attribute) (experimental) + - This is the default - [Store path](#store-path) + - This is assumed if the argument is a Nix store path or a symlink to a Nix store path - [Nix file](#nix-file), optionally qualified by an attribute path + - Specified with `--file`/`-f` - [Nix expression](#nix-expression), optionally qualified by an attribute path + - Specified with `--expr`/`-E` For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. @@ -158,16 +162,28 @@ When the option `-f` / `--file` *path* \[*attrpath*...\] is given, installables If attribute paths are provided, commands will operate on the corresponding values accessible at these paths. The Nix expression in that file, or any selected attribute, must evaluate to a derivation. +To emulate the `nix-build '<nixpkgs>' -A hello` pattern, use: + +```console +$ nix build -f '<nixpkgs>' hello +``` + ### Nix expression Example: `--expr 'import <nixpkgs> {}' hello` -When the option `--expr` *expression* \[*attrpath*...\] is given, installables are interpreted as the value of the of the Nix expression. +When the option `-E` / `--expr` *expression* \[*attrpath*...\] is given, installables are interpreted as the value of the of the Nix expression. If attribute paths are provided, commands will operate on the corresponding values accessible at these paths. The Nix expression, or any selected attribute, must evaluate to a derivation. You may need to specify `--impure` if the expression references impure inputs (such as `<nixpkgs>`). +To emulate the `nix-build -E 'with import <nixpkgs> { }; hello' pattern use: + +```console +$ nix build --impure -E 'with import <nixpkgs> { }; hello' +``` + ## Derivation output selection Derivations can have multiple outputs, each corresponding to a @@ -176,9 +192,10 @@ that contains programs, and a `dev` output that provides development artifacts like C/C++ header files. The outputs on which `nix` commands operate are determined as follows: -* You can explicitly specify the desired outputs using the syntax - *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can - obtain the `dev` and `static` outputs of the `glibc` package: +* You can explicitly specify the desired outputs using the syntax *installable*`^`*output1*`,`*...*`,`*outputN* — that is, a caret followed immediately by a comma-separated list of derivation outputs to select. + For installables specified as [Flake output attributes](#flake-output-attribute) or [Store paths](#store-path), the output is specified in the same argument: + + For example, you can obtain the `dev` and `static` outputs of the `glibc` package: ```console # nix build 'nixpkgs#glibc^dev,static' @@ -193,6 +210,19 @@ operate are determined as follows: … ``` + For `-e`/`--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: + + ```console + $ nix build -f '<nixpkgs>' 'glibc^dev,static' + $ nix build --impure -E 'import <nixpkgs> { }' 'glibc^dev,static' + ``` + + This syntax is the same even if the actual attribute path is empty: + + ```console + $ nix build -E 'let pkgs = import <nixpkgs> { }; in pkgs.glibc' '^dev,static' + ``` + * You can also specify that *all* outputs should be used using the syntax *installable*`^*`. For example, the following shows the size of all outputs of the `glibc` package in the binary cache: diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 2dda866d0..5f31215d7 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Print the store path produced by `nixpkgs#hello`: diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md index a8ce9d36a..a7b1cc2b6 100644 --- a/src/nix/print-dev-env.md +++ b/src/nix/print-dev-env.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Apply the build environment of GNU hello to the current shell: diff --git a/src/nix/profile-install.md b/src/nix/profile-install.md index 4c0f82c09..8fe31ac4d 100644 --- a/src/nix/profile-install.md +++ b/src/nix/profile-install.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Install a package from Nixpkgs: diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md index 5d7fcc0ec..9baea4ada 100644 --- a/src/nix/profile-list.md +++ b/src/nix/profile-list.md @@ -6,13 +6,13 @@ R""( ```console # nix profile list - Index: 0 + Name: gdb Flake attribute: legacyPackages.x86_64-linux.gdb Original flake URL: flake:nixpkgs Locked flake URL: github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca Store paths: /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1 - Index: 1 + Name: blender-bin Flake attribute: packages.x86_64-linux.default Original flake URL: flake:blender-bin Locked flake URL: github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender @@ -26,7 +26,7 @@ R""( # nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default ``` - will build the package with index 1 shown above. + will build the package with name blender-bin shown above. # Description @@ -34,7 +34,7 @@ This command shows what packages are currently installed in a profile. For each installed package, it shows the following information: -* `Index`: An integer that can be used to unambiguously identify the +* `Name`: A unique name used to unambiguously identify the package in invocations of `nix profile remove` and `nix profile upgrade`. diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md index ba85441d8..a001525ae 100644 --- a/src/nix/profile-remove.md +++ b/src/nix/profile-remove.md @@ -1,11 +1,13 @@ R""( +**Note**: unlike [`nix profile install`](./nix3-profile-install.md), this command does *not* take installables. + # Examples -* Remove a package by position: +* Remove a package by name: ```console - # nix profile remove 3 + # nix profile remove hello ``` * Remove a package by attribute path: diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index 39cca428b..3a141384d 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -1,5 +1,7 @@ R""( +**Note**: unlike [`nix profile install`](./nix3-profile-install.md), this command does *not* take installables. + # Examples * Upgrade all packages that were installed using an unlocked flake @@ -9,19 +11,14 @@ R""( # nix profile upgrade '.*' ``` -* Upgrade a specific package: +* Upgrade a specific package by name: ```console - # nix profile upgrade packages.x86_64-linux.hello + # nix profile upgrade hello ``` -* Upgrade a specific profile element by number: - ```console - # nix profile list - 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … - - # nix profile upgrade 0 + # nix profile upgrade packages.x86_64-linux.hello ``` # Description diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 476ddcd60..401d5bd77 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "cmd-profiles.hh" #include "installable-flake.hh" #include "common-args.hh" #include "shared.hh" @@ -17,269 +18,6 @@ using namespace nix; -struct ProfileElementSource -{ - FlakeRef originalRef; - // FIXME: record original attrpath. - FlakeRef lockedRef; - std::string attrPath; - ExtendedOutputsSpec outputs; - - bool operator < (const ProfileElementSource & other) const - { - return - std::tuple(originalRef.to_string(), attrPath, outputs) < - std::tuple(other.originalRef.to_string(), other.attrPath, other.outputs); - } - - std::string to_string() const - { - return fmt("%s#%s%s", originalRef, attrPath, outputs.to_string()); - } -}; - -const int defaultPriority = 5; - -struct ProfileElement -{ - StorePathSet storePaths; - std::optional<ProfileElementSource> source; - bool active = true; - int priority = defaultPriority; - - std::string identifier() const - { - if (source) - return source->to_string(); - StringSet names; - for (auto & path : storePaths) - names.insert(DrvName(path.name()).name); - return concatStringsSep(", ", names); - } - - /** - * Return a string representing an installable corresponding to the current - * element, either a flakeref or a plain store path - */ - std::set<std::string> toInstallables(Store & store) - { - if (source) - return {source->to_string()}; - StringSet rawPaths; - for (auto & path : storePaths) - rawPaths.insert(store.printStorePath(path)); - return rawPaths; - } - - std::string versions() const - { - StringSet versions; - for (auto & path : storePaths) - versions.insert(DrvName(path.name()).version); - return showVersions(versions); - } - - bool operator < (const ProfileElement & other) const - { - return std::tuple(identifier(), storePaths) < std::tuple(other.identifier(), other.storePaths); - } - - void updateStorePaths( - ref<Store> evalStore, - ref<Store> store, - const BuiltPaths & builtPaths) - { - storePaths.clear(); - for (auto & buildable : builtPaths) { - std::visit(overloaded { - [&](const BuiltPath::Opaque & bo) { - storePaths.insert(bo.path); - }, - [&](const BuiltPath::Built & bfd) { - for (auto & output : bfd.outputs) - storePaths.insert(output.second); - }, - }, buildable.raw()); - } - } -}; - -struct ProfileManifest -{ - std::vector<ProfileElement> elements; - - ProfileManifest() { } - - ProfileManifest(EvalState & state, const Path & profile) - { - auto manifestPath = profile + "/manifest.json"; - - if (pathExists(manifestPath)) { - auto json = nlohmann::json::parse(readFile(manifestPath)); - - auto version = json.value("version", 0); - std::string sUrl; - std::string sOriginalUrl; - switch (version) { - case 1: - sUrl = "uri"; - sOriginalUrl = "originalUri"; - break; - case 2: - sUrl = "url"; - sOriginalUrl = "originalUrl"; - break; - default: - throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); - } - - for (auto & e : json["elements"]) { - ProfileElement element; - for (auto & p : e["storePaths"]) - element.storePaths.insert(state.store->parseStorePath((std::string) p)); - element.active = e["active"]; - if(e.contains("priority")) { - element.priority = e["priority"]; - } - if (e.value(sUrl, "") != "") { - element.source = ProfileElementSource { - parseFlakeRef(e[sOriginalUrl]), - parseFlakeRef(e[sUrl]), - e["attrPath"], - e["outputs"].get<ExtendedOutputsSpec>() - }; - } - elements.emplace_back(std::move(element)); - } - } - - else if (pathExists(profile + "/manifest.nix")) { - // FIXME: needed because of pure mode; ugly. - state.allowPath(state.store->followLinksToStore(profile)); - state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix")); - - auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); - - for (auto & drvInfo : drvInfos) { - ProfileElement element; - element.storePaths = {drvInfo.queryOutPath()}; - elements.emplace_back(std::move(element)); - } - } - } - - nlohmann::json toJSON(Store & store) const - { - auto array = nlohmann::json::array(); - for (auto & element : elements) { - auto paths = nlohmann::json::array(); - for (auto & path : element.storePaths) - paths.push_back(store.printStorePath(path)); - nlohmann::json obj; - obj["storePaths"] = paths; - obj["active"] = element.active; - obj["priority"] = element.priority; - if (element.source) { - obj["originalUrl"] = element.source->originalRef.to_string(); - obj["url"] = element.source->lockedRef.to_string(); - obj["attrPath"] = element.source->attrPath; - obj["outputs"] = element.source->outputs; - } - array.push_back(obj); - } - nlohmann::json json; - json["version"] = 2; - json["elements"] = array; - return json; - } - - StorePath build(ref<Store> store) - { - auto tempDir = createTempDir(); - - StorePathSet references; - - Packages pkgs; - for (auto & element : elements) { - for (auto & path : element.storePaths) { - if (element.active) - pkgs.emplace_back(store->printStorePath(path), true, element.priority); - references.insert(path); - } - } - - buildProfile(tempDir, std::move(pkgs)); - - writeFile(tempDir + "/manifest.json", toJSON(*store).dump()); - - /* Add the symlink tree to the store. */ - StringSink sink; - dumpPath(tempDir, sink); - - auto narHash = hashString(htSHA256, sink.s); - - ValidPathInfo info { - *store, - "profile", - FixedOutputInfo { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - .references = { - .others = std::move(references), - // profiles never refer to themselves - .self = false, - }, - }, - narHash, - }; - info.narSize = sink.s.size(); - - StringSource source(sink.s); - store->addToStore(info, source); - - return std::move(info.path); - } - - static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent) - { - auto prevElems = prev.elements; - std::sort(prevElems.begin(), prevElems.end()); - - auto curElems = cur.elements; - std::sort(curElems.begin(), curElems.end()); - - auto i = prevElems.begin(); - auto j = curElems.begin(); - - bool changes = false; - - while (i != prevElems.end() || j != curElems.end()) { - if (j != curElems.end() && (i == prevElems.end() || i->identifier() > j->identifier())) { - logger->cout("%s%s: ∅ -> %s", indent, j->identifier(), j->versions()); - changes = true; - ++j; - } - else if (i != prevElems.end() && (j == curElems.end() || i->identifier() < j->identifier())) { - logger->cout("%s%s: %s -> ∅", indent, i->identifier(), i->versions()); - changes = true; - ++i; - } - else { - auto v1 = i->versions(); - auto v2 = j->versions(); - if (v1 != v2) { - logger->cout("%s%s: %s -> %s", indent, i->identifier(), v1, v2); - changes = true; - } - ++i; - ++j; - } - } - - if (!changes) - logger->cout("%sNo changes.", indent); - } -}; static std::map<Installable *, std::pair<BuiltPaths, ref<ExtraPathInfo>>> builtPathsPerInstallable( @@ -361,13 +99,13 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile : ({ auto * info2 = dynamic_cast<ExtraPathInfoValue *>(&*info); info2 - ? info2->value.priority.value_or(defaultPriority) - : defaultPriority; + ? info2->value.priority.value_or(DEFAULT_PRIORITY) + : DEFAULT_PRIORITY; }); element.updateStorePaths(getEvalStore(), store, res); - manifest.elements.push_back(std::move(element)); + manifest.addElement(std::move(element)); } try { @@ -377,7 +115,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile // See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102 auto findRefByFilePath = [&]<typename Iterator>(Iterator begin, Iterator end) { for (auto it = begin; it != end; it++) { - auto profileElement = *it; + auto & profileElement = it->second; for (auto & storePath : profileElement.storePaths) { if (conflictError.fileA.starts_with(store->printStorePath(storePath))) { return std::pair(conflictError.fileA, profileElement.toInstallables(*store)); @@ -445,35 +183,40 @@ public: std::string pattern; std::regex reg; }; - typedef std::variant<size_t, Path, RegexPattern> Matcher; + using Matcher = std::variant<Path, RegexPattern>; std::vector<Matcher> getMatchers(ref<Store> store) { std::vector<Matcher> res; for (auto & s : _matchers) { - if (auto n = string2Int<size_t>(s)) - res.push_back(*n); - else if (store->isStorePath(s)) + if (auto n = string2Int<size_t>(s)) { + throw Error("'nix profile' no longer supports indices ('%d')", *n); + } else if (store->isStorePath(s)) { res.push_back(s); - else + } else { res.push_back(RegexPattern{s,std::regex(s, std::regex::extended | std::regex::icase)}); + } } return res; } - bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector<Matcher> & matchers) + bool matches( + Store const & store, + // regex_match doesn't take a string_view lol + std::string const & name, + ProfileElement const & element, + std::vector<Matcher> const & matchers + ) { for (auto & matcher : matchers) { - if (auto n = std::get_if<size_t>(&matcher)) { - if (*n == pos) return true; - } else if (auto path = std::get_if<Path>(&matcher)) { + if (auto path = std::get_if<Path>(&matcher)) { if (element.storePaths.count(store.parseStorePath(*path))) return true; } else if (auto regex = std::get_if<RegexPattern>(&matcher)) { - if (element.source - && std::regex_match(element.source->attrPath, regex->reg)) + if (std::regex_match(name, regex->reg)) { return true; + } } } @@ -503,10 +246,9 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem ProfileManifest newManifest; - for (size_t i = 0; i < oldManifest.elements.size(); ++i) { - auto & element(oldManifest.elements[i]); - if (!matches(*store, element, i, matchers)) { - newManifest.elements.push_back(std::move(element)); + for (auto & [name, element] : oldManifest.elements) { + if (!matches(*store, name, element, matchers)) { + newManifest.elements.insert_or_assign(name, std::move(element)); } else { notice("removing '%s'", element.identifier()); } @@ -519,9 +261,7 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem if (removedCount == 0) { for (auto matcher: matchers) { - if (const size_t * index = std::get_if<size_t>(&matcher)){ - warn("'%d' is not a valid index", *index); - } else if (const Path * path = std::get_if<Path>(&matcher)){ + if (const Path * path = std::get_if<Path>(&matcher)) { warn("'%s' does not match any paths", *path); } else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)){ warn("'%s' does not match any packages", regex->pattern); @@ -554,64 +294,99 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf auto matchers = getMatchers(store); Installables installables; - std::vector<size_t> indices; + std::vector<ProfileElement *> elems; + auto matchedCount = 0; auto upgradedCount = 0; - for (size_t i = 0; i < manifest.elements.size(); ++i) { - auto & element(manifest.elements[i]); - if (element.source - && !element.source->originalRef.input.isLocked() - && matches(*store, element, i, matchers)) - { - upgradedCount++; + for (auto & [name, element] : manifest.elements) { + if (!matches(*store, name, element, matchers)) { + continue; + } - Activity act(*logger, lvlChatty, actUnknown, - fmt("checking '%s' for updates", element.source->attrPath)); + matchedCount += 1; - auto installable = make_ref<InstallableFlake>( - this, - getEvalState(), - FlakeRef(element.source->originalRef), - "", - element.source->outputs, - Strings{element.source->attrPath}, - Strings{}, - lockFlags); + if (!element.source) { + warn( + "Found package '%s', but it was not installed from a flake, so it can't be checked for upgrades", + element.identifier() + ); + continue; + } - auto derivedPaths = installable->toDerivedPaths(); - if (derivedPaths.empty()) continue; - auto * infop = dynamic_cast<ExtraPathInfoFlake *>(&*derivedPaths[0].info); - // `InstallableFlake` should use `ExtraPathInfoFlake`. - assert(infop); - auto & info = *infop; + if (element.source->originalRef.input.isLocked()) { + warn( + "Found package '%s', but it was installed from a locked flake reference so it can't be upgraded", + element.identifier() + ); + continue; + } - if (element.source->lockedRef == info.flake.lockedRef) continue; + upgradedCount++; - printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); + Activity act( + *logger, + lvlChatty, + actUnknown, + fmt("checking '%s' for updates", element.source->attrPath), + Logger::Fields{element.source->attrPath} + ); - element.source = ProfileElementSource { - .originalRef = installable->flakeRef, - .lockedRef = info.flake.lockedRef, - .attrPath = info.value.attrPath, - .outputs = installable->extendedOutputsSpec, - }; + auto installable = make_ref<InstallableFlake>( + this, + getEvalState(), + FlakeRef(element.source->originalRef), + "", + element.source->outputs, + Strings{element.source->attrPath}, + Strings{}, + lockFlags + ); + + auto derivedPaths = installable->toDerivedPaths(); + if (derivedPaths.empty()) { + continue; + } + + auto * infop = dynamic_cast<ExtraPathInfoFlake *>(&*derivedPaths[0].info); + // `InstallableFlake` should use `ExtraPathInfoFlake`. + assert(infop); + auto & info = *infop; - installables.push_back(installable); - indices.push_back(i); + if (element.source->lockedRef == info.flake.lockedRef) { + continue; } + + printInfo( + "upgrading '%s' from flake '%s' to '%s'", + element.source->attrPath, + element.source->lockedRef, + info.flake.lockedRef + ); + + element.source = ProfileElementSource { + .originalRef = installable->flakeRef, + .lockedRef = info.flake.lockedRef, + .attrPath = info.value.attrPath, + .outputs = installable->extendedOutputsSpec, + }; + + installables.push_back(installable); + elems.push_back(&element); + } if (upgradedCount == 0) { - for (auto & matcher : matchers) { - if (const size_t * index = std::get_if<size_t>(&matcher)){ - warn("'%d' is not a valid index", *index); - } else if (const Path * path = std::get_if<Path>(&matcher)){ - warn("'%s' does not match any paths", *path); - } else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)){ - warn("'%s' does not match any packages", regex->pattern); + if (matchedCount == 0) { + for (auto & matcher : matchers) { + if (const Path * path = std::get_if<Path>(&matcher)){ + warn("'%s' does not match any paths", *path); + } else if (const RegexPattern * regex = std::get_if<RegexPattern>(&matcher)) { + warn("'%s' does not match any packages", regex->pattern); + } } + } else { + warn("Found some packages but none of them could be upgraded"); } warn ("Use 'nix profile list' to see the current profile."); } @@ -622,7 +397,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf for (size_t i = 0; i < installables.size(); ++i) { auto & installable = installables.at(i); - auto & element = manifest.elements[indices.at(i)]; + auto & element = *elems.at(i); element.updateStorePaths( getEvalStore(), store, @@ -654,12 +429,16 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro if (json) { std::cout << manifest.toJSON(*store).dump() << "\n"; } else { - for (size_t i = 0; i < manifest.elements.size(); ++i) { - auto & element(manifest.elements[i]); - if (i) logger->cout(""); - logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", - i, - element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); + for (auto const & [i, nameElemPair] : enumerate(manifest.elements)) { + auto & [name, element] = nameElemPair; + if (i) { + logger->cout(""); + } + logger->cout( + "Name: " ANSI_BOLD "%s" ANSI_NORMAL "%s", + name, + element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL + ); if (element.source) { logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); diff --git a/src/nix/repl.md b/src/nix/repl.md index c5113be61..f8ad49199 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Display all special commands within the REPL: diff --git a/src/nix/run.md b/src/nix/run.md index 250ea65aa..7639e4d3e 100644 --- a/src/nix/run.md +++ b/src/nix/run.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Run the default app from the `blender-bin` flake: diff --git a/src/nix/search.md b/src/nix/search.md index f65ac9b17..c0e48d4b3 100644 --- a/src/nix/search.md +++ b/src/nix/search.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Show all packages in the `nixpkgs` flake: diff --git a/src/nix/shell.md b/src/nix/shell.md index f36919575..598a39854 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Start a shell providing `youtube-dl` from the `nixpkgs` flake: diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index af219c1b9..ca8080f88 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,5 +1,11 @@ +#include <algorithm> + +#include "cmd-profiles.hh" #include "command.hh" #include "common-args.hh" +#include "local-fs-store.hh" +#include "logging.hh" +#include "profiles.hh" #include "store-api.hh" #include "filetransfer.hh" #include "eval.hh" @@ -10,11 +16,13 @@ using namespace nix; -struct CmdUpgradeNix : MixDryRun, StoreCommand +struct CmdUpgradeNix : MixDryRun, EvalCommand { Path profileDir; std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; + std::optional<Path> overrideStorePath; + CmdUpgradeNix() { addFlag({ @@ -26,6 +34,13 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand }); addFlag({ + .longName = "store-path", + .description = "A specific store path to upgrade Nix to", + .labels = {"store-path"}, + .handler = {&overrideStorePath}, + }); + + addFlag({ .longName = "nix-store-paths-url", .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, @@ -59,12 +74,15 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand { evalSettings.pureEval = true; - if (profileDir == "") + if (profileDir == "") { profileDir = getProfileDir(store); + } + + auto canonProfileDir = canonPath(profileDir, true); printInfo("upgrading Nix in profile '%s'", profileDir); - auto storePath = getLatestNix(store); + StorePath storePath = getLatestNix(store); auto version = DrvName(storePath.name()).version; @@ -89,11 +107,31 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand stopProgressBar(); - { - Activity act(*logger, lvlInfo, actUnknown, - fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir)); - runProgram(settings.nixBinDir + "/nix-env", false, - {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); + auto const fullStorePath = store->printStorePath(storePath); + + if (pathExists(canonProfileDir + "/manifest.nix")) { + + std::string nixEnvCmd = settings.nixBinDir + "/nix-env"; + Strings upgradeArgs = { + "--profile", + this->profileDir, + "--install", + fullStorePath, + "--no-sandbox", + }; + + printTalkative("running %s %s", nixEnvCmd, concatStringsSep(" ", upgradeArgs)); + runProgram(nixEnvCmd, false, upgradeArgs); + } else if (pathExists(canonProfileDir + "/manifest.json")) { + this->upgradeNewStyleProfile(store, storePath); + } else { + // No I will not use std::unreachable. + // That is undefined behavior if you're wrong. + // This will have a better error message and coredump. + assert( + false && "tried to upgrade unexpected kind of profile, " + "we can only handle `user-environment` and `profile`" + ); } printInfo(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); @@ -121,26 +159,112 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Path profileDir = dirOf(where); // Resolve profile to /nix/var/nix/profiles/<name> link. - while (canonPath(profileDir).find("/profiles/") == std::string::npos && isLink(profileDir)) + while (canonPath(profileDir).find("/profiles/") == std::string::npos && isLink(profileDir)) { profileDir = readLink(profileDir); + } printInfo("found profile '%s'", profileDir); Path userEnv = canonPath(profileDir, true); - if (baseNameOf(where) != "bin" || - !userEnv.ends_with("user-environment")) - throw Error("directory '%s' does not appear to be part of a Nix profile", where); + if (baseNameOf(where) != "bin") { + throw Error("directory '%s' does not appear to be part of a Nix profile (no /bin dir?)", where); + } - if (!store->isValidPath(store->parseStorePath(userEnv))) + if (!pathExists(userEnv + "/manifest.nix") && !pathExists(userEnv + "/manifest.json")) { + throw Error( + "directory '%s' does not have a compatible profile manifest; was it created by Nix?", + where + ); + } + + if (!store->isValidPath(store->parseStorePath(userEnv))) { throw Error("directory '%s' is not in the Nix store", userEnv); + } return profileDir; } + // TODO: Is there like, any good naming scheme that distinguishes + // "profiles which nix-env can use" and "profiles which nix profile can use"? + // You can't just say the manifest version since v2 and v3 are both the latter. + void upgradeNewStyleProfile(ref<Store> & store, StorePath const & newNix) + { + auto fsStore = store.dynamic_pointer_cast<LocalFSStore>(); + // TODO(Qyriad): this check is here because we need to cast to a LocalFSStore, + // to pass to createGeneration(), ...but like, there's no way a remote store + // would work with the nix-env based upgrade either right? + if (!fsStore) { + throw Error("nix upgrade-nix cannot be used on a remote store"); + } + + // nb: nothing actually gets evaluated here. + // The ProfileManifest constructor only evaluates anything for manifest.nix + // profiles, which this is not. + auto evalState = this->getEvalState(); + + ProfileManifest manifest(*evalState, profileDir); + + // Find which profile element has Nix in it. + // It should be impossible to *not* have Nix, since we grabbed this + // store path by looking for things with bin/nix-env in them anyway. + auto findNix = [&](std::pair<std::string, ProfileElement> const & nameElemPair) -> bool { + auto const & [name, elem] = nameElemPair; + for (auto const & ePath : elem.storePaths) { + auto const nixEnv = store->printStorePath(ePath) + "/bin/nix-env"; + if (pathExists(nixEnv)) { + return true; + } + } + // We checked each store path in this element. No nixes here boss! + return false; + }; + auto elemWithNix = std::find_if( + manifest.elements.begin(), + manifest.elements.end(), + findNix + ); + // *Should* be impossible... + assert(elemWithNix != std::end(manifest.elements)); + + auto const nixElemName = elemWithNix->first; + + // Now create a new profile element for the new Nix version... + ProfileElement elemForNewNix = { + .storePaths = {newNix}, + }; + + // ...and splork it into the manifest where the old profile element was. + manifest.elements.at(nixElemName) = elemForNewNix; + + // Build the new profile, and switch to it. + StorePath const newProfile = manifest.build(store); + printTalkative("built new profile '%s'", store->printStorePath(newProfile)); + auto const newGeneration = createGeneration(*fsStore, this->profileDir, newProfile); + printTalkative( + "switching '%s' to newly created generation '%s'", + this->profileDir, + newGeneration + ); + // TODO(Qyriad): use switchGeneration? + // switchLink's docstring seems to indicate that's preferred, but it's + // not used for any other `nix profile`-style profile code except for + // rollback, and it assumes you already have a generation number, which + // we don't. + switchLink(profileDir, newGeneration); + } + /* Return the store path of the latest stable Nix. */ StorePath getLatestNix(ref<Store> store) { + if (this->overrideStorePath) { + printTalkative( + "skipping Nix version query and using '%s' as latest Nix", + *this->overrideStorePath + ); + return store->parseStorePath(*this->overrideStorePath); + } + Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? diff --git a/src/nix/why-depends.md b/src/nix/why-depends.md index dc13619e1..fdbebbedc 100644 --- a/src/nix/why-depends.md +++ b/src/nix/why-depends.md @@ -1,5 +1,7 @@ R""( +**Note:** this command's interface is based heavily around [*installables*](./nix.md#installables), which you may want to read about first (`nix --help`). + # Examples * Show one path through the dependency graph leading from Hello to diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in index b054bf834..3d2e44024 100644 --- a/tests/functional/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -24,7 +24,6 @@ if [[ -n $NIX_STORE ]]; then export _NIX_TEST_NO_SANDBOX=1 fi export _NIX_IN_TEST=$TEST_ROOT/shared -export _NIX_TEST_NO_LSOF=1 export NIX_REMOTE=${NIX_REMOTE_-} unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home diff --git a/tests/functional/config.sh b/tests/functional/config.sh index 723f575ed..46d606d3f 100644 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -40,19 +40,19 @@ files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | x # Test that it's possible to load the config from a custom location here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf -var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) +var=$(nix config show | grep '^substituters =' | cut -d '=' -f 2 | xargs) [[ $var == https://example.com ]] # Test that it's possible to load config from the environment -prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) +prev=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes" -exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) -exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) +exp_cores=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) +exp_features=$(nix config show | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] [[ $exp_features == "flakes nix-command" ]] # Test that it's possible to retrieve a single setting's value -val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) -val2=$(nix show-config warn-dirty) +val=$(nix config show | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) +val2=$(nix config show warn-dirty) [[ $val == $val2 ]] diff --git a/tests/functional/experimental-features.sh b/tests/functional/experimental-features.sh index 607bf0a8e..9ee4a53d4 100644 --- a/tests/functional/experimental-features.sh +++ b/tests/functional/experimental-features.sh @@ -31,7 +31,7 @@ source common.sh NIX_CONFIG=' experimental-features = nix-command accept-flake-config = true -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "false" $TEST_ROOT/stdout grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr @@ -39,7 +39,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature ' NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "false" $TEST_ROOT/stdout grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr @@ -47,7 +47,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature ' NIX_CONFIG=' experimental-features = nix-command flakes accept-flake-config = true -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "true" $TEST_ROOT/stdout grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr @@ -55,7 +55,7 @@ grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command flakes -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "true" $TEST_ROOT/stdout grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr diff --git a/tests/functional/gc-runtime.nix b/tests/functional/gc-runtime.nix index ee5980bdf..4303e0880 100644 --- a/tests/functional/gc-runtime.nix +++ b/tests/functional/gc-runtime.nix @@ -1,17 +1,29 @@ with import ./config.nix; -mkDerivation { - name = "gc-runtime"; - builder = - # Test inline source file definitions. - builtins.toFile "builder.sh" '' - mkdir $out +{ + environ = mkDerivation { + name = "gc-runtime-environ"; + buildCommand = "mkdir $out; echo environ > $out/environ"; + }; - cat > $out/program <<EOF - #! ${shell} - sleep 10000 - EOF + open = mkDerivation { + name = "gc-runtime-open"; + buildCommand = "mkdir $out; echo open > $out/open"; + }; - chmod +x $out/program - ''; + program = mkDerivation { + name = "gc-runtime-program"; + builder = + # Test inline source file definitions. + builtins.toFile "builder.sh" '' + mkdir $out + + cat > $out/program <<EOF + #! ${shell} + sleep 10000 < \$1 + EOF + + chmod +x $out/program + ''; + }; } diff --git a/tests/functional/gc-runtime.sh b/tests/functional/gc-runtime.sh index dc1826a55..6e17acfc0 100644 --- a/tests/functional/gc-runtime.sh +++ b/tests/functional/gc-runtime.sh @@ -1,38 +1,44 @@ source common.sh -case $system in - *linux*) - ;; - *) - skipTest "Not running Linux"; -esac - set -m # enable job control, needed for kill profiles="$NIX_STATE_DIR"/profiles rm -rf $profiles -nix-env -p $profiles/test -f ./gc-runtime.nix -i gc-runtime +nix-env -p $profiles/test -f ./gc-runtime.nix -i gc-runtime-{program,environ,open} -outPath=$(nix-env -p $profiles/test -q --no-name --out-path gc-runtime) -echo $outPath +programPath=$(nix-env -p $profiles/test -q --no-name --out-path gc-runtime-program) +environPath=$(nix-env -p $profiles/test -q --no-name --out-path gc-runtime-environ) +openPath=$(nix-env -p $profiles/test -q --no-name --out-path gc-runtime-open) +echo $programPath $environPath $openPath echo "backgrounding program..." -$profiles/test/program & +export environPath +$profiles/test/program $openPath/open & sleep 2 # hack - wait for the program to get started child=$! echo PID=$child -nix-env -p $profiles/test -e gc-runtime +nix-env -p $profiles/test -e gc-runtime-{program,environ,open} nix-env -p $profiles/test --delete-generations old nix-store --gc kill -- -$child -if ! test -e $outPath; then +if ! test -e $programPath; then echo "running program was garbage collected!" exit 1 fi +if ! test -e $environPath; then + echo "file in environment variable was garbage collected!" + exit 1 +fi + +if ! test -e $openPath; then + echo "opened file was garbage collected!" + exit 1 +fi + exit 0 diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 7c478a0cd..ed014f9ef 100644 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -47,9 +47,9 @@ cp ./config.nix $flake1Dir/ # Test upgrading from nix-env. nix-env -f ./user-envs.nix -i foo-1.0 -nix profile list | grep -A2 'Index:.*0' | grep 'Store paths:.*foo-1.0' +nix profile list | grep -A2 'Name:.*foo' | grep 'Store paths:.*foo-1.0' nix profile install $flake1Dir -L -nix profile list | grep -A4 'Index:.*1' | grep 'Locked flake URL:.*narHash' +nix profile list | grep -A4 'Name:.*flake1' | grep 'Locked flake URL:.*narHash' [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) @@ -60,7 +60,7 @@ nix profile diff-closures | grep 'env-manifest.nix: ε → ∅' # Test XDG Base Directories support export NIX_CONFIG="use-xdg-base-directories = true" -nix profile remove 1 +nix profile remove flake1 2>&1 | grep 'removed 1 packages' nix profile install $flake1Dir [[ $($TEST_HOME/.local/state/nix/profile/bin/hello) = "Hello World" ]] unset NIX_CONFIG @@ -68,7 +68,7 @@ unset NIX_CONFIG # Test upgrading a package. printf NixOS > $flake1Dir/who printf 2.0 > $flake1Dir/version -nix profile upgrade 1 +nix profile upgrade flake1 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]] nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man" @@ -81,7 +81,7 @@ nix profile rollback # Test uninstall. [ -e $TEST_HOME/.nix-profile/bin/foo ] -nix profile remove 0 +nix profile remove "foo" 2>&1 | grep 'removed 1 packages' (! [ -e $TEST_HOME/.nix-profile/bin/foo ]) nix profile history | grep 'foo: 1.0 -> ∅' nix profile diff-closures | grep 'Version 3 -> 4' @@ -89,10 +89,18 @@ nix profile diff-closures | grep 'Version 3 -> 4' # Test installing a non-flake package. nix profile install --file ./simple.nix '' [[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] -nix profile remove 1 +nix profile remove simple 2>&1 | grep 'removed 1 packages' nix profile install $(nix-build --no-out-link ./simple.nix) [[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] +# Test packages with same name from different sources +mkdir $TEST_ROOT/simple-too +cp ./simple.nix ./config.nix simple.builder.sh $TEST_ROOT/simple-too +nix profile install --file $TEST_ROOT/simple-too/simple.nix '' +nix profile list | grep -A4 'Name:.*simple' | grep 'Name:.*simple-1' +nix profile remove simple 2>&1 | grep 'removed 1 packages' +nix profile remove simple-1 2>&1 | grep 'removed 1 packages' + # Test wipe-history. nix profile wipe-history [[ $(nix profile history | grep Version | wc -l) -eq 1 ]] @@ -100,11 +108,11 @@ nix profile wipe-history # Test upgrade to CA package. printf true > $flake1Dir/ca.nix printf 3.0 > $flake1Dir/version -nix profile upgrade 0 +nix profile upgrade flake1 nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man" # Test new install of CA package. -nix profile remove 0 +nix profile remove flake1 2>&1 | grep 'removed 1 packages' printf 4.0 > $flake1Dir/version printf Utrecht > $flake1Dir/who nix profile install $flake1Dir @@ -112,26 +120,27 @@ nix profile install $flake1Dir [[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]] # Override the outputs. -nix profile remove 0 1 +nix profile remove simple flake1 nix profile install "$flake1Dir^*" [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] [ -e $TEST_HOME/.nix-profile/share/man ] [ -e $TEST_HOME/.nix-profile/include ] printf Nix > $flake1Dir/who -nix profile upgrade 0 +nix profile list +nix profile upgrade flake1 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]] [ -e $TEST_HOME/.nix-profile/share/man ] [ -e $TEST_HOME/.nix-profile/include ] -nix profile remove 0 +nix profile remove flake1 2>&1 | grep 'removed 1 packages' nix profile install "$flake1Dir^man" (! [ -e $TEST_HOME/.nix-profile/bin/hello ]) [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) # test priority -nix profile remove 0 +nix profile remove flake1 # Make another flake. flake2Dir=$TEST_ROOT/flake2 @@ -185,3 +194,12 @@ nix profile install $flake2Dir --priority 0 clearProfiles nix profile install $(nix build $flake1Dir --no-link --print-out-paths) expect 1 nix profile install --impure --expr "(builtins.getFlake ''$flake2Dir'').packages.$system.default" + +# Test upgrading from profile version 2. +clearProfiles +mkdir -p $TEST_ROOT/import-profile +outPath=$(nix build --no-link --print-out-paths $flake1Dir/flake.nix^out) +printf '{ "version": 2, "elements": [ { "active": true, "attrPath": "legacyPackages.x86_64-linux.hello", "originalUrl": "flake:nixpkgs", "outputs": null, "priority": 5, "storePaths": [ "%s" ], "url": "github:NixOS/nixpkgs/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } ] }' "$outPath" > $TEST_ROOT/import-profile/manifest.json +nix build --profile $TEST_HOME/.nix-profile $(nix store add-path $TEST_ROOT/import-profile) +nix profile list | grep -A4 'Name:.*hello' | grep "Store paths:.*$outPath" +nix profile remove hello 2>&1 | grep 'removed 1 packages, kept 0 packages' diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index f7a8588e5..3d0a1f0c6 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -141,6 +141,8 @@ in nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix; + nix-upgrade-nix = runNixOSTestFor "x86_64-linux" ./nix-upgrade-nix.nix; + nssPreload = runNixOSTestFor "x86_64-linux" ./nss-preload.nix; githubFlakes = runNixOSTestFor "x86_64-linux" ./github-flakes.nix; diff --git a/tests/nixos/nix-upgrade-nix.nix b/tests/nixos/nix-upgrade-nix.nix new file mode 100644 index 000000000..039b2d9b3 --- /dev/null +++ b/tests/nixos/nix-upgrade-nix.nix @@ -0,0 +1,80 @@ +{ lib, config, ... }: + +/** + * Test that nix upgrade-nix works regardless of whether /nix/var/nix/profiles/default + * is a nix-env style profile or a nix profile style profile. + */ + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + lix = pkgs.nix; + lixVersion = lib.getVersion lix; + + newNix = pkgs.nixVersions.unstable; + newNixVersion = lib.getVersion newNix; + +in { + name = "nix-upgrade-nix"; + + nodes = { + machine = { config, lib, pkgs, ... }: { + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgs.hello.drvPath ]; + nix.settings.substituters = lib.mkForce [ ]; + nix.settings.experimental-features = [ "nix-command" "flakes" ]; + services.getty.autologinUser = "root"; + + }; + }; + + testScript = { nodes }: '' + # fmt: off + + start_all() + + machine.succeed("nix --version >&2") + + # Install Lix into the default profile, overriding /run/current-system/sw/bin/nix, + # and thus making Lix think we're not on NixOS. + machine.succeed("nix-env --install '${lib.getBin lix}' --profile /nix/var/nix/profiles/default >&2") + + # Make sure that correctly got inserted into our PATH. + default_profile_nix_path = machine.succeed("command -v nix") + print(default_profile_nix_path) + assert default_profile_nix_path.strip() == "/nix/var/nix/profiles/default/bin/nix", \ + f"{default_profile_nix_path.strip()=} != /nix/var/nix/profiles/default/bin/nix" + + # And that it's the Nix we specified. + default_profile_version = machine.succeed("nix --version") + assert "${lixVersion}" in default_profile_version, f"${lixVersion} not in {default_profile_version}" + + # Upgrade to a different version of Nix, and make sure that also worked. + + machine.succeed("nix upgrade-nix --store-path ${newNix} >&2") + default_profile_version = machine.succeed("nix --version") + print(default_profile_version) + assert "${newNixVersion}" in default_profile_version, f"${newNixVersion} not in {default_profile_version}" + + # Now 'break' this profile -- use nix profile on it so nix-env will no longer work on it. + machine.succeed( + "nix profile install --profile /nix/var/nix/profiles/default '${pkgs.hello.drvPath}^*' >&2" + ) + + # Confirm that nix-env is broken. + machine.fail( + "nix-env --query --installed --profile /nix/var/nix/profiles/default >&2" + ) + + # And use nix upgrade-nix one more time, on the `nix profile` style profile. + # (Specifying Lix by full path so we can use --store-path.) + machine.succeed( + "${lib.getBin lix}/bin/nix upgrade-nix --store-path '${lix}' >&2" + ) + + default_profile_version = machine.succeed("nix --version") + print(default_profile_version) + assert "${lixVersion}" in default_profile_version, f"${lixVersion} not in {default_profile_version}" + ''; + +} diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix index 5ff471607..8deb9a504 100644 --- a/tests/nixos/remote-builds-ssh-ng.nix +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -95,6 +95,10 @@ in builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd.service") + + out = client.fail("nix-build ${expr nodes.client 1} 2>&1") + assert "error: failed to start SSH connection to 'root@builder': Host key verification failed" in out, f"No host verification error in {out}" + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world' >&2") # Perform a build diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc index ffc4fc607..8ddda8027 100644 --- a/tests/unit/libstore-support/tests/path.cc +++ b/tests/unit/libstore-support/tests/path.cc @@ -1,3 +1,4 @@ +#include <rapidcheck/gen/Arbitrary.h> #include <regex> #include <rapidcheck.h> @@ -20,65 +21,60 @@ void showValue(const StorePath & p, std::ostream & os) namespace rc { using namespace nix; -Gen<StorePathName> Arbitrary<StorePathName>::arbitrary() +Gen<char> storePathChar() { - auto len = *gen::inRange<size_t>( - 1, - StorePath::MaxPathLen - StorePath::HashLen); - - std::string pre; - pre.reserve(len); - - for (size_t c = 0; c < len; ++c) { - switch (auto i = *gen::inRange<uint8_t>(0, 10 + 2 * 26 + 6)) { + return rc::gen::apply([](uint8_t i) -> char { + switch (i) { case 0 ... 9: - pre += static_cast<uint8_t>('0' + i); - break; + return '0' + i; case 10 ... 35: - pre += static_cast<uint8_t>('A' + (i - 10)); - break; + return 'A' + (i - 10); case 36 ... 61: - pre += static_cast<uint8_t>('a' + (i - 36)); - break; + return 'a' + (i - 36); case 62: - pre += '+'; - break; + return '+'; case 63: - pre += '-'; - break; + return '-'; case 64: - // names aren't permitted to start with a period, - // so just fall through to the next case here - if (c != 0) { - pre += '.'; - break; - } - [[fallthrough]]; + return '.'; case 65: - pre += '_'; - break; + return '_'; case 66: - pre += '?'; - break; + return '?'; case 67: - pre += '='; - break; + return '='; default: assert(false); } - } + }, + gen::inRange<uint8_t>(0, 10 + 2 * 26 + 6)); +} - return gen::just(StorePathName { - .name = std::move(pre), - }); +Gen<StorePathName> Arbitrary<StorePathName>::arbitrary() +{ + return gen::construct<StorePathName>( + gen::suchThat( + gen::container<std::string>(storePathChar()), + [](const std::string & s) { + return + !( s == "" + || s == "." + || s == ".." + || s.starts_with(".-") + || s.starts_with("..-") + ); + } + ) + ); } Gen<StorePath> Arbitrary<StorePath>::arbitrary() { - return gen::just(StorePath { - *gen::arbitrary<Hash>(), - (*gen::arbitrary<StorePathName>()).name, - }); + return + gen::construct<StorePath>( + gen::arbitrary<Hash>(), + gen::apply([](StorePathName n){ return n.name; }, gen::arbitrary<StorePathName>()) + ); } } // namespace rc diff --git a/tests/unit/libstore/path.cc b/tests/unit/libstore/path.cc index 30631b5fd..213b6e95f 100644 --- a/tests/unit/libstore/path.cc +++ b/tests/unit/libstore/path.cc @@ -39,7 +39,12 @@ TEST_DONT_PARSE(double_star, "**") TEST_DONT_PARSE(star_first, "*,foo") TEST_DONT_PARSE(star_second, "foo,*") TEST_DONT_PARSE(bang, "foo!o") -TEST_DONT_PARSE(dotfile, ".gitignore") +TEST_DONT_PARSE(dot, ".") +TEST_DONT_PARSE(dot_dot, "..") +TEST_DONT_PARSE(dot_dot_dash, "..-1") +TEST_DONT_PARSE(dot_dash, ".-1") +TEST_DONT_PARSE(dot_dot_dash_a, "..-a") +TEST_DONT_PARSE(dot_dash_a, ".-a") #undef TEST_DONT_PARSE @@ -63,6 +68,11 @@ TEST_DO_PARSE(underscore, "foo_bar") TEST_DO_PARSE(period, "foo.txt") TEST_DO_PARSE(question_mark, "foo?why") TEST_DO_PARSE(equals_sign, "foo=foo") +TEST_DO_PARSE(dotfile, ".gitignore") +TEST_DO_PARSE(triple_dot_a, "...a") +TEST_DO_PARSE(triple_dot_1, "...1") +TEST_DO_PARSE(triple_dot_dash, "...-") +TEST_DO_PARSE(triple_dot, "...") #undef TEST_DO_PARSE @@ -84,6 +94,64 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(p == store->parseStorePath(store->printStorePath(p))); } + +RC_GTEST_FIXTURE_PROP( + StorePathTest, + prop_check_regex_eq_parse, + ()) +{ + static auto nameFuzzer = + rc::gen::container<std::string>( + rc::gen::oneOf( + // alphanum, repeated to weigh heavier + rc::gen::oneOf( + rc::gen::inRange('0', '9'), + rc::gen::inRange('a', 'z'), + rc::gen::inRange('A', 'Z') + ), + // valid symbols + rc::gen::oneOf( + rc::gen::just('+'), + rc::gen::just('-'), + rc::gen::just('.'), + rc::gen::just('_'), + rc::gen::just('?'), + rc::gen::just('=') + ), + // symbols for scary .- and ..- cases, repeated for weight + rc::gen::just('.'), rc::gen::just('.'), + rc::gen::just('.'), rc::gen::just('.'), + rc::gen::just('-'), rc::gen::just('-'), + // ascii symbol ranges + rc::gen::oneOf( + rc::gen::inRange(' ', '/'), + rc::gen::inRange(':', '@'), + rc::gen::inRange('[', '`'), + rc::gen::inRange('{', '~') + ), + // typical whitespace + rc::gen::oneOf( + rc::gen::just(' '), + rc::gen::just('\t'), + rc::gen::just('\n'), + rc::gen::just('\r') + ), + // some chance of control codes, non-ascii or other garbage we missed + rc::gen::inRange('\0', '\xff') + )); + + auto name = *nameFuzzer; + + std::string path = store->storeDir + "/575s52sh487i0ylmbs9pvi606ljdszr0-" + name; + bool parsed = false; + try { + store->parseStorePath(path); + parsed = true; + } catch (const BadStorePath &) { + } + RC_ASSERT(parsed == std::regex_match(std::string { name }, nameRegex)); +} + #endif } diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc index 577e9890e..7cc994b40 100644 --- a/tests/unit/libutil-support/tests/hash.cc +++ b/tests/unit/libutil-support/tests/hash.cc @@ -11,10 +11,17 @@ using namespace nix; Gen<Hash> Arbitrary<Hash>::arbitrary() { - Hash hash(htSHA1); - for (size_t i = 0; i < hash.hashSize; ++i) - hash.hash[i] = *gen::arbitrary<uint8_t>(); - return gen::just(hash); + Hash prototype(htSHA1); + return + gen::apply( + [](const std::vector<uint8_t> & v) { + Hash hash(htSHA1); + assert(v.size() == hash.hashSize); + std::copy(v.begin(), v.end(), hash.hash); + return hash; + }, + gen::container<std::vector<uint8_t>>(prototype.hashSize, gen::arbitrary<uint8_t>()) + ); } } diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index f55c56548..720370066 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -404,6 +404,45 @@ namespace nix { ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); } + TEST(rewriteStrings, intransitive) { + StringMap rewrites; + // transitivity can happen both in forward and reverse iteration order of the rewrite map. + rewrites["a"] = "b"; + rewrites["b"] = "c"; + rewrites["e"] = "b"; + + ASSERT_EQ(rewriteStrings("abcde", rewrites), "bccdb"); + } + + TEST(rewriteStrings, nonoverlapping) { + StringMap rewrites; + rewrites["ab"] = "ca"; + + ASSERT_EQ(rewriteStrings("abb", rewrites), "cab"); + } + + TEST(rewriteStrings, differentLength) { + StringMap rewrites; + rewrites["a"] = "an has a trea"; + + ASSERT_EQ(rewriteStrings("cat", rewrites), "can has a treat"); + } + + TEST(rewriteStrings, sorted) { + StringMap rewrites; + rewrites["a"] = "meow"; + rewrites["abc"] = "puppy"; + + ASSERT_EQ(rewriteStrings("abcde", rewrites), "meowbcde"); + } + + TEST(rewriteStrings, multiple) { + StringMap rewrites; + rewrites["a"] = "b"; + + ASSERT_EQ(rewriteStrings("a1a2a3a", rewrites), "b1b2b3b"); + } + TEST(rewriteStrings, doesntOccur) { StringMap rewrites; rewrites["foo"] = "bar"; diff --git a/tests/unit/libutil/url-name.cc b/tests/unit/libutil/url-name.cc new file mode 100644 index 000000000..164bb26d7 --- /dev/null +++ b/tests/unit/libutil/url-name.cc @@ -0,0 +1,69 @@ +#include "url-name.hh" +#include <gtest/gtest.h> + +namespace nix { + +/* ----------- tests for url-name.hh --------------------------------------------------*/ + + TEST(getNameFromURL, getNameFromURL) { + ASSERT_EQ(getNameFromURL(parseURL("path:/home/user/project")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#legacyPackages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:~/repos/nixpkgs#packages.x86_64-linux.Hello")), "Hello"); + ASSERT_EQ(getNameFromURL(parseURL("path:.#nonStandardAttr.mylaptop")), "mylaptop"); + ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "mylaptop"); + ASSERT_EQ(getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man")), "complex"); + ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*")), "myproj"); + + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("github:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + ASSERT_EQ(getNameFromURL(parseURL("github:edolstra/nix-warez?rev=1234&dir=blender&ref=master")), "blender"); + + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("gitlab:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nixpkgs#hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix#packages.x86_64-linux.default")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix#")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:NixOS/nix")), "nix"); + ASSERT_EQ(getNameFromURL(parseURL("sourcehut:cachix/devenv/main#packages.x86_64-linux.default")), "devenv"); + + ASSERT_EQ(getNameFromURL(parseURL("git://github.com/edolstra/dwarffs")), "dwarffs"); + ASSERT_EQ(getNameFromURL(parseURL("git://github.com/edolstra/nix-warez?dir=blender")), "blender"); + ASSERT_EQ(getNameFromURL(parseURL("git+file:///home/user/project")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("git+file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("git+ssh://git@github.com/someuser/my-repo#")), "my-repo"); + ASSERT_EQ(getNameFromURL(parseURL("git+git://github.com/someuser/my-repo?rev=v1.2.3")), "my-repo"); + ASSERT_EQ(getNameFromURL(parseURL("git+ssh:///home/user/project?dir=subproject&rev=v2.4")), "subproject"); + ASSERT_EQ(getNameFromURL(parseURL("git+http://not-even-real#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("git+https://not-even-real#packages.aarch64-darwin.hello")), "hello"); + + ASSERT_EQ(getNameFromURL(parseURL("tarball+http://github.com/NixOS/nix/archive/refs/tags/2.18.1#packages.x86_64-linux.jq")), "jq"); + ASSERT_EQ(getNameFromURL(parseURL("tarball+https://github.com/NixOS/nix/archive/refs/tags/2.18.1#packages.x86_64-linux.hg")), "hg"); + ASSERT_EQ(getNameFromURL(parseURL("tarball+file:///home/user/Downloads/nixpkgs-2.18.1#packages.aarch64-darwin.ripgrep")), "ripgrep"); + + ASSERT_EQ(getNameFromURL(parseURL("https://github.com/NixOS/nix/archive/refs/tags/2.18.1.tar.gz#packages.x86_64-linux.pv")), "pv"); + ASSERT_EQ(getNameFromURL(parseURL("http://github.com/NixOS/nix/archive/refs/tags/2.18.1.tar.gz#packages.x86_64-linux.pv")), "pv"); + + ASSERT_EQ(getNameFromURL(parseURL("file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("file+file:///home/user/project?ref=fa1e2d23a22")), "project"); + ASSERT_EQ(getNameFromURL(parseURL("file+http://not-even-real#packages.x86_64-linux.hello")), "hello"); + ASSERT_EQ(getNameFromURL(parseURL("file+http://gitfantasy.com/org/user/notaflake")), "notaflake"); + ASSERT_EQ(getNameFromURL(parseURL("file+https://not-even-real#packages.aarch64-darwin.hello")), "hello"); + + ASSERT_EQ(getNameFromURL(parseURL("https://www.github.com/")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("file:.#")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default")), std::nullopt); + ASSERT_EQ(getNameFromURL(parseURL("path:.#packages.x86_64-linux.default^*")), std::nullopt); + } +} diff --git a/tests/unit/meson.build b/tests/unit/meson.build index ae850df47..339ac9a4a 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -52,6 +52,7 @@ libutil_tests_sources = files( 'libutil/suggestions.cc', 'libutil/tests.cc', 'libutil/url.cc', + 'libutil/url-name.cc', 'libutil/xml-writer.cc', ) |