diff options
123 files changed, 1291 insertions, 834 deletions
diff --git a/.clang-tidy b/.clang-tidy index 0cc1f2520..87f6d0404 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -16,6 +16,20 @@ Checks: - -bugprone-unchecked-optional-access # many warnings, seems like a questionable lint - -bugprone-branch-clone + # all thrown exceptions must derive from std::exception + - hicpp-exception-baseclass + # capturing async lambdas are dangerous + - cppcoreguidelines-avoid-capturing-lambda-coroutines + # crimes must be appropriately declared as crimes + - cppcoreguidelines-pro-type-cstyle-cast + - lix-* + # This can not yet be applied to Lix itself since we need to do source + # reorganization so that lix/ include paths work. + - -lix-fixincludes + # This lint is included as an example, but the lib function it replaces is + # already gone. + - -lix-hasprefixsuffix + CheckOptions: bugprone-reserved-identifier.AllowedIdentifiers: '__asan_default_options' diff --git a/doc/manual/change-authors.yml b/doc/manual/change-authors.yml index 630af29ff..e18abada1 100644 --- a/doc/manual/change-authors.yml +++ b/doc/manual/change-authors.yml @@ -103,6 +103,11 @@ midnightveil: ncfavier: github: ncfavier +piegames: + display_name: piegames + forgejo: piegames + github: piegamesde + puck: display_name: puck forgejo: puck diff --git a/doc/manual/rl-next/pipe-operator.md b/doc/manual/rl-next/pipe-operator.md new file mode 100644 index 000000000..49dc01308 --- /dev/null +++ b/doc/manual/rl-next/pipe-operator.md @@ -0,0 +1,10 @@ +--- +synopsis: Pipe operator `|>` (experimental) +issues: [fj#438] +cls: [1654] +category: Features +credits: [piegames, horrors] +--- + +Implementation of the pipe operator (`|>`) in the language as described in [RFC 148](https://github.com/NixOS/rfcs/pull/148). +The feature is still marked experimental, enable `--extra-experimental-features pipe-operator` to use it. diff --git a/doc/manual/rl-next/repl-log-drv.md b/doc/manual/rl-next/repl-log-drv.md new file mode 100644 index 000000000..37d6f5cb2 --- /dev/null +++ b/doc/manual/rl-next/repl-log-drv.md @@ -0,0 +1,9 @@ +--- +synopsis: "`:log` in repl now works on derivation paths" +issues: [fj#51] +cls: [1716] +category: Improvements +credits: [goldstein] +--- + +`:log` can now accept store derivation paths in addition to derivation expressions. diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index e9006a3ae..ee93f5672 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -292,6 +292,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6dcdc6eb0..2d4707814 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,6 +26,8 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* <code>\|\|</code> *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | none | 14 | +| \[Experimental\] [Function piping] | *expr* |> *func* | left | 15 | +| \[Experimental\] [Function piping] | *expr* <| *func* | right | 16 | [string]: ./values.md#type-string [path]: ./values.md#type-path @@ -215,3 +217,33 @@ nix-repl> let f = x: 1; s = { func = f; }; in [ (f == f) (s == s) ] Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication + +## \[Experimental\] Function piping + +*This language feature is still experimental and may change at any time. Enable `--extra-experimental-features pipe-operator` to use it.* + +Pipes are a dedicated operator for function application, but with reverse order and a lower binding strength. +This allows you to chain function calls together in way that is more natural to read and requires less parentheses. + +`a |> f b |> g` is equivalent to `g (f b a)`. +`g <| f b <| a` is equivalent to `g (f b a)`. + +Example code snippet: + +```nix +defaultPrefsFile = defaultPrefs + |> lib.mapAttrsToList ( + key: value: '' + // ${value.reason} + pref("${key}", ${builtins.toJSON value.value}); + '' + ) + |> lib.concatStringsSep "\n" + |> pkgs.writeText "nixos-default-prefs.js"; +``` + +Note how `mapAttrsToList` is called with two arguments (the lambda and `defaultPrefs`), +but moving the last argument in front of the rest improves the reading flow. +This is common for functions with long first argument, including all `map`-like functions. + +[Function piping]: #experimental-function-piping @@ -282,6 +282,10 @@ # cheaper x86_64-linux compute in CI. # It is clangStdenv because clang's sanitizers are nicer. asanBuild = self.packages.x86_64-linux.nix-clangStdenv.override { + # Improve caching of non-code changes by not changing the + # derivation name every single time, since this will never be seen + # by users anyway. + versionSuffix = ""; sanitize = [ "address" "undefined" @@ -310,6 +314,17 @@ touch $out ''; + # clang-tidy run against the Lix codebase using the Lix clang-tidy plugin + clang-tidy = + let + nixpkgs = nixpkgsFor.x86_64-linux.native; + inherit (nixpkgs) pkgs; + in + pkgs.callPackage ./package.nix { + versionSuffix = ""; + lintInsteadOfBuild = true; + }; + # Make sure that nix-env still produces the exact same result # on a particular version of Nixpkgs. evalNixpkgs = diff --git a/package.nix b/package.nix index 0f006fef4..ddceb23a5 100644 --- a/package.nix +++ b/package.nix @@ -28,6 +28,8 @@ libcpuid, libseccomp, libsodium, + lix-clang-tidy ? null, + llvmPackages, lsof, lowdown, mdbook, @@ -58,12 +60,17 @@ buildUnreleasedNotes ? true, internalApiDocs ? false, + # Support garbage collection in the evaluator. + enableGC ? sanitize == null || !builtins.elem "address" sanitize, # List of Meson sanitize options. Accepts values of b_sanitize, e.g. # "address", "undefined", "thread". + # Enabling the "address" sanitizer will disable garbage collection in the evaluator. sanitize ? null, # Turn compiler warnings into errors. werror ? false, + lintInsteadOfBuild ? false, + # Not a real argument, just the only way to approximate let-binding some # stuff for argument defaults. __forDefaults ? { @@ -118,10 +125,7 @@ let # The internal API docs need these for the build, but if we're not building # Nix itself, then these don't need to be propagated. - maybePropagatedInputs = [ - boehmgc-nix - nlohmann_json - ]; + maybePropagatedInputs = lib.optional enableGC boehmgc-nix ++ [ nlohmann_json ]; # .gitignore has already been processed, so any changes in it are irrelevant # at this point. It is not represented verbatim for test purposes because @@ -144,6 +148,7 @@ let (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) ]; in +assert (lintInsteadOfBuild -> lix-clang-tidy != null); stdenv.mkDerivation (finalAttrs: { inherit pname version; @@ -156,12 +161,13 @@ stdenv.mkDerivation (finalAttrs: { topLevelBuildFiles functionalTestFiles ] - ++ lib.optionals (!finalAttrs.dontBuild || internalApiDocs) [ + ++ lib.optionals (!finalAttrs.dontBuild || internalApiDocs || lintInsteadOfBuild) [ ./doc ./misc ./src ./COPYING ] + ++ lib.optionals lintInsteadOfBuild [ ./.clang-tidy ] ) ); }; @@ -175,14 +181,13 @@ stdenv.mkDerivation (finalAttrs: { "doc" ]; - dontBuild = false; + dontBuild = lintInsteadOfBuild; mesonFlags = let - sanitizeOpts = lib.optionals (sanitize != null) ( - [ "-Db_sanitize=${builtins.concatStringsSep "," sanitize}" ] - ++ lib.optional (builtins.elem "address" sanitize) "-Dgc=disabled" - ); + sanitizeOpts = lib.optional ( + sanitize != null + ) "-Db_sanitize=${builtins.concatStringsSep "," sanitize}"; in lib.optionals hostPlatform.isLinux [ # You'd think meson could just find this in PATH, but busybox is in buildInputs, @@ -191,13 +196,15 @@ stdenv.mkDerivation (finalAttrs: { "-Dsandbox-shell=${lib.getExe' busybox-sandbox-shell "busybox"}" ] ++ lib.optional hostPlatform.isStatic "-Denable-embedded-sandbox-shell=true" - ++ lib.optional (finalAttrs.dontBuild) "-Denable-build=false" + ++ lib.optional (finalAttrs.dontBuild && !lintInsteadOfBuild) "-Denable-build=false" + ++ lib.optional lintInsteadOfBuild "-Dlix-clang-tidy-checks-path=${lix-clang-tidy}/lib/liblix-clang-tidy.so" ++ [ # mesonConfigurePhase automatically passes -Dauto_features=enabled, # so we must explicitly enable or disable features that we are not passing # dependencies for. + (lib.mesonEnable "gc" enableGC) (lib.mesonEnable "internal-api-docs" internalApiDocs) - (lib.mesonBool "enable-tests" finalAttrs.finalPackage.doCheck) + (lib.mesonBool "enable-tests" (finalAttrs.finalPackage.doCheck || lintInsteadOfBuild)) (lib.mesonBool "enable-docs" canRunInstalled) (lib.mesonBool "werror" werror) ] @@ -230,7 +237,13 @@ stdenv.mkDerivation (finalAttrs: { ] ++ lib.optional hostPlatform.isLinux util-linuxMinimal ++ lib.optional (!officialRelease && buildUnreleasedNotes) build-release-notes - ++ lib.optional internalApiDocs doxygen; + ++ lib.optional internalApiDocs doxygen + ++ lib.optionals lintInsteadOfBuild [ + # required for a wrapped clang-tidy + llvmPackages.clang-tools + # required for run-clang-tidy + llvmPackages.clang-unwrapped + ]; buildInputs = [ @@ -257,7 +270,10 @@ stdenv.mkDerivation (finalAttrs: { ++ lib.optional hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (hostPlatform.canExecute buildPlatform) aws-sdk-cpp-nix - ++ lib.optionals (finalAttrs.dontBuild) maybePropagatedInputs; + ++ lib.optionals (finalAttrs.dontBuild) maybePropagatedInputs + # I am so sorry. This is because checkInputs are required to pass + # configure, but we don't actually want to *run* the checks here. + ++ lib.optionals lintInsteadOfBuild finalAttrs.checkInputs; nativeCheckInputs = [ expect ]; @@ -315,7 +331,7 @@ stdenv.mkDerivation (finalAttrs: { enableParallelBuilding = true; - doCheck = canRunInstalled; + doCheck = canRunInstalled && !lintInsteadOfBuild; mesonCheckFlags = [ "--suite=check" @@ -327,8 +343,19 @@ stdenv.mkDerivation (finalAttrs: { # Make sure the internal API docs are already built, because mesonInstallPhase # won't let us build them there. They would normally be built in buildPhase, # but the internal API docs are conventionally built with doBuild = false. - preInstall = lib.optional internalApiDocs '' - meson ''${mesonBuildFlags:-} compile "$installTargets" + preInstall = + (lib.optionalString internalApiDocs '' + meson ''${mesonBuildFlags:-} compile "$installTargets" + '') + # evil, but like above, we do not want to run an actual build phase + + lib.optionalString lintInsteadOfBuild '' + ninja clang-tidy + ''; + + installPhase = lib.optionalString lintInsteadOfBuild '' + runHook preInstall + touch $out + runHook postInstall ''; postInstall = @@ -396,12 +423,10 @@ stdenv.mkDerivation (finalAttrs: { mkShell, bashInteractive, - clang-tools, clangbuildanalyzer, doxygen, glibcLocales, just, - llvmPackages, nixfmt-rfc-style, skopeo, xonsh, @@ -423,6 +448,7 @@ stdenv.mkDerivation (finalAttrs: { p.python-frontmatter p.requests p.xdg-base-dirs + p.packaging (p.toPythonModule xonsh.passthru.unwrapped) ] ); @@ -454,7 +480,7 @@ stdenv.mkDerivation (finalAttrs: { ++ [ (lib.mesonBool "enable-pch-std" stdenv.cc.isClang) ]; packages = - lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) clang-tools + lib.optional (stdenv.cc.isClang && hostPlatform == buildPlatform) llvmPackages.clang-tools ++ [ # Why are we providing a bashInteractive? Well, when you run # `bash` from inside `nix develop`, say, because you are using it diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 4bef020d3..ac79f2777 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -77,7 +77,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base::Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -103,7 +103,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); + auto s = info->narHash.to_string(base32 ? Base::Base32 : Base::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -205,7 +205,7 @@ SV * hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath(parseHashType(algo), path).first; - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? Base::Base32 : Base::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -216,7 +216,7 @@ SV * hashFile(char * algo, int base32, char * path) PPCODE: try { Hash h = hashFile(parseHashType(algo), path); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? Base::Base32 : Base::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -227,7 +227,7 @@ SV * hashString(char * algo, int base32, char * s) PPCODE: try { Hash h = hashString(parseHashType(algo), s); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? Base::Base32 : Base::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -238,7 +238,7 @@ SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { auto h = Hash::parseAny(s, parseHashType(algo)); - auto s = h.to_string(toBase32 ? Base32 : Base16, false); + auto s = h.to_string(toBase32 ? Base::Base32 : Base::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); diff --git a/releng/README.md b/releng/README.md index 2aa3b959f..75bd4ab35 100644 --- a/releng/README.md +++ b/releng/README.md @@ -57,12 +57,10 @@ Next, we do the publication with `python -m releng upload`: `nix upgrade-nix`. * s3://releases/lix/lix-VERSION/ gets the following contents * Binary tarballs - * Docs: `manual/` (FIXME: should we actually do this? what about putting it - on docs.lix.systems? I think doing both is correct, since the Web site - should not be an archive of random old manuals) + * Docs: `manual/`, primarily as an archive of old manuals * Docs as tarball in addition to web. * Source tarball - * Docker image (FIXME: upload to forgejo registry and github registry [in the future][upload-docker]) + * Docker image * s3://docs/manual/lix/MAJOR * s3://docs/manual/lix/stable @@ -80,6 +78,7 @@ Next, we do the publication with `python -m releng upload`: FIXME: automate branch-off to `release-*` branch. * **Manually** (FIXME?) switch back to the release branch, which now has the correct revision. +* Deal with the external systems (see sections below). * Post!! * Merge release blog post to [lix-website]. * Toot about it! https://chaos.social/@lix_project @@ -87,22 +86,33 @@ Next, we do the publication with `python -m releng upload`: [lix-website]: https://git.lix.systems/lix-project/lix-website -[upload-docker]: https://git.lix.systems/lix-project/lix/issues/252 - ### Installer -The installer is cross-built to several systems from a Mac using -`build-all.xsh` and `upload-to-lix.xsh` in the installer repo (FIXME: currently -at least; maybe this should be moved here?) . +The installer is cross-built to several systems from a Mac using `build-all.xsh` and `upload-to-lix.xsh` in the installer repo (FIXME: currently at least; maybe this should be moved here?). + +It installs a binary tarball (FIXME: [it should be taught to substitute from cache instead][installer-substitute]) from some URL; this is the `hydraJobs.binaryTarball`. +The default URLs differ by architecture and are [configured here][tarball-urls]. -It installs a binary tarball (FIXME: [it should be taught to substitute from -cache instead][installer-substitute]) -from some URL; this is the `hydraJobs.binaryTarball`. The default URLs differ -by architecture and are [configured here][tarball-urls]. +To automatically do the file changes for a new version, run `python3 set_version.py NEW_VERSION`, and submit the result for review. [installer-substitute]: https://git.lix.systems/lix-project/lix-installer/issues/13 [tarball-urls]: https://git.lix.systems/lix-project/lix-installer/src/commit/693592ed10d421a885bec0a9dd45e87ab87eb90a/src/settings.rs#L14-L28 +### Web site + +The website has various release-version dependent pieces. +You can update them with `python3 update_version.py NEW_VERSION`, which will regenerate the affected page sources. + +These need the release to have been done first as they need hashes for tarballs and such. + +### NixOS module + +The NixOS module has underdeveloped releng in it. +Currently you have to do the whole branch-off dance manually to a `release-VERSION` branch and update the tarball URLs to point to the release versions manually. + +FIXME: this should be unified with the `set_version.py` work in `lix-installer` and probably all the releng kept in here, or kept elsewhere. +Related: https://git.lix.systems/lix-project/lix/issues/439 + ## Infrastructure summary * releases.lix.systems (`s3://releases`): diff --git a/releng/cli.py b/releng/cli.py index f78d4b12d..f2d193444 100644 --- a/releng/cli.py +++ b/releng/cli.py @@ -1,10 +1,13 @@ +import logging +import argparse +import sys + from . import create_release from . import docker from .environment import RelengEnvironment from . import environment -import argparse -import sys +log = logging.getLogger(__name__) def do_build(args): if args.target == 'all': @@ -21,6 +24,9 @@ def do_tag(args): create_release.do_tag_merge(force_tag=args.force_tag, no_check_git=args.no_check_git) + log.info('Merged the release commit into your last branch, and switched to a detached HEAD of the artifact to be released.') + log.info('After you are done with releasing, switch to your previous branch and push that branch for review.') + def do_upload(env: RelengEnvironment, args): create_release.setup_creds(env) diff --git a/releng/create_release.xsh b/releng/create_release.xsh index 62114350b..d8d6e8a83 100644 --- a/releng/create_release.xsh +++ b/releng/create_release.xsh @@ -2,6 +2,7 @@ import json import subprocess import itertools import textwrap +import logging from pathlib import Path import tempfile import hashlib @@ -15,6 +16,8 @@ from .version import VERSION, RELEASE_NAME, MAJOR, OFFICIAL_RELEASE from .gitutils import verify_are_on_tag, git_preconditions from . import release_notes +log = logging.getLogger(__name__) + $RAISE_SUBPROC_ERROR = True $XONSH_SHOW_TRACEBACK = True @@ -55,6 +58,9 @@ def official_release_commit_tag(force_tag=False): git commit -m @(message) git tag @(['-f'] if force_tag else []) -a -m @(message) @(VERSION) + with open('releng/prev-git-branch.txt', 'w') as fh: + fh.write(prev_branch) + return prev_branch @@ -235,8 +241,24 @@ def upload_artifacts(env: RelengEnvironment, noconfirm=False, no_check_git=False print('[+] Upload manual') upload_manual(env) - print('[+] git push tag') - git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}' + prev_branch = None + try: + with open('releng/prev-git-branch.txt', 'r') as fh: + prev_branch = fh.read().strip() + except FileNotFoundError: + log.warn('Cannot find previous git branch file, skipping pushing git objects') + + if prev_branch: + print('[+] git push to the repo') + # We have to push the ref to gerrit for review at least such that the + # commit is known, before we can push it as a tag. + if env.git_repo_is_gerrit: + git push @(env.git_repo) f'{prev_branch}:refs/for/{prev_branch}' + else: + git push @(env.git_repo) f'{prev_branch}:{prev_branch}' + + print('[+] git push tag') + git push @(['-f'] if force_push_tag else []) @(env.git_repo) f'{VERSION}:refs/tags/{VERSION}' def do_tag_merge(force_tag=False, no_check_git=False): diff --git a/releng/environment.py b/releng/environment.py index ca8194fe5..2b6554ede 100644 --- a/releng/environment.py +++ b/releng/environment.py @@ -52,6 +52,7 @@ class RelengEnvironment: releases_bucket: str docs_bucket: str git_repo: str + git_repo_is_gerrit: bool docker_targets: list[DockerTarget] @@ -79,6 +80,7 @@ STAGING = RelengEnvironment( cache_store_overlay={'secret-key': 'staging.key'}, releases_bucket='s3://staging-releases', git_repo='ssh://git@git.lix.systems/lix-project/lix-releng-staging', + git_repo_is_gerrit=False, docker_targets=[ # latest will be auto tagged if appropriate DockerTarget('git.lix.systems/lix-project/lix-releng-staging', @@ -113,6 +115,7 @@ PROD = RelengEnvironment( cache_store_overlay={'secret-key': 'prod.key'}, releases_bucket='s3://releases', git_repo=guess_gerrit_remote(), + git_repo_is_gerrit=True, docker_targets=[ # latest will be auto tagged if appropriate DockerTarget('git.lix.systems/lix-project/lix', diff --git a/releng/gitutils.xsh b/releng/gitutils.xsh index e18b4da5f..d3b3774c0 100644 --- a/releng/gitutils.xsh +++ b/releng/gitutils.xsh @@ -1,11 +1,24 @@ import subprocess -import json +from packaging.version import Version from .version import VERSION +def remote_is_plausible(url: str) -> bool: + return ('git.lix.systems' in url and 'lix-project/lix' in url) or ('gerrit.lix.systems' in url and url.endswith('lix')) + + def version_compare(v1: str, v2: str): - return json.loads($(nix-instantiate --eval --json --argstr v1 @(v1) --argstr v2 @(v2) --expr '{v1, v2}: builtins.compareVersions v1 v2')) + v1 = Version(v1) + v2 = Version(v2) + if v1 < v2: + return -1 + elif v1 > v2: + return 1 + elif v1 == v2: + return 0 + else: + raise ValueError('these versions are beyond each others celestial plane') def latest_tag_on_branch(branch: str) -> str: @@ -13,16 +26,18 @@ def latest_tag_on_branch(branch: str) -> str: def is_maintenance_branch(branch: str) -> bool: - try: - main_tag = latest_tag_on_branch('main') - current_tag = latest_tag_on_branch(branch) - - return version_compare(current_tag, main_tag) < 0 - except subprocess.CalledProcessError: - # This is the case before Lix releases 2.90, since main *has* no - # release tag on it. - # FIXME: delete this case after 2.91 - return False + """ + Returns whether the given branch is probably a maintenance branch. + + This uses a heuristic: `main` should have a newer tag than a given + maintenance branch if there has been a major release since that maintenance + branch. + """ + assert remote_is_plausible($(git remote get-url origin).strip()) + main_tag = latest_tag_on_branch('origin/main') + current_tag = latest_tag_on_branch(branch) + + return version_compare(current_tag, main_tag) < 0 def verify_are_on_tag(): diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 99bbc62d5..2450e80c2 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -42,7 +42,7 @@ static std::string makeLockFilename(const std::string & storeUri) { // This avoids issues with the escaped URI being very long and causing // path too long errors, while also avoiding any possibility of collision // caused by simple truncation. - auto hash = hashString(HashType::htSHA256, storeUri).to_string(Base::Base32, false); + auto hash = hashString(HashType::SHA256, storeUri).to_string(Base::Base32, false); return escapeUri(storeUri).substr(0, 48) + "-" + hash.substr(0, 16); } diff --git a/src/libcmd/cmd-profiles.cc b/src/libcmd/cmd-profiles.cc index 8a04100f6..cfce4789e 100644 --- a/src/libcmd/cmd-profiles.cc +++ b/src/libcmd/cmd-profiles.cc @@ -246,7 +246,7 @@ StorePath ProfileManifest::build(ref<Store> store) StringSink sink; sink << dumpPath(tempDir); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashType::SHA256, sink.s); ValidPathInfo info{ *store, diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 0cf4e34b8..6979e3db4 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -96,7 +96,7 @@ static int listPossibleCallback(char * s, char *** avp) return p; }; - vp = check((char **) malloc(possible.size() * sizeof(char *))); + vp = check(static_cast<char **>(malloc(possible.size() * sizeof(char *)))); for (auto & p : possible) vp[ac++] = check(strdup(p.c_str())); diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 5086e9999..b8bfc25eb 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -536,7 +536,7 @@ ProcessLineResult NixRepl::processLine(std::string line) << " :t <expr> Describe result of evaluation\n" << " :u <expr> Build derivation, then start nix-shell\n" << " :doc <expr> Show documentation for the provided function (experimental lambda support)\n" - << " :log <expr> Show logs for a derivation\n" + << " :log <expr | .drv path> Show logs for a derivation\n" << " :te, :trace-enable [bool] Enable, disable or toggle showing traces for\n" << " errors\n" << " :?, :help Brings up this help menu\n" @@ -676,7 +676,49 @@ ProcessLineResult NixRepl::processLine(std::string line) runNix("nix-shell", {state->store->printStorePath(drvPath)}); } - else if (command == ":b" || command == ":bl" || command == ":i" || command == ":sh" || command == ":log") { + else if (command == ":log") { + StorePath drvPath = ([&] { + auto maybeDrvPath = state->store->maybeParseStorePath(arg); + if (maybeDrvPath && maybeDrvPath->isDerivation()) { + return std::move(*maybeDrvPath); + } else { + Value v; + evalString(arg, v); + return getDerivationPath(v); + } + })(); + Path drvPathRaw = state->store->printStorePath(drvPath); + + settings.readOnlyMode = true; + Finally roModeReset([&]() { + settings.readOnlyMode = false; + }); + auto subs = getDefaultSubstituters(); + + subs.push_front(state->store); + + bool foundLog = false; + RunPager pager; + for (auto & sub : subs) { + auto * logSubP = dynamic_cast<LogStore *>(&*sub); + if (!logSubP) { + printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri()); + continue; + } + auto & logSub = *logSubP; + + auto log = logSub.getBuildLog(drvPath); + if (log) { + printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.getUri()); + logger->writeToStdout(*log); + foundLog = true; + break; + } + } + if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw); + } + + else if (command == ":b" || command == ":bl" || command == ":i" || command == ":sh") { Value v; evalString(arg, v); StorePath drvPath = getDerivationPath(v); @@ -712,34 +754,6 @@ ProcessLineResult NixRepl::processLine(std::string line) } } else if (command == ":i") { runNix("nix-env", {"-i", drvPathRaw}); - } else if (command == ":log") { - settings.readOnlyMode = true; - Finally roModeReset([&]() { - settings.readOnlyMode = false; - }); - auto subs = getDefaultSubstituters(); - - subs.push_front(state->store); - - bool foundLog = false; - RunPager pager; - for (auto & sub : subs) { - auto * logSubP = dynamic_cast<LogStore *>(&*sub); - if (!logSubP) { - printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri()); - continue; - } - auto & logSub = *logSubP; - - auto log = logSub.getBuildLog(drvPath); - if (log) { - printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.getUri()); - logger->writeToStdout(*log); - foundLog = true; - break; - } - } - if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw); } else { runNix("nix-shell", {drvPathRaw}); } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 9d35bab81..83bfd4fb0 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -50,7 +50,7 @@ struct AttrDb Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); - Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + Path dbPath = cacheDir + "/" + fingerprint.to_string(Base::Base16, false) + ".sqlite"; state->db = SQLite(dbPath); state->db.isCache(); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a925ce2d8..702b9b6ac 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -21,7 +21,6 @@ #include "flake/flakeref.hh" #include <algorithm> -#include <chrono> #include <iostream> #include <sstream> #include <cstring> @@ -71,11 +70,6 @@ std::string printValue(EvalState & state, Value & v) return out.str(); } -void Value::print(EvalState & state, std::ostream & str, PrintOptions options) -{ - printValue(state, str, *this, options); -} - const Value * getPrimOp(const Value &v) { const Value * primOp = &v; while (primOp->isPrimOpApp()) { @@ -125,32 +119,6 @@ std::string showType(const Value & v) #pragma GCC diagnostic pop } -PosIdx Value::determinePos(const PosIdx pos) const -{ - // Allow selecting a subset of enum values - #pragma GCC diagnostic push - #pragma GCC diagnostic ignored "-Wswitch-enum" - switch (internalType) { - case tAttrs: return attrs->pos; - case tLambda: return lambda.fun->pos; - case tApp: return app.left->determinePos(pos); - default: return pos; - } - #pragma GCC diagnostic pop -} - -bool Value::isTrivial() const -{ - return - internalType != tApp - && internalType != tPrimOpApp - && (internalType != tThunk - || (dynamic_cast<ExprAttrs *>(thunk.expr) - && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) - || dynamic_cast<ExprLambda *>(thunk.expr) - || dynamic_cast<ExprList *>(thunk.expr)); -} - #if HAVE_BOEHMGC /* Called when the Boehm GC runs out of memory. */ @@ -249,6 +217,12 @@ EvalState::EvalState( , sRight(symbols.create("right")) , sWrong(symbols.create("wrong")) , sStructuredAttrs(symbols.create("__structuredAttrs")) + , sAllowedReferences(symbols.create("allowedReferences")) + , sAllowedRequisites(symbols.create("allowedRequisites")) + , sDisallowedReferences(symbols.create("disallowedReferences")) + , sDisallowedRequisites(symbols.create("disallowedRequisites")) + , sMaxSize(symbols.create("maxSize")) + , sMaxClosureSize(symbols.create("maxClosureSize")) , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) , sContentAddressed(symbols.create("__contentAddressed")) @@ -493,36 +467,6 @@ std::ostream & operator<<(std::ostream & output, PrimOp & primOp) return output; } - -Value::Value(primop_t, PrimOp & primop) - : internalType(tPrimOp) - , primOp(&primop) - , _primop_pad(0) -{ - primop.check(); -} - -PrimOp * Value::primOpAppPrimOp() const -{ - Value * left = primOpApp.left; - while (left && !left->isPrimOp()) { - left = left->primOpApp.left; - } - - if (!left) - return nullptr; - return left->primOp; -} - -void Value::mkPrimOp(PrimOp * p) -{ - p->check(); - clearValue(); - internalType = tPrimOp; - primOp = p; -} - - Value * EvalState::addPrimOp(PrimOp && primOp) { /* Hack to make constants lazy: turn them into a application of @@ -774,42 +718,6 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t) evalState.runDebugRepl(nullptr, trace.env, trace.expr); } -void Value::mkString(std::string_view s) -{ - mkString(gcCopyStringIfNeeded(s)); -} - - -static void copyContextToValue(Value & v, const NixStringContext & context) -{ - if (!context.empty()) { - size_t n = 0; - v.string.context = gcAllocType<char const *>(context.size() + 1); - for (auto & i : context) - v.string.context[n++] = gcCopyStringIfNeeded(i.to_string()); - v.string.context[n] = 0; - } -} - -void Value::mkString(std::string_view s, const NixStringContext & context) -{ - mkString(s); - copyContextToValue(*this, context); -} - -void Value::mkStringMove(const char * s, const NixStringContext & context) -{ - mkString(s); - copyContextToValue(*this, context); -} - - -void Value::mkPath(const SourcePath & path) -{ - mkPath(gcCopyStringIfNeeded(path.path.abs())); -} - - inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) { for (auto l = var.level; l; --l, env = env->up) ; @@ -2802,20 +2710,20 @@ Expr & EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<Sta } -Expr & EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv) +Expr & EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv, const ExperimentalFeatureSettings & xpSettings) { // NOTE this method (and parseStdin) must take care to *fully copy* their input // into their respective Pos::Origin until the parser stops overwriting its input // data. auto s = make_ref<std::string>(s_); s_.append("\0\0", 2); - return *parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv); + return *parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv, xpSettings); } -Expr & EvalState::parseExprFromString(std::string s, const SourcePath & basePath) +Expr & EvalState::parseExprFromString(std::string s, const SourcePath & basePath, const ExperimentalFeatureSettings & xpSettings) { - return parseExprFromString(std::move(s), basePath, staticBaseEnv); + return parseExprFromString(std::move(s), basePath, staticBaseEnv, xpSettings); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e54eede40..ff45efc08 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -13,7 +13,6 @@ #include "search-path.hh" #include "repl-exit-status.hh" -#include <gc/gc_allocator.h> #include <map> #include <optional> #include <unordered_map> @@ -162,7 +161,10 @@ public: const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sRight, sWrong, sStructuredAttrs, + sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites, + sMaxSize, sMaxClosureSize, + sBuilder, sArgs, sContentAddressed, sImpure, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, @@ -342,8 +344,8 @@ public: /** * Parse a Nix expression from the specified string. */ - Expr & parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv); - Expr & parseExprFromString(std::string s, const SourcePath & basePath); + Expr & parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + Expr & parseExprFromString(std::string s, const SourcePath & basePath, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); Expr & parseStdin(); @@ -566,7 +568,8 @@ private: size_t length, Pos::Origin origin, const SourcePath & basePath, - std::shared_ptr<StaticEnv> & staticEnv); + std::shared_ptr<StaticEnv> & staticEnv, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run out of system stack. diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 7379fc8fb..7f0e5b1e9 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -937,7 +937,7 @@ Fingerprint LockedFlake::getFingerprint() const // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. - return hashString(htSHA256, + return hashString(HashType::SHA256, fmt("%s;%s;%d;%d;%s", flake.sourceInfo->storePath.to_string(), flake.lockedRef.subdir, diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index b11e4c2e0..af72bd549 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -31,6 +31,7 @@ libexpr_sources = files( 'print-ambiguous.cc', 'print.cc', 'search-path.cc', + 'value.cc', 'value-to-json.cc', 'value-to-xml.cc', 'flake/config.cc', diff --git a/src/libexpr/parser/grammar.hh b/src/libexpr/parser/grammar.hh index 82df63bc5..2c5a3d1be 100644 --- a/src/libexpr/parser/grammar.hh +++ b/src/libexpr/parser/grammar.hh @@ -434,6 +434,8 @@ struct op { struct and_ : _op<TAO_PEGTL_STRING("&&"), 12> {}; struct or_ : _op<TAO_PEGTL_STRING("||"), 13> {}; struct implies : _op<TAO_PEGTL_STRING("->"), 14, kind::rightAssoc> {}; + struct pipe_right : _op<TAO_PEGTL_STRING("|>"), 15> {}; + struct pipe_left : _op<TAO_PEGTL_STRING("<|"), 16, kind::rightAssoc> {}; }; struct _expr { @@ -521,6 +523,7 @@ struct _expr { app > {}; + /* Order matters here. The order is the parsing order, not the precedence order: '<=' must be parsed before '<'. */ struct _binary_operator : sor< operator_<op::implies>, operator_<op::update>, @@ -529,6 +532,8 @@ struct _expr { operator_<op::minus>, operator_<op::mul>, operator_<op::div>, + operator_<op::pipe_right>, + operator_<op::pipe_left>, operator_<op::less_eq>, operator_<op::greater_eq>, operator_<op::less>, @@ -649,6 +654,8 @@ struct operator_semantics { grammar::op::minus, grammar::op::mul, grammar::op::div, + grammar::op::pipe_right, + grammar::op::pipe_left, has_attr > op; }; diff --git a/src/libexpr/parser/parser.cc b/src/libexpr/parser/parser.cc index b7a105fe7..6d496d141 100644 --- a/src/libexpr/parser/parser.cc +++ b/src/libexpr/parser/parser.cc @@ -113,6 +113,29 @@ struct ExprState return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args)); } + std::unique_ptr<Expr> pipe(PosIdx pos, State & state, bool flip = false) + { + if (!state.xpSettings.isEnabled(Xp::PipeOperator)) + throw ParseError({ + .msg = HintFmt("Pipe operator is disabled"), + .pos = state.positions[pos] + }); + + // Reverse the order compared to normal function application: arg |> fn + std::unique_ptr<Expr> fn, arg; + if (flip) { + fn = popExprOnly(); + arg = popExprOnly(); + } else { + arg = popExprOnly(); + fn = popExprOnly(); + } + std::vector<std::unique_ptr<Expr>> args{1}; + args[0] = std::move(arg); + + return std::make_unique<ExprCall>(pos, std::move(fn), std::move(args)); + } + std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state) { return call(pos, state.s.lessThan, !less); @@ -162,6 +185,8 @@ struct ExprState [&] (Op::concat) { return applyBinary<ExprOpConcatLists>(pos); }, [&] (has_attr & a) { return applyUnary<ExprOpHasAttr>(std::move(a.path)); }, [&] (Op::unary_minus) { return negate(pos, state); }, + [&] (Op::pipe_right) { return pipe(pos, state, true); }, + [&] (Op::pipe_left) { return pipe(pos, state); }, })(op) }; } @@ -631,7 +656,7 @@ template<> struct BuildAST<grammar::expr::path> : p::maybe_nothing {}; template<> struct BuildAST<grammar::expr::uri> { static void apply(const auto & in, ExprState & s, State & ps) { - static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); + bool noURLLiterals = ps.xpSettings.isEnabled(Xp::NoUrlLiterals); if (noURLLiterals) throw ParseError({ .msg = HintFmt("URL literals are disabled"), @@ -832,7 +857,8 @@ Expr * EvalState::parse( size_t length, Pos::Origin origin, const SourcePath & basePath, - std::shared_ptr<StaticEnv> & staticEnv) + std::shared_ptr<StaticEnv> & staticEnv, + const ExperimentalFeatureSettings & xpSettings) { parser::State s = { symbols, @@ -840,6 +866,7 @@ Expr * EvalState::parse( basePath, positions.addOrigin(origin, length), exprSymbols, + xpSettings }; parser::ExprState x; diff --git a/src/libexpr/parser/state.hh b/src/libexpr/parser/state.hh index 29889152e..30803a37e 100644 --- a/src/libexpr/parser/state.hh +++ b/src/libexpr/parser/state.hh @@ -19,6 +19,7 @@ struct State SourcePath basePath; PosTable::Origin origin; const Expr::AstSymbols & s; + const ExperimentalFeatureSettings & xpSettings; void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0951a54de..dab96d6d4 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -346,7 +346,7 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu state.error<EvalError>("could not open '%1%': %2%", path, dlerror()).debugThrow(); dlerror(); - ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); + ValueInitializer func = reinterpret_cast<ValueInitializer>(dlsym(handle, sym.c_str())); if(!func) { char *message = dlerror(); if (message) @@ -1213,6 +1213,20 @@ drvName, Bindings * attrs, Value & v) handleOutputs(ss); } + if (i->name == state.sAllowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks.<output>.allowedReferences' instead", drvName); + if (i->name == state.sAllowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks.<output>.allowedRequisites' instead", drvName); + if (i->name == state.sDisallowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks.<output>.disallowedReferences' instead", drvName); + if (i->name == state.sDisallowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks.<output>.disallowedRequisites' instead", drvName); + if (i->name == state.sMaxSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks.<output>.maxSize' instead", drvName); + if (i->name == state.sMaxClosureSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks.<output>.maxClosureSize' instead", drvName); + + } else { auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); drv.env.emplace(key, s); @@ -1322,7 +1336,7 @@ drvName, Bindings * attrs, Value & v) state.error<EvalError>("derivation cannot be both content-addressed and impure") .atPos(v).debugThrow(); - auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256); + auto ht = parseHashTypeOpt(outputHashAlgo).value_or(HashType::SHA256); auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); for (auto & i : outputs) { @@ -1750,7 +1764,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = realisePath(state, pos, *args[1]); - v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false)); + v.mkString(hashString(*ht, path.readFile()).to_string(Base::Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -2332,7 +2346,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value else if (n == "recursive") method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") }; else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashType::SHA256); else state.error<EvalError>( "unsupported argument '%1%' to 'addPath'", @@ -2425,6 +2439,8 @@ static void prim_attrValues(EvalState & state, const PosIdx pos, Value * * args, state.mkList(v, args[0]->attrs->size()); + // FIXME: this is incredibly evil, *why* + // NOLINTBEGIN(cppcoreguidelines-pro-type-cstyle-cast) unsigned int n = 0; for (auto & i : *args[0]->attrs) v.listElems()[n++] = (Value *) &i; @@ -2438,6 +2454,7 @@ static void prim_attrValues(EvalState & state, const PosIdx pos, Value * * args, for (unsigned int i = 0; i < n; ++i) v.listElems()[i] = ((Attr *) v.listElems()[i])->value; + // NOLINTEND(cppcoreguidelines-pro-type-cstyle-cast) } static RegisterPrimOp primop_attrValues({ @@ -3847,7 +3864,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); - v.mkString(hashString(*ht, s).to_string(Base16, false)); + v.mkString(hashString(*ht, s).to_string(Base::Base16, false)); } static RegisterPrimOp primop_hashString({ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 261722d1b..2031be299 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a // be both a revision or a branch/tag name. auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial"); if (std::regex_match(value.begin(), value.end(), revRegex)) - rev = Hash::parseAny(value, htSHA1); + rev = Hash::parseAny(value, HashType::SHA1); else ref = value; } @@ -73,7 +73,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2.getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(HashType::SHA1)); attrs2.alloc("rev").mkString(rev2.gitRev()); attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12)); if (auto revCount = input2.getRevCount()) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e289fe9ca..b0e14a26e 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -32,7 +32,7 @@ void emitTreeAttrs( auto narHash = input.getNarHash(); assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); + attrs.alloc("narHash").mkString(narHash->to_string(Base::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( @@ -45,7 +45,7 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(rev->gitShortRev()); } else if (emptyRevFallback) { // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev - auto emptyHash = Hash(htSHA1); + auto emptyHash = Hash(HashType::SHA1); attrs.alloc("rev").mkString(emptyHash.gitRev()); attrs.alloc("shortRev").mkString(emptyHash.gitShortRev()); } @@ -226,7 +226,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (n == "url") url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashType::SHA256); else if (n == "name") name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); else @@ -252,7 +252,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.error<EvalError>("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow(); // early exit if pinned and already in the store - if (expectedHash && expectedHash->type == htSHA256) { + if (expectedHash && expectedHash->type == HashType::SHA256) { auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { @@ -277,13 +277,13 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, state.store->toRealPath(storePath)); + : hashFile(HashType::SHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) { state.error<EvalError>( "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, - expectedHash->to_string(Base32, true), - hash.to_string(Base32, true) + expectedHash->to_string(Base::Base32, true), + hash.to_string(Base::Base32, true) ).withExitStatus(102) .debugThrow(); } diff --git a/src/libexpr/value.cc b/src/libexpr/value.cc new file mode 100644 index 000000000..7e172f989 --- /dev/null +++ b/src/libexpr/value.cc @@ -0,0 +1,106 @@ +#include "value.hh" + +#include <ostream> + +#include "eval.hh" +#include "print.hh" + + +namespace nix +{ + +static void copyContextToValue(Value & v, const NixStringContext & context) +{ + if (!context.empty()) { + size_t n = 0; + v.string.context = gcAllocType<char const *>(context.size() + 1); + for (auto & i : context) + v.string.context[n++] = gcCopyStringIfNeeded(i.to_string()); + v.string.context[n] = 0; + } +} + +Value::Value(primop_t, PrimOp & primop) + : internalType(tPrimOp) + , primOp(&primop) + , _primop_pad(0) +{ + primop.check(); +} + + +void Value::print(EvalState & state, std::ostream & str, PrintOptions options) +{ + printValue(state, str, *this, options); +} + +PosIdx Value::determinePos(const PosIdx pos) const +{ + // Allow selecting a subset of enum values + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" + switch (internalType) { + case tAttrs: return attrs->pos; + case tLambda: return lambda.fun->pos; + case tApp: return app.left->determinePos(pos); + default: return pos; + } + #pragma GCC diagnostic pop +} + +bool Value::isTrivial() const +{ + return + internalType != tApp + && internalType != tPrimOpApp + && (internalType != tThunk + || (dynamic_cast<ExprAttrs *>(thunk.expr) + && static_cast<ExprAttrs *>(thunk.expr)->dynamicAttrs.empty()) + || dynamic_cast<ExprLambda *>(thunk.expr) + || dynamic_cast<ExprList *>(thunk.expr)); +} + +PrimOp * Value::primOpAppPrimOp() const +{ + Value * left = primOpApp.left; + while (left && !left->isPrimOp()) { + left = left->primOpApp.left; + } + + if (!left) + return nullptr; + return left->primOp; +} + +void Value::mkPrimOp(PrimOp * p) +{ + p->check(); + clearValue(); + internalType = tPrimOp; + primOp = p; +} + +void Value::mkString(std::string_view s) +{ + mkString(gcCopyStringIfNeeded(s)); +} + +void Value::mkString(std::string_view s, const NixStringContext & context) +{ + mkString(s); + copyContextToValue(*this, context); +} + +void Value::mkStringMove(const char * s, const NixStringContext & context) +{ + mkString(s); + copyContextToValue(*this, context); +} + + +void Value::mkPath(const SourcePath & path) +{ + mkPath(gcCopyStringIfNeeded(path.path.abs())); +} + +} diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index f830a4963..b9f828d9a 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -18,8 +18,8 @@ StorePath fetchToStore( return settings.readOnlyMode - ? store.computeStorePathForPath(name, path.path.abs(), method, htSHA256, filter2).first - : store.addToStore(name, path.path.abs(), method, htSHA256, filter2, repair); + ? store.computeStorePathForPath(name, path.path.abs(), method, HashType::SHA256, filter2).first + : store.addToStore(name, path.path.abs(), method, HashType::SHA256, filter2, repair); } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e09513d8f..c441ffb12 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -153,12 +153,12 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const }; auto narHash = store->queryPathInfo(tree.storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true)); + input.attrs.insert_or_assign("narHash", narHash.to_string(Base::SRI, true)); if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true)); + to_string(), tree.actualPath, prevNarHash->to_string(Base::SRI, true), narHash.to_string(Base::SRI, true)); } if (auto prevLastModified = getLastModified()) { @@ -240,8 +240,8 @@ std::string Input::getType() const std::optional<Hash> Input::getNarHash() const { if (auto s = maybeGetStrAttr(attrs, "narHash")) { - auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s); - if (hash.type != htSHA256) + auto hash = s->empty() ? Hash(HashType::SHA256) : Hash::parseSRI(*s); + if (hash.type != HashType::SHA256) throw UsageError("narHash must use SHA-256"); return hash; } @@ -264,7 +264,7 @@ std::optional<Hash> Input::getRev() const hash = Hash::parseAnyPrefixed(*s); } catch (BadHash &e) { // Default to sha1 for backwards compatibility with existing flakes - hash = Hash::parseAny(*s, htSHA1); + hash = Hash::parseAny(*s, HashType::SHA1); } } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8e3165ff6..7d16d3f57 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -49,7 +49,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(htSHA256, key).to_string(Base32, false); + hashString(HashType::SHA256, key).to_string(Base::Base32, false); } // Returns the name of the HEAD branch. @@ -238,7 +238,7 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashType::SHA256, filter); // FIXME: maybe we should use the timestamp of the last // modified dirty file? @@ -437,8 +437,8 @@ struct GitInputScheme : InputScheme auto checkHashType = [&](const std::optional<Hash> & hash) { - if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) - throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); + if (hash.has_value() && !(hash->type == HashType::SHA1 || hash->type == HashType::SHA256)) + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base::Base16, true)); }; auto getLockedAttrs = [&]() @@ -501,7 +501,7 @@ struct GitInputScheme : InputScheme if (!input.getRev()) input.attrs.insert_or_assign("rev", - Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev()); + Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), HashType::SHA1).gitRev()); repoDir = actualUrl; } else { @@ -521,7 +521,7 @@ struct GitInputScheme : InputScheme } if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); + auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashType::SHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); @@ -599,7 +599,7 @@ struct GitInputScheme : InputScheme } if (!input.getRev()) - input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), HashType::SHA1).gitRev()); // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } @@ -695,7 +695,7 @@ struct GitInputScheme : InputScheme unpackTarfile(*proc.getStdout(), tmpDir); } - auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, HashType::SHA256, filter); auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index b971781ae..32032258a 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -149,7 +149,7 @@ struct GitArchiveInputScheme : InputScheme auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; - if (rev) path += "/" + rev->to_string(Base16, false); + if (rev) path += "/" + rev->to_string(Base::Base16, false); return ParsedURL { .scheme = type(), .path = path, @@ -274,7 +274,7 @@ struct GitHubInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); + auto rev = Hash::parseAny(std::string { json["sha"] }, HashType::SHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } @@ -295,7 +295,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://api.%s/repos/%s/%s/tarball/%s"; const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(Base::Base16, false)); return DownloadUrl { url, headers }; } @@ -347,7 +347,7 @@ struct GitLabInputScheme : GitArchiveInputScheme store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { - auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1); + auto rev = Hash::parseAny(std::string(json[0]["id"]), HashType::SHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } else if (json.is_array() && json.size() == 0) { @@ -367,7 +367,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(Base::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; @@ -444,7 +444,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme if(!id) throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); - auto rev = Hash::parseAny(*id, htSHA1); + auto rev = Hash::parseAny(*id, HashType::SHA1); debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev()); return rev; } @@ -454,7 +454,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(Base::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 8c0176e84..f3912b82f 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -22,14 +22,14 @@ struct IndirectInputScheme : InputScheme if (path.size() == 1) { } else if (path.size() == 2) { if (std::regex_match(path[1], revRegex)) - rev = Hash::parseAny(path[1], htSHA1); + rev = Hash::parseAny(path[1], HashType::SHA1); else if (std::regex_match(path[1], refRegex)) ref = path[1]; else throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); } else if (path.size() == 3) { ref = path[1]; - rev = Hash::parseAny(path[2], htSHA1); + rev = Hash::parseAny(path[2], HashType::SHA1); } else throw BadURL("GitHub URL '%s' is invalid", url.url); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index b4150e9df..a1e4c1918 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -198,7 +198,7 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashType::SHA256, filter); return {std::move(storePath), input}; } @@ -208,8 +208,8 @@ struct MercurialInputScheme : InputScheme auto checkHashType = [&](const std::optional<Hash> & hash) { - if (hash.has_value() && hash->type != htSHA1) - throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); + if (hash.has_value() && hash->type != HashType::SHA1) + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base::Base16, true)); }; @@ -248,14 +248,14 @@ struct MercurialInputScheme : InputScheme }); if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); + auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashType::SHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashType::SHA256, actualUrl).to_string(Base::Base32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ @@ -289,7 +289,7 @@ struct MercurialInputScheme : InputScheme runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev()); + input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], HashType::SHA1).gitRev()); auto revCount = std::stoull(tokens[1]); input.attrs.insert_or_assign("ref", tokens[2]); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index b11665805..4c30193e2 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -72,7 +72,7 @@ DownloadFileResult downloadFile( } else { StringSink sink; sink << dumpString(res.data); - auto hash = hashString(htSHA256, res.data); + auto hash = hashString(HashType::SHA256, res.data); ValidPathInfo info { *store, name, @@ -81,7 +81,7 @@ DownloadFileResult downloadFile( .hash = hash, .references = {}, }, - hashString(htSHA256, sink.s), + hashString(HashType::SHA256, sink.s), }; info.narSize = sink.s.size(); auto source = StringSource { sink.s }; @@ -155,7 +155,7 @@ DownloadTarballResult downloadTarball( throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, HashType::SHA256, defaultPathFilter, NoRepair); } Attrs infoAttrs({ @@ -238,7 +238,7 @@ struct CurlInputScheme : InputScheme // NAR hashes are preferred over file hashes since tar/zip // files don't have a canonical representation. if (auto narHash = input.getNarHash()) - url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + url.query.insert_or_assign("narHash", narHash->to_string(Base::SRI, true)); return url; } diff --git a/src/libmain/stack.cc b/src/libmain/stack.cc index 10f71c1dc..493829b55 100644 --- a/src/libmain/stack.cc +++ b/src/libmain/stack.cc @@ -2,8 +2,6 @@ #include "shared.hh" #include <cstring> -#include <cstddef> -#include <cstdlib> #include <unistd.h> #include <signal.h> @@ -17,17 +15,17 @@ static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) the stack pointer. Unfortunately, getting the stack pointer is not portable. */ bool haveSP = true; - char * sp = 0; + int64_t sp = 0; #if defined(__x86_64__) && defined(REG_RSP) - sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_RSP]; + sp = static_cast<ucontext_t *>(ctx)->uc_mcontext.gregs[REG_RSP]; #elif defined(REG_ESP) - sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_ESP]; + sp = static_cast<ucontext_t *>(ctx)->uc_mcontext.gregs[REG_ESP]; #else haveSP = false; #endif if (haveSP) { - ptrdiff_t diff = (char *) info->si_addr - sp; + int64_t diff = int64_t(info->si_addr) - sp; if (diff < 0) diff = -diff; if (diff < 4096) { nix::stackOverflowHandler(info, ctx); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 6ab5a0a1c..fc0569a66 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -3,17 +3,15 @@ #include "compression.hh" #include "derivations.hh" #include "fs-accessor.hh" -#include "globals.hh" #include "nar-info.hh" #include "sync.hh" #include "remote-fs-accessor.hh" -#include "nar-info-disk-cache.hh" +#include "nar-info-disk-cache.hh" // IWYU pragma: keep #include "nar-accessor.hh" #include "thread-pool.hh" #include "signals.hh" #include <chrono> -#include <future> #include <regex> #include <fstream> #include <sstream> @@ -128,9 +126,9 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon( /* Read the NAR simultaneously into a CompressionSink+FileSink (to write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ - HashSink fileHashSink { htSHA256 }; + HashSink fileHashSink { HashType::SHA256 }; std::shared_ptr<FSAccessor> narAccessor; - HashSink narHashSink { htSHA256 }; + HashSink narHashSink { HashType::SHA256 }; { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; @@ -150,7 +148,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(Base::Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "zstd" ? ".zst" : @@ -288,7 +286,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name, FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) + if (method != FileIngestionMethod::Recursive || hashAlgo != HashType::SHA256) unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { @@ -425,7 +423,7 @@ StorePath BinaryCacheStore::addTextToStore( const StorePathSet & references, RepairFlag repair) { - auto textHash = hashString(htSHA256, s); + auto textHash = hashString(HashType::SHA256, s); auto path = makeTextPath(name, TextInfo { { textHash }, references }); if (!repair && isValidPath(path)) @@ -480,7 +478,8 @@ void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSe when addSignatures() is called sequentially on a path, because S3 might return an outdated cached version. */ - auto narInfo = make_ref<NarInfo>((NarInfo &) *queryPathInfo(storePath)); + // downcast: BinaryCacheStore always returns NarInfo from queryPathInfoUncached, making it sound + auto narInfo = make_ref<NarInfo>(dynamic_cast<NarInfo const &>(*queryPathInfo(storePath))); narInfo->sigs.insert(sigs.begin(), sigs.end()); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 17a2b04f1..aa89f9e7d 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -174,10 +174,9 @@ Goal::WorkResult DerivationGoal::getDerivation() return loadDerivation(); } - addWaitee(worker.makePathSubstitutionGoal(drvPath)); state = &DerivationGoal::loadDerivation; - return StillAlive{}; + return WaitForGoals{{worker.makePathSubstitutionGoal(drvPath)}}; } @@ -268,11 +267,12 @@ Goal::WorkResult DerivationGoal::haveDerivation() /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ + WaitForGoals result; if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) for (auto & [outputName, status] : initialOutputs) { if (!status.wanted) continue; if (!status.known) - addWaitee( + result.goals.insert( worker.makeDrvOutputSubstitutionGoal( DrvOutput{status.outputHash, outputName}, buildMode == bmRepair ? Repair : NoRepair @@ -280,18 +280,18 @@ Goal::WorkResult DerivationGoal::haveDerivation() ); else { auto * cap = getDerivationCA(*drv); - addWaitee(worker.makePathSubstitutionGoal( + result.goals.insert(worker.makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, cap ? std::optional { *cap } : std::nullopt)); } } - if (waitees.empty()) { /* to prevent hang (no wake-up event) */ + if (result.goals.empty()) { /* to prevent hang (no wake-up event) */ return outputsSubstitutionTried(); } else { state = &DerivationGoal::outputsSubstitutionTried; - return StillAlive{}; + return result; } } @@ -362,6 +362,8 @@ Goal::WorkResult DerivationGoal::outputsSubstitutionTried() produced using a substitute. So we have to build instead. */ Goal::WorkResult DerivationGoal::gaveUpOnSubstitution() { + WaitForGoals result; + /* At this point we are building all outputs, so if more are wanted there is no need to restart. */ needRestart = NeedRestartForMoreOutputs::BuildInProgressWillNotNeed; @@ -373,7 +375,7 @@ Goal::WorkResult DerivationGoal::gaveUpOnSubstitution() addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) { if (!inputNode.value.empty()) - addWaitee(worker.makeGoal( + result.goals.insert(worker.makeGoal( DerivedPath::Built { .drvPath = inputDrv, .outputs = inputNode.value, @@ -418,14 +420,14 @@ Goal::WorkResult DerivationGoal::gaveUpOnSubstitution() if (!settings.useSubstitutes) throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - addWaitee(worker.makePathSubstitutionGoal(i)); + result.goals.insert(worker.makePathSubstitutionGoal(i)); } - if (waitees.empty()) {/* to prevent hang (no wake-up event) */ + if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ return inputsRealised(); } else { state = &DerivationGoal::inputsRealised; - return StillAlive{}; + return result; } } @@ -466,6 +468,7 @@ Goal::WorkResult DerivationGoal::repairClosure() } /* Check each path (slow!). */ + WaitForGoals result; for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; printError( @@ -473,9 +476,9 @@ Goal::WorkResult DerivationGoal::repairClosure() worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - addWaitee(worker.makePathSubstitutionGoal(i, Repair)); + result.goals.insert(worker.makePathSubstitutionGoal(i, Repair)); else - addWaitee(worker.makeGoal( + result.goals.insert(worker.makeGoal( DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath2->second), .outputs = OutputsSpec::All { }, @@ -483,12 +486,12 @@ Goal::WorkResult DerivationGoal::repairClosure() bmRepair)); } - if (waitees.empty()) { + if (result.goals.empty()) { return done(BuildResult::AlreadyValid, assertPathValidity()); } state = &DerivationGoal::closureRepaired; - return StillAlive{}; + return result; } @@ -580,10 +583,9 @@ Goal::WorkResult DerivationGoal::inputsRealised() resolvedDrvGoal = worker.makeDerivationGoal( pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedDrvGoal); state = &DerivationGoal::resolvedFinished; - return StillAlive{}; + return WaitForGoals{{resolvedDrvGoal}}; } std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths; @@ -648,8 +650,7 @@ Goal::WorkResult DerivationGoal::inputsRealised() slot to become available, since we don't need one if there is a build hook. */ state = &DerivationGoal::tryToBuild; - worker.wakeUp(shared_from_this()); - return StillAlive{}; + return ContinueImmediately{}; } Goal::WorkResult DerivationGoal::started() @@ -701,8 +702,7 @@ Goal::WorkResult DerivationGoal::tryToBuild() if (!actLock) actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); - worker.waitForAWhile(shared_from_this()); - return StillAlive{}; + return WaitForAWhile{}; } actLock.reset(); @@ -753,9 +753,8 @@ Goal::WorkResult DerivationGoal::tryToBuild() if (!actLock) actLock = std::make_unique<Activity>(*logger, lvlTalkative, actBuildWaiting, fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); - return StillAlive{}; + return WaitForAWhile{}; case rpDecline: /* We should do it ourselves. */ break; @@ -765,8 +764,7 @@ Goal::WorkResult DerivationGoal::tryToBuild() actLock.reset(); state = &DerivationGoal::tryLocalBuild; - worker.wakeUp(shared_from_this()); - return StillAlive{}; + return ContinueImmediately{}; } Goal::WorkResult DerivationGoal::tryLocalBuild() { @@ -1508,10 +1506,6 @@ Goal::Finished DerivationGoal::done( buildResult.status = status; if (ex) buildResult.errorMsg = fmt("%s", Uncolored(ex->info().msg)); - if (buildResult.status == BuildResult::TimedOut) - worker.timedOut = true; - if (buildResult.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; mcExpectedBuilds.reset(); mcRunningBuilds.reset(); @@ -1537,6 +1531,10 @@ Goal::Finished DerivationGoal::done( return Finished{ .result = buildResult.success() ? ecSuccess : ecFailed, .ex = ex ? std::make_unique<Error>(std::move(*ex)) : nullptr, + .permanentFailure = buildResult.status == BuildResult::PermanentFailure, + .timedOut = buildResult.status == BuildResult::TimedOut, + .hashMismatch = anyHashMismatchSeen, + .checkMismatch = anyCheckMismatchSeen, }; } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index c43e2aed5..257282308 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -107,6 +107,9 @@ struct DerivationGoal : public Goal */ NeedRestartForMoreOutputs needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; + bool anyHashMismatchSeen = false; + bool anyCheckMismatchSeen = false; + /** * See `retrySubstitution`; just for that field. */ diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 62e86e1cc..b41dae5d6 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -41,8 +41,7 @@ Goal::WorkResult DrvOutputSubstitutionGoal::tryNext() if maxSubstitutionJobs == 0, we still allow a substituter to run. This prevents infinite waiting. */ if (worker.runningSubstitutions >= std::max(1U, settings.maxSubstitutionJobs.get())) { - worker.waitForBuildSlot(shared_from_this()); - return StillAlive{}; + return WaitForSlot{}; } maintainRunningSubstitutions = @@ -101,6 +100,7 @@ Goal::WorkResult DrvOutputSubstitutionGoal::realisationFetched() return tryNext(); } + WaitForGoals result; for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { if (depId != id) { if (auto localOutputInfo = worker.store.queryRealisation(depId); @@ -116,17 +116,17 @@ Goal::WorkResult DrvOutputSubstitutionGoal::realisationFetched() ); return tryNext(); } - addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); + result.goals.insert(worker.makeDrvOutputSubstitutionGoal(depId)); } } - addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); + result.goals.insert(worker.makePathSubstitutionGoal(outputInfo->outPath)); - if (waitees.empty()) { + if (result.goals.empty()) { return outPathValid(); } else { state = &DrvOutputSubstitutionGoal::outPathValid; - return StillAlive{}; + return result; } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index f26c2c671..a4e66d989 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -11,13 +11,6 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { } -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - waitee->waiters.insert(shared_from_this()); -} - - void Goal::trace(std::string_view s) { debug("%1%: %2%", name, s); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index dd29b9fc4..5d7bb72b8 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -106,12 +106,28 @@ struct Goal : public std::enable_shared_from_this<Goal> public: struct [[nodiscard]] StillAlive {}; + struct [[nodiscard]] WaitForSlot {}; + struct [[nodiscard]] WaitForAWhile {}; + struct [[nodiscard]] ContinueImmediately {}; + struct [[nodiscard]] WaitForGoals { + Goals goals; + }; struct [[nodiscard]] Finished { ExitCode result; std::unique_ptr<Error> ex; + bool permanentFailure = false; + bool timedOut = false; + bool hashMismatch = false; + bool checkMismatch = false; }; - struct [[nodiscard]] WorkResult : std::variant<StillAlive, Finished> + struct [[nodiscard]] WorkResult : std::variant< + StillAlive, + WaitForSlot, + WaitForAWhile, + ContinueImmediately, + WaitForGoals, + Finished> { WorkResult() = delete; using variant::variant; @@ -133,8 +149,6 @@ public: virtual WorkResult work() = 0; - void addWaitee(GoalPtr waitee); - virtual void waiteeDone(GoalPtr waitee) { } virtual WorkResult handleChildOutput(int fd, std::string_view data) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 23903117a..8b640e4ad 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -158,9 +158,8 @@ Goal::WorkResult LocalDerivationGoal::tryLocalBuild() unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { state = &DerivationGoal::tryToBuild; - worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); - return StillAlive{}; + return WaitForSlot{}; } assert(derivationType); @@ -202,8 +201,7 @@ Goal::WorkResult LocalDerivationGoal::tryLocalBuild() if (!actLock) actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - worker.waitForAWhile(shared_from_this()); - return StillAlive{}; + return WaitForAWhile{}; } } @@ -237,8 +235,9 @@ Goal::WorkResult LocalDerivationGoal::tryLocalBuild() } catch (BuildError & e) { outputLocks.unlock(); buildUser.reset(); - worker.permanentFailure = true; - return done(BuildResult::InputRejected, {}, std::move(e)); + auto report = done(BuildResult::InputRejected, {}, std::move(e)); + report.permanentFailure = true; + return report; } /* This state will be reached when we get EOF on the child's @@ -818,8 +817,8 @@ void LocalDerivationGoal::initTmpDir() { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - auto hash = hashString(htSHA256, i.first); - std::string fn = ".attr-" + hash.to_string(Base32, false); + auto hash = hashString(HashType::SHA256, i.first); + std::string fn = ".attr-" + hash.to_string(Base::Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -1226,7 +1225,7 @@ void LocalDerivationGoal::startDaemon() socklen_t remoteAddrLen = sizeof(remoteAddr); AutoCloseFD remote{accept(daemonSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen)}; + reinterpret_cast<struct sockaddr *>(&remoteAddr), &remoteAddrLen)}; if (!remote) { if (errno == EINTR || errno == EAGAIN) continue; if (errno == EINVAL || errno == ECONNABORTED) break; @@ -2147,7 +2146,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string(newInfo0.path.hashPart())}}); } - HashResult narHashAndSize = hashPath(htSHA256, actualPath); + HashResult narHashAndSize = hashPath(HashType::SHA256, actualPath); newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; @@ -2167,7 +2166,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); rewriteOutput(outputRewrites); - auto narHashAndSize = hashPath(htSHA256, actualPath); + auto narHashAndSize = hashPath(HashType::SHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); @@ -2197,15 +2196,15 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (wanted != got) { /* Throw an error after registering the path as valid. */ - worker.hashMismatch = true; + anyHashMismatchSeen = true; // XXX: shameless layering violation hack that makes the hash mismatch error at least not utterly worthless auto guessedUrl = getOr(drv->env, "urls", getOr(drv->env, "url", "(unknown)")); delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n likely URL: %s\n specified: %s\n got: %s", worker.store.printStorePath(drvPath), guessedUrl, - wanted.to_string(SRI, true), - got.to_string(SRI, true))); + wanted.to_string(Base::SRI, true), + got.to_string(Base::SRI, true))); } if (!newInfo0.references.empty()) delayedException = std::make_exception_ptr( @@ -2284,7 +2283,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (!worker.store.isValidPath(newInfo.path)) continue; ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path)); if (newInfo.narHash != oldInfo.narHash) { - worker.checkMismatch = true; + anyCheckMismatchSeen = true; if (settings.runDiffHook || settings.keepFailed) { auto dst = worker.store.toRealPath(finalDestPath + checkSuffix); deletePath(dst); @@ -2517,6 +2516,24 @@ void LocalDerivationGoal::checkOutputs(const std::map<std::string, ValidPathInfo }; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + if (get(*structuredAttrs, "allowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "allowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxClosureSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead"); + } if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { if (auto output = get(*outputChecks, outputName)) { Checks checks; @@ -2590,7 +2607,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); + Hash(HashType::SHA256), outputPathName(drv->name, outputName)); } @@ -2598,7 +2615,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); + Hash(HashType::SHA256), path.name()); } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index fb2949fd0..67a5f20bb 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -152,15 +152,16 @@ Goal::WorkResult PathSubstitutionGoal::tryNext() /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ + WaitForGoals result; for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); + result.goals.insert(worker.makePathSubstitutionGoal(i)); - if (waitees.empty()) {/* to prevent hang (no wake-up event) */ + if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ return referencesValid(); } else { state = &PathSubstitutionGoal::referencesValid; - return StillAlive{}; + return result; } } @@ -181,8 +182,7 @@ Goal::WorkResult PathSubstitutionGoal::referencesValid() assert(worker.store.isValidPath(i)); state = &PathSubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); - return StillAlive{}; + return ContinueImmediately{}; } @@ -194,8 +194,7 @@ Goal::WorkResult PathSubstitutionGoal::tryToRun() if maxSubstitutionJobs == 0, we still allow a substituter to run. This prevents infinite waiting. */ if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return StillAlive{}; + return WaitForSlot{}; } maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); @@ -256,8 +255,7 @@ Goal::WorkResult PathSubstitutionGoal::finished() /* Try the next substitute. */ state = &PathSubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return StillAlive{}; + return ContinueImmediately{}; } worker.markContentsGood(storePath); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 84727a377..a27cb0076 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,10 +1,11 @@ +#include "charptr-cast.hh" #include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" #include "local-derivation-goal.hh" #include "signals.hh" -#include "hook-instance.hh" +#include "hook-instance.hh" // IWYU pragma: keep #include <poll.h> @@ -21,10 +22,6 @@ Worker::Worker(Store & store, Store & evalStore) nrLocalBuilds = 0; nrSubstitutions = 0; lastWokenUp = steady_time_point::min(); - permanentFailure = false; - timedOut = false; - hashMismatch = false; - checkMismatch = false; } @@ -145,6 +142,11 @@ void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) assert(!goal->exitCode.has_value()); goal->exitCode = f.result; + permanentFailure |= f.permanentFailure; + timedOut |= f.timedOut; + hashMismatch |= f.hashMismatch; + checkMismatch |= f.checkMismatch; + if (f.ex) { if (!goal->waiters.empty()) logError(f.ex->info()); @@ -187,6 +189,15 @@ void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) std::visit( overloaded{ [&](Goal::StillAlive) {}, + [&](Goal::WaitForSlot) { waitForBuildSlot(goal); }, + [&](Goal::WaitForAWhile) { waitForAWhile(goal); }, + [&](Goal::ContinueImmediately) { wakeUp(goal); }, + [&](Goal::WaitForGoals & w) { + for (auto & dep : w.goals) { + goal->waitees.insert(dep); + dep->waiters.insert(goal); + } + }, [&](Goal::Finished & f) { goalFinished(goal, f); }, }, how @@ -521,7 +532,7 @@ void Worker::waitForInput() if (rd == 0 || (rd == -1 && errno == EIO)) { debug("%1%: got EOF", goal->getName()); goal->handleEOF(k); - wakeUp(goal); + handleWorkResult(goal, Goal::ContinueImmediately{}); j->fds.erase(k); } else if (rd == -1) { if (errno != EINTR) @@ -529,7 +540,7 @@ void Worker::waitForInput() } else { printMsg(lvlVomit, "%1%: read %2% bytes", goal->getName(), rd); - std::string_view data((char *) buffer.data(), rd); + std::string_view data(charptr_cast<char *>(buffer.data()), rd); j->lastOutput = after; handleWorkResult(goal, goal->handleChildOutput(k, data)); } @@ -580,7 +591,7 @@ bool Worker::pathContentsGood(const StorePath & path) res = false; else { HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); + Hash nullHash(HashType::SHA256); res = info->narHash == nullHash || info->narHash == current.first; } pathContentsGoodCache.insert_or_assign(path, res); diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 5af93b49e..a741b2672 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -105,35 +105,59 @@ private: */ std::map<StorePath, bool> pathContentsGoodCache; - void goalFinished(GoalPtr goal, Goal::Finished & f); - void handleWorkResult(GoalPtr goal, Goal::WorkResult how); - -public: - - const Activity act; - const Activity actDerivations; - const Activity actSubstitutions; - /** * Set if at least one derivation had a BuildError (i.e. permanent * failure). */ - bool permanentFailure; + bool permanentFailure = false; /** * Set if at least one derivation had a timeout. */ - bool timedOut; + bool timedOut = false; /** * Set if at least one derivation fails with a hash mismatch. */ - bool hashMismatch; + bool hashMismatch = false; /** * Set if at least one derivation is not deterministic in check mode. */ - bool checkMismatch; + bool checkMismatch = false; + + void goalFinished(GoalPtr goal, Goal::Finished & f); + void handleWorkResult(GoalPtr goal, Goal::WorkResult how); + + /** + * Put `goal` to sleep until a build slot becomes available (which + * might be right away). + */ + void waitForBuildSlot(GoalPtr goal); + + /** + * Wait for a few seconds and then retry this goal. Used when + * waiting for a lock held by another process. This kind of + * polling is inefficient, but POSIX doesn't really provide a way + * to wait for multiple locks in the main select() loop. + */ + void waitForAWhile(GoalPtr goal); + + /** + * Wake up a goal (i.e., there is something for it to do). + */ + void wakeUp(GoalPtr goal); + + /** + * Wait for input to become available. + */ + void waitForInput(); + +public: + + const Activity act; + const Activity actDerivations; + const Activity actSubstitutions; Store & store; Store & evalStore; @@ -206,11 +230,6 @@ public: void removeGoal(GoalPtr goal); /** - * Wake up a goal (i.e., there is something for it to do). - */ - void wakeUp(GoalPtr goal); - - /** * Return the number of local build processes currently running (but not * remote builds via the build hook). */ @@ -234,29 +253,10 @@ public: void childTerminated(Goal * goal); /** - * Put `goal` to sleep until a build slot becomes available (which - * might be right away). - */ - void waitForBuildSlot(GoalPtr goal); - - /** - * Wait for a few seconds and then retry this goal. Used when - * waiting for a lock held by another process. This kind of - * polling is inefficient, but POSIX doesn't really provide a way - * to wait for multiple locks in the main select() loop. - */ - void waitForAWhile(GoalPtr goal); - - /** * Loop until the specified top-level goals have finished. */ void run(const Goals & topGoals); - /** - * Wait for input to become available. - */ - void waitForInput(); - /*** * The exit status in case of failure. * diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 3a4cdd6e7..4049d1c6c 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -60,7 +60,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hashedMirror.ends_with("/")) hashedMirror += '/'; std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo")); Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 811ddbed7..6aa6d598d 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -61,7 +61,7 @@ std::string ContentAddress::render() const + makeFileIngestionPrefix(method); }, }, method.raw) - + this->hash.to_string(Base32, true); + + this->hash.to_string(Base::Base32, true); } /** diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 2e0fd8ca5..6f4a36735 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,3 +1,4 @@ +#include "charptr-cast.hh" #include "crypto.hh" #include "file-system.hh" #include "globals.hh" @@ -44,16 +45,16 @@ std::string SecretKey::signDetached(std::string_view data) const { unsigned char sig[crypto_sign_BYTES]; unsigned long long sigLen; - crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), - (unsigned char *) key.data()); - return name + ":" + base64Encode(std::string((char *) sig, sigLen)); + crypto_sign_detached(sig, &sigLen, charptr_cast<const unsigned char *>(data.data()), data.size(), + charptr_cast<const unsigned char *>(key.data())); + return name + ":" + base64Encode(std::string(reinterpret_cast<char *>(sig), sigLen)); } PublicKey SecretKey::toPublicKey() const { unsigned char pk[crypto_sign_PUBLICKEYBYTES]; - crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); - return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); + crypto_sign_ed25519_sk_to_pk(pk, charptr_cast<const unsigned char *>(key.data())); + return PublicKey(name, std::string(reinterpret_cast<char *>(pk), crypto_sign_PUBLICKEYBYTES)); } SecretKey SecretKey::generate(std::string_view name) @@ -63,7 +64,7 @@ SecretKey SecretKey::generate(std::string_view name) if (crypto_sign_keypair(pk, sk) != 0) throw Error("key generation failed"); - return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); + return SecretKey(name, std::string(reinterpret_cast<char *>(sk), crypto_sign_SECRETKEYBYTES)); } PublicKey::PublicKey(std::string_view s) @@ -85,9 +86,9 @@ bool verifyDetached(const std::string & data, const std::string & sig, if (sig2.size() != crypto_sign_BYTES) throw Error("signature is not valid"); - return crypto_sign_verify_detached((unsigned char *) sig2.data(), - (unsigned char *) data.data(), data.size(), - (unsigned char *) key->second.key.data()) == 0; + return crypto_sign_verify_detached(charptr_cast<unsigned char *>(sig2.data()), + charptr_cast<const unsigned char *>(data.data()), data.size(), + charptr_cast<const unsigned char *>(key->second.key.data())) == 0; } PublicKeys getDefaultPublicKeys() diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index b09645f46..5ac9cd2ef 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -416,7 +416,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { [&](const TextIngestionMethod &) { - if (hashType != htSHA256) + if (hashType != HashType::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", name, printHashType(hashType)); // We could stream this by changing Store @@ -453,7 +453,11 @@ static void performOp(TunnelLogger * logger, ref<Store> store, hashAlgo = parseHashType(hashAlgoRaw); } - GeneratorSource dumpSource{[&]() -> WireFormatGenerator { + // Note to future maintainers: do *not* inline this into the + // generator statement as the lambda itself needs to live to the + // end of the generator's lifetime and is otherwise a UAF. + // NOLINTNEXTLINE(cppcoreguidelines-avoid-capturing-lambda-coroutines): does not outlive the outer function + auto g = [&]() -> WireFormatGenerator { if (method == FileIngestionMethod::Recursive) { /* We parse the NAR dump through into `saved` unmodified, so why all this extra work? We still parse the NAR so @@ -489,7 +493,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store, } co_yield std::move(file->contents); } - }()}; + }; + GeneratorSource dumpSource{g()}; logger->startWork(); auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); logger->stopWork(); @@ -875,7 +880,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, bool repair, dontCheckSigs; auto path = store->parseStorePath(readString(from)); auto deriver = readString(from); - auto narHash = Hash::parseAny(readString(from), htSHA256); + auto narHash = Hash::parseAny(readString(from), HashType::SHA256); ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 9d342892d..7f41e6865 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -592,7 +592,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base::Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); @@ -823,9 +823,9 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut std::map<std::string, Hash> outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw); - auto hash = hashString(htSHA256, "fixed:out:" + auto hash = hashString(HashType::SHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" - + dof.ca.hash.to_string(Base16, false) + ":" + + dof.ca.hash.to_string(Base::Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -870,11 +870,11 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut const auto h = get(res.hashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(Base16, false)].value.insert(outputName); + inputs2[h->to_string(Base::Base16, false)].value.insert(outputName); } } - auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + auto hash = hashString(HashType::SHA256, drv.unparse(store, maskOutputs, &inputs2)); std::map<std::string, Hash> outputHashes; for (const auto & [outputName, _] : drv.outputs) { @@ -975,7 +975,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.ca.printMethodAlgo() - << dof.ca.hash.to_string(Base16, false); + << dof.ca.hash.to_string(Base::Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" @@ -1007,7 +1007,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); + return "/" + hashString(HashType::SHA256, concatStrings("nix-output:", outputName)).to_string(Base::Base32, false); } @@ -1199,7 +1199,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const } -const Hash impureOutputHash = hashString(htSHA256, "impure"); +const Hash impureOutputHash = hashString(HashType::SHA256, "impure"); nlohmann::json DerivationOutput::toJSON( const Store & store, std::string_view drvName, OutputNameView outputName) const @@ -1212,7 +1212,7 @@ nlohmann::json DerivationOutput::toJSON( [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["hashAlgo"] = dof.ca.printMethodAlgo(); - res["hash"] = dof.ca.hash.to_string(Base16, false); + res["hash"] = dof.ca.hash.to_string(Base::Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 7e3f7548d..d6af5c951 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -5,7 +5,7 @@ namespace nix { std::string DownstreamPlaceholder::render() const { - return "/" + hash.to_string(Base32, false); + return "/" + hash.to_string(Base::Base32, false); } @@ -19,7 +19,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); return DownstreamPlaceholder { - hashString(htSHA256, clearText) + hashString(HashType::SHA256, clearText) }; } @@ -31,10 +31,10 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( xpSettings.require(Xp::DynamicDerivations); auto compressed = compressHash(placeholder.hash, 20); auto clearText = "nix-computed-output:" - + compressed.to_string(Base32, false) + + compressed.to_string(Base::Base32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { - hashString(htSHA256, clearText) + hashString(HashType::SHA256, clearText) }; } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 8f95919b1..2ccb7f213 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -30,7 +30,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); - HashSink hashSink(htSHA256); + HashSink hashSink(HashType::SHA256); TeeSink teeSink(sink, hashSink); teeSink << narFromPath(path); @@ -41,7 +41,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashSink.currentHash().first; if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash.to_string(Base::Base32, true), hash.to_string(Base::Base32, true)); teeSink << exportMagic @@ -77,7 +77,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) auto references = CommonProto::Serialise<StorePathSet>::read(*this, CommonProto::ReadConn { .from = source }); auto deriver = readString(source); - auto narHash = hashString(htSHA256, saved.s); + auto narHash = hashString(HashType::SHA256, saved.s); ValidPathInfo info { path, narHash }; if (deriver != "") diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 566dc65d4..11c8a755c 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -5,7 +5,6 @@ #include "s3.hh" #include "signals.hh" #include "compression.hh" -#include "finally.hh" #if ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -143,9 +142,9 @@ struct curlFileTransfer : public FileTransfer if (successfulStatuses.count(getHTTPStatus()) && this->dataCallback) { writtenToSink += realSize; - dataCallback(*this, {(const char *) contents, realSize}); + dataCallback(*this, {static_cast<const char *>(contents), realSize}); } else { - this->result.data.append((const char *) contents, realSize); + this->result.data.append(static_cast<const char *>(contents), realSize); } return realSize; @@ -157,13 +156,13 @@ struct curlFileTransfer : public FileTransfer static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((TransferItem *) userp)->writeCallback(contents, size, nmemb); + return static_cast<TransferItem *>(userp)->writeCallback(contents, size, nmemb); } size_t headerCallback(void * contents, size_t size, size_t nmemb) { size_t realSize = size * nmemb; - std::string line((char *) contents, realSize); + std::string line(static_cast<char *>(contents), realSize); printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); @@ -204,7 +203,7 @@ struct curlFileTransfer : public FileTransfer static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) { - return ((TransferItem *) userp)->headerCallback(contents, size, nmemb); + return static_cast<TransferItem *>(userp)->headerCallback(contents, size, nmemb); } int progressCallback(double dltotal, double dlnow) @@ -219,7 +218,7 @@ struct curlFileTransfer : public FileTransfer static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) { - return ((TransferItem *) userp)->progressCallback(dltotal, dlnow); + return static_cast<TransferItem *>(userp)->progressCallback(dltotal, dlnow); } static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) @@ -246,7 +245,7 @@ struct curlFileTransfer : public FileTransfer static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) { - return ((TransferItem *) userp)->readCallback(buffer, size, nitems); + return static_cast<TransferItem *>(userp)->readCallback(buffer, size, nitems); } void init() @@ -479,7 +478,7 @@ struct curlFileTransfer : public FileTransfer { try { stopWorkerThread(); - } catch (nix::Error e) { + } catch (nix::Error & e) { // This can only fail if a socket to our own process cannot be // written to, so it is always a bug in the program if it fails. // diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index a088d633f..7b4a12dc9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -42,7 +42,7 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(htSHA1, path).to_string(Base32, false); + std::string hash = hashString(HashType::SHA1, path).to_string(Base::Base32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 032611f7c..5650282d6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -183,7 +183,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << ServeProto::Command::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(Base::Base16, false); conn->to << ServeProto::write(*this, *conn, info.references); conn->to << info.registrationTime diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f09d1bdab..4c8e2ea2f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -11,7 +11,6 @@ #include "finally.hh" #include "compression.hh" -#include <iostream> #include <algorithm> #include <cstring> @@ -539,9 +538,10 @@ void LocalStore::openDB(State & state, bool create) { SQLiteStmt stmt; stmt.create(db, "pragma main.journal_mode;"); - if (sqlite3_step(stmt) != SQLITE_ROW) + auto use = stmt.use(); + if (use.step() != SQLITE_ROW) SQLiteError::throw_(db, "querying journal mode"); - prevMode = std::string((const char *) sqlite3_column_text(stmt, 0)); + prevMode = use.getStr(0); } if (prevMode != mode && sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) @@ -842,7 +842,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmts->RegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(Base::Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -916,19 +916,22 @@ std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & s info->registrationTime = useQueryPathInfo.getInt(2); - auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); - if (s) info->deriver = parseStorePath(s); + if (auto deriver = useQueryPathInfo.getStrNullable(3); deriver.has_value()) { + info->deriver = parseStorePath(*deriver); + } /* Note that narSize = NULL yields 0. */ info->narSize = useQueryPathInfo.getInt(4); info->ultimate = useQueryPathInfo.getInt(5) == 1; - s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6); - if (s) info->sigs = tokenizeString<StringSet>(s, " "); + if (auto sigs = useQueryPathInfo.getStrNullable(6); sigs.has_value()) { + info->sigs = tokenizeString<StringSet>(*sigs, " "); + } - s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); - if (s) info->ca = ContentAddress::parseOpt(s); + if (auto ca = useQueryPathInfo.getStrNullable(7); ca.has_value()) { + info->ca = ContentAddress::parseOpt(*ca); + } /* Get the references. */ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); @@ -945,7 +948,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmts->UpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(Base::Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -1063,9 +1066,9 @@ std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & h if (!useQueryPathFromHashPart.next()) return {}; - const char * s = (const char *) sqlite3_column_text(state->stmts->QueryPathFromHashPart, 0); - if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0) - return parseStorePath(s); + auto s = useQueryPathFromHashPart.getStrNullable(0); + if (s.has_value() && s->starts_with(prefix)) + return parseStorePath(*s); return {}; }); } @@ -1123,7 +1126,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & [_, i] : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash.type == HashType::SHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1241,7 +1244,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ - HashSink hashSink(htSHA256); + HashSink hashSink(HashType::SHA256); TeeSource wrapperSource { source, hashSink }; @@ -1252,7 +1255,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); + printStorePath(info.path), info.narHash.to_string(Base::Base32, true), hashResult.first.to_string(Base::Base32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1268,8 +1271,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), - specified.hash.to_string(Base32, true), - actualHash.hash.to_string(Base32, true)); + specified.hash.to_string(Base::Base32, true), + actualHash.hash.to_string(Base::Base32, true)); } } @@ -1318,7 +1321,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto *toRealloc = dumpBuffer.release(); if (auto realloced = realloc(toRealloc, oldSize + want)) { - dumpBuffer.reset((char*) realloced); + dumpBuffer.reset(static_cast<char *>(realloced)); } else { free(toRealloc); throw std::bad_alloc(); @@ -1404,8 +1407,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name /* For computing the nar hash. In recursive SHA-256 mode, this is the same as the store hash, so no need to do it again. */ auto narHash = std::pair { hash, size }; - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { - HashSink narSink { htSHA256 }; + if (method != FileIngestionMethod::Recursive || hashAlgo != HashType::SHA256) { + HashSink narSink { HashType::SHA256 }; narSink << dumpPath(realPath); narHash = narSink.finish(); } @@ -1436,7 +1439,7 @@ StorePath LocalStore::addTextToStore( std::string_view s, const StorePathSet & references, RepairFlag repair) { - auto hash = hashString(htSHA256, s); + auto hash = hashString(HashType::SHA256, s); auto dstPath = makeTextPath(name, TextInfo { .hash = hash, .references = references, @@ -1462,7 +1465,7 @@ StorePath LocalStore::addTextToStore( StringSink sink; sink << dumpString(s); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashType::SHA256, sink.s); optimisePath(realPath, repair); @@ -1573,7 +1576,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); + std::string hash = hashPath(HashType::SHA256, linkPath).first.to_string(Base::Base32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1590,7 +1593,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking store hashes..."); - Hash nullHash(htSHA256); + Hash nullHash(HashType::SHA256); for (auto & i : validPaths) { try { @@ -1606,7 +1609,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); + printStorePath(i), info->narHash.to_string(Base::Base32, true), current.first.to_string(Base::Base32, true)); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 4e6d9dfe5..abb6e9889 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -45,7 +45,7 @@ std::map<StorePath, StorePath> makeContentAddressed( auto narModuloHash = [&] { StringSource source{sink.s}; - return computeHashModulo(htSHA256, oldHashPart, source).first; + return computeHashModulo(HashType::SHA256, oldHashPart, source).first; }(); ValidPathInfo info { @@ -63,7 +63,7 @@ std::map<StorePath, StorePath> makeContentAddressed( const auto rewritten = rewriteStrings(sink.s, {{oldHashPart, std::string(info.path.hashPart())}}); - info.narHash = hashString(htSHA256, rewritten); + info.narHash = hashString(HashType::SHA256, rewritten); info.narSize = sink.s.size(); StringSource source(rewritten); diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 169b63819..5c0bb17b9 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -334,9 +334,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base::Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash.to_string(Base::Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d17253741..e557b4677 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -104,11 +104,11 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash && fileHash->type == htSHA256); - res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; + assert(fileHash && fileHash->type == HashType::SHA256); + res += "FileHash: " + fileHash->to_string(Base::Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + assert(narHash.type == HashType::SHA256); + res += "NarHash: " + narHash.to_string(Base::Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 4f02296c3..9c871b78f 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -2,14 +2,11 @@ #include "globals.hh" #include "signals.hh" -#include <cstdlib> #include <cstring> #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> #include <errno.h> -#include <stdio.h> -#include <regex> namespace nix { @@ -145,17 +142,17 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path).first; - debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true)); + Hash hash = hashPath(HashType::SHA256, path).first; + debug("'%1%' has hash '%2%'", path, hash.to_string(Base::Base32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(Base::Base32, false); /* Maybe delete the link, if it has been corrupted. */ if (pathExists(linkPath)) { auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size - || (repair && hash != hashPath(htSHA256, linkPath).first)) + || (repair && hash != hashPath(HashType::SHA256, linkPath).first)) { // XXX: Consider overwriting linkPath with our valid version. warn("removing corrupted link '%s'", linkPath); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 336a9fcfc..4dc2823ce 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -29,7 +29,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash.to_string(Base::Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 9ba95f706..d2ed85e52 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -46,7 +46,7 @@ std::pair<StorePathSet, HashResult> scanForReferences( const std::string & path, const StorePathSet & refs) { - HashSink hashSink { htSHA256 }; + HashSink hashSink { HashType::SHA256 }; auto found = scanForReferences(hashSink, path, refs); auto hash = hashSink.finish(); return std::pair<StorePathSet, HashResult>(found, hash); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index d029e986b..d4b5fc0dc 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -46,7 +46,7 @@ StorePath::StorePath(std::string_view _baseName) } StorePath::StorePath(const Hash & hash, std::string_view _name) - : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) + : baseName((hash.to_string(Base::Base32, false) + "-").append(std::string(_name))) { checkName(baseName, name()); } @@ -60,7 +60,7 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath StorePath::random(std::string_view name) { - Hash hash(htSHA1); + Hash hash(HashType::SHA1); randombytes_buf(hash.hash, hash.hashSize); return StorePath(hash, name); } diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 559483ce3..f2b228fa0 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -39,7 +39,7 @@ struct DrvOutput { std::string to_string() const; std::string strHash() const - { return drvHash.to_string(Base16, true); } + { return drvHash.to_string(Base::Base16, true); } static DrvOutput parse(const std::string &); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1b0524316..36223051b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -397,7 +397,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( std::visit(overloaded { [&](const TextIngestionMethod & thm) -> void { - if (hashType != htSHA256) + if (hashType != HashType::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", name, printHashType(hashType)); std::string s = dump.drain(); @@ -409,7 +409,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddToStore << name - << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << ((hashType == HashType::SHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << (fim == FileIngestionMethod::Recursive ? 1 : 0) << printHashType(hashType); @@ -461,7 +461,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(Base::Base16, false); conn->to << WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) @@ -533,7 +533,7 @@ StorePath RemoteStore::addTextToStore( RepairFlag repair) { StringSource source(s); - return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path; + return addCAToStore(source, name, TextIngestionMethod {}, HashType::SHA256, references, repair)->path; } void RemoteStore::registerDrvOutput(const Realisation & info) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 264869df6..5553d84f0 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -84,7 +84,7 @@ public: * Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 723a494a5..d752bdecd 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -88,7 +88,7 @@ WireFormatGenerator ServeProto::Serialise<UnkeyedValidPathInfo>::write(const Sto co_yield info.narSize; // downloadSize, lie a little co_yield info.narSize; if (GET_PROTOCOL_MINOR(conn.version) >= 4) { - co_yield info.narHash.to_string(Base32, true); + co_yield info.narHash.to_string(Base::Base32, true); co_yield renderContentAddress(info.ca); co_yield info.sigs; } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index f40217734..8d0bfcb11 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,3 +1,4 @@ +#include "charptr-cast.hh" #include "sqlite.hh" #include "globals.hh" #include "logging.hh" @@ -199,11 +200,20 @@ bool SQLiteStmt::Use::next() return r == SQLITE_ROW; } +std::optional<std::string> SQLiteStmt::Use::getStrNullable(int col) +{ + auto s = charptr_cast<const char *>(sqlite3_column_text(stmt, col)); + return s != nullptr ? std::make_optional<std::string>((s)) : std::nullopt; +} + std::string SQLiteStmt::Use::getStr(int col) { - auto s = (const char *) sqlite3_column_text(stmt, col); - assert(s); - return s; + if (auto res = getStrNullable(col); res.has_value()) { + return *res; + } else { + // FIXME: turn into fatal non-exception error with actual formatting when we have those + assert(false && "sqlite3 retrieved unexpected null"); + } } int64_t SQLiteStmt::Use::getInt(int col) diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 003e4d101..ca021087f 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -107,6 +107,7 @@ struct SQLiteStmt bool next(); std::string getStr(int col); + std::optional<std::string> getStrNullable(int col); int64_t getInt(int col); bool isNull(int col); }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cb027d311..6d9fec41b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -156,7 +156,7 @@ StorePath Store::makeStorePath(std::string_view type, /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ auto s = std::string(type) + ":" + std::string(hash) + ":" + storeDir + ":" + std::string(name); - auto h = compressHash(hashString(htSHA256, s), 20); + auto h = compressHash(hashString(HashType::SHA256, s), 20); return StorePath(h, name); } @@ -164,7 +164,7 @@ StorePath Store::makeStorePath(std::string_view type, StorePath Store::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { - return makeStorePath(type, hash.to_string(Base16, true), name); + return makeStorePath(type, hash.to_string(Base::Base16, true), name); } @@ -194,7 +194,7 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + if (info.hash.type == HashType::SHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { if (!info.references.empty()) { @@ -202,10 +202,10 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf name); } return makeStorePath("output:out", - hashString(htSHA256, + hashString(HashType::SHA256, "fixed:out:" + makeFileIngestionPrefix(info.method) - + info.hash.to_string(Base16, true) + ":"), + + info.hash.to_string(Base::Base16, true) + ":"), name); } } @@ -213,7 +213,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.type == htSHA256); + assert(info.hash.type == HashType::SHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, @@ -259,7 +259,7 @@ StorePath Store::computeStorePathForText( const StorePathSet & references) const { return makeTextPath(name, TextInfo { - .hash = hashString(htSHA256, s), + .hash = hashString(HashType::SHA256, s), .references = references, }); } @@ -407,7 +407,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, std::optional<Hash> expectedCAHash) { - HashSink narHashSink { htSHA256 }; + HashSink narHashSink { HashType::SHA256 }; HashSink caHashSink { hashAlgo }; /* Note that fileSink and unusualHashTee must be mutually exclusive, since @@ -416,7 +416,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, RetrieveRegularNARSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; - auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashType::SHA256 ? static_cast<Sink &>(unusualHashTee) : narHashSink; @@ -442,7 +442,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, finish. */ auto [narHash, narSize] = narHashSink.finish(); - auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashType::SHA256 ? narHash : caHashSink.finish().first; @@ -854,7 +854,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash.to_string(Base::Base16, false) + "\n"; s += fmt("%1%\n", info->narSize); } @@ -1257,7 +1257,7 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre if (!hashGiven) { std::string s; getline(str, s); - auto narHash = Hash::parseAny(s, htSHA256); + auto narHash = Hash::parseAny(s, HashType::SHA256); getline(str, s); auto narSize = string2Int<uint64_t>(s); if (!narSize) throw Error("number expected"); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 91a8ef2ca..25bc0c823 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -311,7 +311,7 @@ public: */ std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; + HashType hashAlgo = HashType::SHA256, PathFilter & filter = defaultPathFilter) const; /** * Preparatory part of addTextToStore(). @@ -524,7 +524,7 @@ public: std::string_view name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, + HashType hashAlgo = HashType::SHA256, PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()); @@ -535,7 +535,7 @@ public: * memory. */ ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, std::optional<Hash> expectedCAHash = {}); /** @@ -548,7 +548,7 @@ public: * \todo remove? */ virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } @@ -679,7 +679,7 @@ public: */ nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, bool includeImpureInfo, bool showClosureSize, - Base hashBase = Base32, + Base hashBase = Base::Base32, AllowInvalidFlag allowInvalid = DisallowInvalid); /** diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index f85fba0ff..b9189df2b 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -158,7 +158,7 @@ WireFormatGenerator WorkerProto::Serialise<ValidPathInfo>::write(const Store & s UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const Store & store, ReadConn conn) { auto deriver = readString(conn.from); - auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + auto narHash = Hash::parseAny(readString(conn.from), HashType::SHA256); UnkeyedValidPathInfo info(narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); info.references = WorkerProto::Serialise<StorePathSet>::read(store, conn); @@ -174,7 +174,7 @@ UnkeyedValidPathInfo WorkerProto::Serialise<UnkeyedValidPathInfo>::read(const St WireFormatGenerator WorkerProto::Serialise<UnkeyedValidPathInfo>::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { co_yield (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : ""); - co_yield pathInfo.narHash.to_string(Base16, false); + co_yield pathInfo.narHash.to_string(Base::Base16, false); co_yield WorkerProto::write(store, conn, pathInfo.references); co_yield pathInfo.registrationTime; co_yield pathInfo.narSize; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 71f8f88fa..5fdbaba7e 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -15,7 +15,7 @@ namespace nix { -enum HashType : char; +enum class HashType : char; class MultiCommand; diff --git a/src/libutil/charptr-cast.hh b/src/libutil/charptr-cast.hh new file mode 100644 index 000000000..ec53d8924 --- /dev/null +++ b/src/libutil/charptr-cast.hh @@ -0,0 +1,141 @@ +#pragma once +/** @file Safe casts between character pointer types. */ + +#include <concepts> // IWYU pragma: keep +#include <type_traits> + +namespace nix { + +namespace charptr_cast_detail { + +/** Custom version of std::decay that does not eat CV qualifiers on \c {char * const}. */ +template<typename T> +struct DecayArrayInternal +{ + using type = T; +}; + +template <typename T> +struct DecayArrayInternal<T[]> +{ + using type = T *; +}; + +template <typename T, std::size_t N> +struct DecayArrayInternal<T[N]> +{ + using type = T *; +}; + +template <typename T> +using DecayArray = DecayArrayInternal<T>::type; + +/** Is a character type for the purposes of safe reinterpret_cast. */ +template<typename T> +concept IsChar = std::same_as<T, char> || std::same_as<T, unsigned char>; + +template<typename T> +concept IsConvertibleToChar = std::same_as<T, char8_t> || std::same_as<T, void> || IsChar<T>; + +template<typename T> +concept IsDecayOrPointer = std::is_pointer_v<T> || std::is_pointer_v<DecayArray<T>>; + +template<typename From, typename To> +concept ValidQualifiers = requires { + // Does not discard const + requires !std::is_const_v<From> || std::is_const_v<To>; + // Don't deal with volatile + requires !std::is_volatile_v<From> && !std::is_volatile_v<To>; +}; + +template<typename From, typename To> +concept BaseCase = requires { + // Cannot cast away const + requires ValidQualifiers<From, To>; + // At base case, neither should be pointers + requires !std::is_pointer_v<From> && !std::is_pointer_v<To>; + // Finally are the types compatible? + requires IsConvertibleToChar<std::remove_cv_t<From>>; + requires IsChar<std::remove_cv_t<To>>; +}; + +static_assert(BaseCase<char, char>); +static_assert(BaseCase<unsigned char, char>); +static_assert(BaseCase<char8_t, char>); +static_assert(!BaseCase<const char8_t, char>); +static_assert(!BaseCase<const char8_t, unsigned char>); +static_assert(BaseCase<void, unsigned char>); +// Not legal to cast to char8_t +static_assert(!BaseCase<void, char8_t>); +// No pointers +static_assert(!BaseCase<void *, char8_t>); +static_assert(!BaseCase<char *, char *>); + +// Required to be written in the old style because recursion in concepts is not +// allowed. Personally I think the committee hates fun. +template<typename From, typename To, typename = void> +struct RecursionHelper : std::false_type +{}; + +template<typename From, typename To> +struct RecursionHelper<From, To, std::enable_if_t<BaseCase<From, To>>> : std::true_type +{}; + +template<typename From, typename To> +struct RecursionHelper< + From, + To, + std::enable_if_t<std::is_pointer_v<From> && std::is_pointer_v<To> && ValidQualifiers<From, To>>> + : RecursionHelper<std::remove_pointer_t<From>, std::remove_pointer_t<To>> +{}; + +template<typename From, typename To> +concept IsCharCastable = requires { + // We only decay arrays in From for safety reasons. There is almost no reason + // to cast *into* an array and such code probably needs closer inspection + // anyway. + requires RecursionHelper<DecayArray<From>, To>::value; + requires IsDecayOrPointer<From> && std::is_pointer_v<To>; +}; + +static_assert(!IsCharCastable<char **, char *>); +static_assert(IsCharCastable<char *, char *>); +static_assert(!IsCharCastable<const char *, char *>); +static_assert(!IsCharCastable<volatile char *, char *>); +static_assert(!IsCharCastable<char *, volatile char *>); +static_assert(IsCharCastable<char *, const char *>); +static_assert(IsCharCastable<char **, const char **>); +static_assert(IsCharCastable<char **, const char * const *>); +static_assert(!IsCharCastable<char * const *, const char **>); +static_assert(!IsCharCastable<char, char>); +static_assert(IsCharCastable<const char *, const unsigned char *>); +static_assert(!IsCharCastable<char [64][64], char **>); +static_assert(IsCharCastable<char [64], char *>); +} + +/** Casts between character pointers with guaranteed safety. If this compiles, + * it is at least a sound conversion per C++23 §7.2.1 line 11. + * + * This will not let you: + * - Turn things into void * + * - Turn things that are not char into char + * - Turn things into things that are not char + * - Cast away const + * + * At every level in the pointer indirections, \c To must as const or more + * const than \c From. + * + * \c From may be any character pointer or void pointer or an array of characters. + * + * N.B. Be careful, the template args are in the possibly-surprising + * order To, From due to deduction. + */ +template<typename To, typename From> + requires charptr_cast_detail::IsCharCastable<From, To> +inline To charptr_cast(From p) +{ + // NOLINTNEXTLINE(lix-charptrcast): stop the linter ever getting too clever and causing funny recursion + return reinterpret_cast<To>(p); +} + +} diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index 773617031..5152a2146 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,3 +1,4 @@ +#include "charptr-cast.hh" #include "compression.hh" #include "tarfile.hh" #include "signals.hh" @@ -119,8 +120,8 @@ private: static ssize_t callback_write(struct archive * archive, void * _self, const void * buffer, size_t length) { - auto self = (ArchiveCompressionSink *) _self; - self->nextSink({(const char *) buffer, length}); + auto self = static_cast<ArchiveCompressionSink *>(_self); + self->nextSink({static_cast<const char *>(buffer), length}); return length; } }; @@ -160,7 +161,7 @@ struct BrotliDecompressionSource : Source size_t read(char * data, size_t len) override { - uint8_t * out = (uint8_t *) data; + uint8_t * out = charptr_cast<uint8_t *>(data); const auto * begin = out; while (len && !BrotliDecoderIsFinished(state.get())) { @@ -172,7 +173,7 @@ struct BrotliDecompressionSource : Source } catch (EndOfFile &) { break; } - next_in = (const uint8_t *) buf.get(); + next_in = charptr_cast<const uint8_t *>(buf.get()); } if (!BrotliDecoderDecompressStream( @@ -238,7 +239,7 @@ struct BrotliCompressionSink : ChunkedCompressionSink void writeInternal(std::string_view data) override { - auto next_in = (const uint8_t *) data.data(); + auto next_in = charptr_cast<const uint8_t *>(data.data()); size_t avail_in = data.size(); uint8_t * next_out = outbuf; size_t avail_out = sizeof(outbuf); @@ -254,7 +255,7 @@ struct BrotliCompressionSink : ChunkedCompressionSink throw CompressionError("error while compressing brotli compression"); if (avail_out < sizeof(outbuf) || avail_in == 0) { - nextSink({(const char *) outbuf, sizeof(outbuf) - avail_out}); + nextSink({reinterpret_cast<const char *>(outbuf), sizeof(outbuf) - avail_out}); next_out = outbuf; avail_out = sizeof(outbuf); } diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index 756175f95..8e3a1e408 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -66,9 +66,13 @@ void BaseSetting<T>::appendOrSet(T newValue, bool append) template<typename T> void BaseSetting<T>::set(const std::string & str, bool append) { - if (experimentalFeatureSettings.isEnabled(experimentalFeature)) - appendOrSet(parse(str), append); - else { + if (experimentalFeatureSettings.isEnabled(experimentalFeature)) { + auto parsed = parse(str); + if (deprecated && (append || parsed != value)) { + warn("deprecated setting '%s' found (set to '%s')", name, str); + } + appendOrSet(std::move(parsed), append); + } else { assert(experimentalFeature); warn("Ignoring setting '%s' because experimental feature '%s' is not enabled", name, diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 8a4d24456..d7bf9cdc9 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -175,6 +175,10 @@ class AbstractSetting public: + struct deprecated_t { + explicit deprecated_t() = default; + }; + const std::string name; const std::string description; const std::set<std::string> aliases; @@ -225,6 +229,7 @@ protected: T value; const T defaultValue; const bool documentDefault; + const bool deprecated; /** * Parse the string into a `T`. @@ -250,11 +255,13 @@ public: const std::string & name, const std::string & description, const std::set<std::string> & aliases = {}, - std::optional<ExperimentalFeature> experimentalFeature = std::nullopt) + std::optional<ExperimentalFeature> experimentalFeature = std::nullopt, + bool deprecated = false) : AbstractSetting(name, description, aliases, experimentalFeature) , value(def) , defaultValue(def) , documentDefault(documentDefault) + , deprecated(deprecated) { } operator const T &() const { return value; } @@ -322,12 +329,25 @@ public: const std::string & description, const std::set<std::string> & aliases = {}, const bool documentDefault = true, - std::optional<ExperimentalFeature> experimentalFeature = std::nullopt) - : BaseSetting<T>(def, documentDefault, name, description, aliases, std::move(experimentalFeature)) + std::optional<ExperimentalFeature> experimentalFeature = std::nullopt, + bool deprecated = false) + : BaseSetting<T>(def, documentDefault, name, description, aliases, std::move(experimentalFeature), deprecated) { options->addSetting(this); } + Setting(AbstractSetting::deprecated_t, + Config * options, + const T & def, + const std::string & name, + const std::string & description, + const std::set<std::string> & aliases = {}, + const bool documentDefault = true, + std::optional<ExperimentalFeature> experimentalFeature = std::nullopt) + : Setting(options, def, name, description, aliases, documentDefault, std::move(experimentalFeature), true) + { + } + void operator =(const T & v) { this->assign(v); } }; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 3a834293a..15a18c770 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -167,6 +167,16 @@ constexpr std::array<ExperimentalFeatureDetails, numXpFeatures> xpFeatureDetails )", }, { + .tag = Xp::PipeOperator, + .name = "pipe-operator", + .description = R"( + Enable new operators for function application to "pipe" arguments through a chain of functions similar to `lib.pipe`. + This implementation is based on Nix [RFC 148](https://github.com/NixOS/rfcs/pull/148). + + Tracking issue: https://git.lix.systems/lix-project/lix/issues/438 + )", + }, + { .tag = Xp::FetchClosure, .name = "fetch-closure", .description = R"( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 38889e7bc..121318d23 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -21,6 +21,7 @@ enum struct ExperimentalFeature NixCommand, RecursiveNix, NoUrlLiterals, + PipeOperator, FetchClosure, ReplFlake, AutoAllocateUids, diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 037cd5297..be9f8c889 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -1,3 +1,4 @@ +#include "charptr-cast.hh" #include "file-system.hh" #include "finally.hh" #include "logging.hh" @@ -115,7 +116,7 @@ Generator<Bytes> drainFDSource(int fd, bool block) throw SysError("reading from file"); } else if (rd == 0) break; - else co_yield std::span{(char *) buf.data(), (size_t) rd}; + else co_yield std::span{charptr_cast<char *>(buf.data()), (size_t) rd}; } } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index e2319ec59..631cf076b 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -567,9 +567,8 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix, std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) { Path tmpl(defaultTempDir() + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + AutoCloseFD fd(mkstemp(tmpl.data())); if (!fd) throw SysError("creating temporary file '%s'", tmpl); closeOnExec(fd.get()); diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh index cbfd6195b..dc51d7b1e 100644 --- a/src/libutil/finally.hh +++ b/src/libutil/finally.hh @@ -22,7 +22,7 @@ public: Finally(Finally &&other) : fun(std::move(other.fun)) { other.movedFrom = true; } - ~Finally() noexcept(false) + ~Finally() noexcept(noexcept(fun())) { try { if (!movedFrom) diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 0ce82f273..925f71f80 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -18,10 +18,10 @@ namespace nix { static size_t regularHashSize(HashType type) { switch (type) { - case htMD5: return md5HashSize; - case htSHA1: return sha1HashSize; - case htSHA256: return sha256HashSize; - case htSHA512: return sha512HashSize; + case HashType::MD5: return md5HashSize; + case HashType::SHA1: return sha1HashSize; + case HashType::SHA256: return sha256HashSize; + case HashType::SHA512: return sha512HashSize; } abort(); } @@ -109,34 +109,33 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { - assert(hash.type); - return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); + return hash.to_string(hash.type == HashType::MD5 ? Base::Base16 : Base::Base32, false); } std::string Hash::to_string(Base base, bool includeType) const { std::string s; - if (base == SRI || includeType) { + if (base == Base::SRI || includeType) { s += printHashType(type); - s += base == SRI ? '-' : ':'; + s += base == Base::SRI ? '-' : ':'; } switch (base) { - case Base16: + case Base::Base16: s += printHash16(*this); break; - case Base32: + case Base::Base32: s += printHash32(*this); break; - case Base64: - case SRI: - s += base64Encode(std::string_view((const char *) hash, hashSize)); + case Base::Base64: + case Base::SRI: + s += base64Encode(std::string_view(reinterpret_cast<const char *>(hash), hashSize)); break; } return s; } -Hash Hash::dummy(htSHA256); +Hash Hash::dummy(HashType::SHA256); Hash Hash::parseSRI(std::string_view original) { auto rest = original; @@ -266,7 +265,7 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional<HashType> ht) if (!ht) throw BadHash("empty hash requires explicit hash type"); Hash h(*ht); - warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); + warn("found empty hash, assuming '%s'", h.to_string(Base::SRI, true)); return h; } else return Hash::parseAny(hashStr, ht); @@ -284,29 +283,29 @@ union Ctx static void start(HashType ht, Ctx & ctx) { - if (ht == htMD5) MD5_Init(&ctx.md5); - else if (ht == htSHA1) SHA1_Init(&ctx.sha1); - else if (ht == htSHA256) SHA256_Init(&ctx.sha256); - else if (ht == htSHA512) SHA512_Init(&ctx.sha512); + if (ht == HashType::MD5) MD5_Init(&ctx.md5); + else if (ht == HashType::SHA1) SHA1_Init(&ctx.sha1); + else if (ht == HashType::SHA256) SHA256_Init(&ctx.sha256); + else if (ht == HashType::SHA512) SHA512_Init(&ctx.sha512); } static void update(HashType ht, Ctx & ctx, std::string_view data) { - if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size()); - else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); - else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); - else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); + if (ht == HashType::MD5) MD5_Update(&ctx.md5, data.data(), data.size()); + else if (ht == HashType::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); + else if (ht == HashType::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); + else if (ht == HashType::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); } static void finish(HashType ht, Ctx & ctx, unsigned char * hash) { - if (ht == htMD5) MD5_Final(hash, &ctx.md5); - else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); - else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); - else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512); + if (ht == HashType::MD5) MD5_Final(hash, &ctx.md5); + else if (ht == HashType::SHA1) SHA1_Final(hash, &ctx.sha1); + else if (ht == HashType::SHA256) SHA256_Final(hash, &ctx.sha256); + else if (ht == HashType::SHA512) SHA512_Final(hash, &ctx.sha512); } @@ -387,10 +386,10 @@ Hash compressHash(const Hash & hash, unsigned int newSize) std::optional<HashType> parseHashTypeOpt(std::string_view s) { - if (s == "md5") return htMD5; - else if (s == "sha1") return htSHA1; - else if (s == "sha256") return htSHA256; - else if (s == "sha512") return htSHA512; + if (s == "md5") return HashType::MD5; + else if (s == "sha1") return HashType::SHA1; + else if (s == "sha256") return HashType::SHA256; + else if (s == "sha512") return HashType::SHA512; else return std::optional<HashType> {}; } @@ -406,10 +405,10 @@ HashType parseHashType(std::string_view s) std::string_view printHashType(HashType ht) { switch (ht) { - case htMD5: return "md5"; - case htSHA1: return "sha1"; - case htSHA256: return "sha256"; - case htSHA512: return "sha512"; + case HashType::MD5: return "md5"; + case HashType::SHA1: return "sha1"; + case HashType::SHA256: return "sha256"; + case HashType::SHA512: return "sha512"; default: // illegal hash type enum value internally, as opposed to external input // which should be validated with nice error message. diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 1c2f8493b..47e970e17 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -12,7 +12,7 @@ namespace nix { MakeError(BadHash, Error); -enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 }; +enum class HashType : char { MD5 = 42, SHA1, SHA256, SHA512 }; const int md5HashSize = 16; @@ -24,7 +24,7 @@ extern std::set<std::string> hashTypes; extern const std::string base32Chars; -enum Base : int { Base64, Base32, Base16, SRI }; +enum class Base : int { Base64, Base32, Base16, SRI }; struct Hash @@ -119,12 +119,12 @@ public: std::string gitRev() const { - return to_string(Base16, false); + return to_string(Base::Base16, false); } std::string gitShortRev() const { - return std::string(to_string(Base16, false), 0, 7); + return std::string(to_string(Base::Base16, false), 0, 7); } static Hash dummy; diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 01fe65207..4740ea64d 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -52,6 +52,7 @@ libutil_headers = files( 'box_ptr.hh', 'canon-path.hh', 'cgroup.hh', + 'charptr-cast.hh', 'checked-arithmetic.hh', 'chunked-vector.hh', 'closure.hh', diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc index 866ba9647..61e1ad556 100644 --- a/src/libutil/processes.cc +++ b/src/libutil/processes.cc @@ -9,9 +9,7 @@ #include <cerrno> #include <cstdlib> #include <cstring> -#include <future> #include <iostream> -#include <thread> #include <grp.h> #include <sys/types.h> @@ -176,7 +174,7 @@ static pid_t doFork(std::function<void()> fun) #if __linux__ static int childEntry(void * arg) { - auto main = (std::function<void()> *) arg; + auto main = static_cast<std::function<void()> *>(arg); (*main)(); return 1; } @@ -212,8 +210,8 @@ Pid startProcess(std::function<void()> fun, const ProcessOptions & options) assert(!(options.cloneFlags & CLONE_VM)); size_t stackSize = 1 * 1024 * 1024; - auto stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + auto stack = static_cast<char *>(mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0)); if (stack == MAP_FAILED) throw SysError("allocating stack"); Finally freeStack([&]() { munmap(stack, stackSize); }); diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index df48e9203..de9a0eb9f 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -8,7 +8,8 @@ namespace nix { std::vector<char *> stringsToCharPtrs(const Strings & ss) { std::vector<char *> res; - for (auto & s : ss) res.push_back((char *) s.c_str()); + // This is const cast since this exists for OS APIs that want char * + for (auto & s : ss) res.push_back(const_cast<char *>(s.data())); res.push_back(0); return res; } diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 03dff8160..7330e2063 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -20,6 +20,8 @@ constexpr char treeNull[] = " "; * Convert a list of strings to a null-terminated vector of `char * *`s. The result must not be accessed beyond the lifetime of the * list of strings. + * + * Modifying the resulting array elements violates the constness of ss. */ std::vector<char *> stringsToCharPtrs(const Strings & ss); diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index c7f9499fd..316751533 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -1,6 +1,7 @@ #include <archive.h> #include <archive_entry.h> +#include "charptr-cast.hh" #include "file-system.hh" #include "logging.hh" #include "serialise.hh" @@ -15,11 +16,11 @@ static int callback_open(struct archive *, void * self) static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) { - auto self = (TarArchive *) _self; + auto self = static_cast<TarArchive *>(_self); *buffer = self->buffer.data(); try { - return self->source->read((char *) self->buffer.data(), self->buffer.size()); + return self->source->read(charptr_cast<char *>(self->buffer.data()), self->buffer.size()); } catch (EndOfFile &) { return 0; } catch (std::exception & err) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 8d9a49536..5762b0644 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -404,8 +404,8 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + assert(info->narHash.type == HashType::SHA256); + cout << fmt("%s\n", info->narHash.to_string(Base::Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -540,7 +540,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (canonicalise) canonicalisePathMetaData(store->printStorePath(info->path), {}); if (!hashGiven) { - HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); + HashResult hash = hashPath(HashType::SHA256, store->printStorePath(info->path)); info->narHash = hash.first; info->narSize = hash.second; } @@ -768,8 +768,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), - current.first.to_string(Base32, true)); + info->narHash.to_string(Base::Base32, true), + current.first.to_string(Base::Base32, true)); status = 1; } } @@ -970,7 +970,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto deriver = readString(in); ValidPathInfo info { store->parseStorePath(path), - Hash::parseAny(readString(in), htSHA256), + Hash::parseAny(readString(in), HashType::SHA256), }; if (deriver != "") info.deriver = store->parseStorePath(deriver); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 7e16e12c3..b9f7ece89 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -32,11 +32,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand StringSink sink; sink << dumpPath(path); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashType::SHA256, sink.s); Hash hash = narHash; if (ingestionMethod == FileIngestionMethod::Flat) { - HashSink hsink(htSHA256); + HashSink hsink(HashType::SHA256); hsink << readFileSource(path); hash = hsink.finish().first; } diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index a9211d64a..ca65c38e6 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -316,7 +316,7 @@ static void daemonLoop(std::optional<TrustedFlag> forceTrustClientOpt) socklen_t remoteAddrLen = sizeof(remoteAddr); AutoCloseFD remote{accept(fdSocket.get(), - (struct sockaddr *) &remoteAddr, &remoteAddrLen)}; + reinterpret_cast<struct sockaddr *>(&remoteAddr), &remoteAddrLen)}; checkInterrupt(); if (!remote) { if (errno == EINTR) continue; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 672930342..5ea9e077b 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -226,7 +226,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) - j["revision"] = rev->to_string(Base16, false); + j["revision"] = rev->to_string(Base::Base16, false); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) j["dirtyRevision"] = *dirtyRev; if (auto revCount = flake.lockedRef.input.getRevCount()) @@ -253,7 +253,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", - rev->to_string(Base16, false)); + rev->to_string(Base::Base16, false)); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -1434,13 +1434,13 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(tree.storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(Base::SRI, true); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), store->printStorePath(tree.storePath), - hash.to_string(SRI, true)); + hash.to_string(Base::SRI, true)); } } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 12b66a83c..f6add527a 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -11,9 +11,9 @@ using namespace nix; struct CmdHashBase : Command { FileIngestionMethod mode; - Base base = SRI; + Base base = Base::SRI; bool truncate = false; - HashType ht = htSHA256; + HashType ht = HashType::SHA256; std::vector<std::string> paths; std::optional<std::string> modulus; @@ -22,25 +22,25 @@ struct CmdHashBase : Command addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", - .handler = {&base, SRI}, + .handler = {&base, Base::SRI}, }); addFlag({ .longName = "base64", .description = "Print the hash in base-64 format.", - .handler = {&base, Base64}, + .handler = {&base, Base::Base64}, }); addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&base, Base32}, + .handler = {&base, Base::Base32}, }); addFlag({ .longName = "base16", .description = "Print the hash in base-16 format.", - .handler = {&base, Base16}, + .handler = {&base, Base::Base16}, }); addFlag(Flag::mkHashTypeFlag("type", &ht)); @@ -90,7 +90,7 @@ struct CmdHashBase : Command ? computeHashModulo(ht, *modulus, source).first : hashSource(ht, source).first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(base, base == SRI)); + logger->cout(h.to_string(base, base == Base::SRI)); } } }; @@ -110,16 +110,16 @@ struct CmdToBase : Command std::string description() override { return fmt("convert a hash to %s representation", - base == Base16 ? "base-16" : - base == Base32 ? "base-32" : - base == Base64 ? "base-64" : + base == Base::Base16 ? "base-16" : + base == Base::Base32 ? "base-32" : + base == Base::Base64 ? "base-64" : "SRI"); } void run() override { for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(base, base == Base::SRI)); } }; @@ -129,10 +129,10 @@ struct CmdHash : NixMultiCommand : MultiCommand({ {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }}, - {"to-base16", []() { return make_ref<CmdToBase>(Base16); }}, - {"to-base32", []() { return make_ref<CmdToBase>(Base32); }}, - {"to-base64", []() { return make_ref<CmdToBase>(Base64); }}, - {"to-sri", []() { return make_ref<CmdToBase>(SRI); }}, + {"to-base16", []() { return make_ref<CmdToBase>(Base::Base16); }}, + {"to-base32", []() { return make_ref<CmdToBase>(Base::Base32); }}, + {"to-base64", []() { return make_ref<CmdToBase>(Base::Base64); }}, + {"to-sri", []() { return make_ref<CmdToBase>(Base::SRI); }}, }) { } @@ -158,7 +158,7 @@ static int compatNixHash(int argc, char * * argv) { std::optional<HashType> ht; bool flat = false; - Base base = Base16; + Base base = Base::Base16; bool truncate = false; enum { opHash, opTo } op = opHash; std::vector<std::string> ss; @@ -169,10 +169,10 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base16") base = Base16; - else if (*arg == "--base32") base = Base32; - else if (*arg == "--base64") base = Base64; - else if (*arg == "--sri") base = SRI; + else if (*arg == "--base16") base = Base::Base16; + else if (*arg == "--base32") base = Base::Base32; + else if (*arg == "--base64") base = Base::Base64; + else if (*arg == "--sri") base = Base::SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); @@ -180,19 +180,19 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base16") { op = opTo; - base = Base16; + base = Base::Base16; } else if (*arg == "--to-base32") { op = opTo; - base = Base32; + base = Base::Base32; } else if (*arg == "--to-base64") { op = opTo; - base = Base64; + base = Base::Base64; } else if (*arg == "--to-sri") { op = opTo; - base = SRI; + base = Base::SRI; } else if (*arg != "" && arg->at(0) == '-') return false; @@ -203,7 +203,7 @@ static int compatNixHash(int argc, char * * argv) if (op == opHash) { CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); - if (!ht.has_value()) ht = htMD5; + if (!ht.has_value()) ht = HashType::MD5; cmd.ht = ht.value(); cmd.base = base; cmd.truncate = truncate; diff --git a/src/nix/main.cc b/src/nix/main.cc index 2f52a352f..1c1e9df7e 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,5 +1,3 @@ -#include <algorithm> - #include "args/root.hh" #include "command.hh" #include "common-args.hh" @@ -62,12 +60,12 @@ static bool haveInternet() for (auto i = addrs; i; i = i->ifa_next) { if (!i->ifa_addr) continue; if (i->ifa_addr->sa_family == AF_INET) { - if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { + if (ntohl(reinterpret_cast<sockaddr_in *>(i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { return true; } } else if (i->ifa_addr->sa_family == AF_INET6) { - if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr) && - !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr)) + if (!IN6_IS_ADDR_LOOPBACK(&reinterpret_cast<sockaddr_in6 *>(i->ifa_addr)->sin6_addr) && + !IN6_IS_ADDR_LINKLOCAL(&reinterpret_cast<sockaddr_in6 *>(i->ifa_addr)->sin6_addr)) return true; } } @@ -505,6 +503,11 @@ void mainWrapped(int argc, char * * argv) int main(int argc, char * * argv) { + if (argc < 1) { + std::cerr << "no, we don't have pkexec at home. provide argv[0]." << std::endl; + std::abort(); + } + // Increase the default stack size for the evaluator and for // libstdc++'s std::regex. nix::setStackSize(64 * 1024 * 1024); diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 613c5b191..b14eef467 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -90,7 +90,7 @@ struct CmdPathInfo : StorePathsCommand, MixJSON std::cout << store->pathInfoToJSON( // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, SRI, AllowInvalid).dump(); + true, showClosureSize, Base::SRI, AllowInvalid).dump(); } else { diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 13d94d645..0b04a04e6 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -133,7 +133,7 @@ std::tuple<StorePath, Hash> prefetchFile( static int main_nix_prefetch_url(int argc, char * * argv) { { - HashType ht = htSHA256; + HashType ht = HashType::SHA256; std::vector<std::string> args; bool printPath = getEnv("PRINT_PATH") == "1"; bool fromExpr = false; @@ -256,7 +256,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON bool executable = false; bool unpack = false; std::optional<std::string> name; - HashType hashType = htSHA256; + HashType hashType = HashType::SHA256; std::optional<Hash> expectedHash; CmdStorePrefetchFile() @@ -316,13 +316,13 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(Base::SRI, true); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", url, store->printStorePath(storePath), - hash.to_string(SRI, true)); + hash.to_string(Base::SRI, true)); } } }; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index eb68e67bc..8783d4e04 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -109,8 +109,8 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), - hash.first.to_string(Base32, true)); + info->narHash.to_string(Base::Base32, true), + hash.first.to_string(Base::Base32, true)); } } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 5bef11c4d..6df84e9e4 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -171,7 +171,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions and `dependency`. */ std::function<void(Node &, const std::string &, const std::string &)> printNode; - struct BailOut { }; + struct BailOut : std::exception { }; printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { auto pathS = store->printStorePath(node.path); diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc deleted file mode 100644 index 319f7108e..000000000 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ /dev/null @@ -1,190 +0,0 @@ -#include "derivations.hh" -#include "globals.hh" -#include "shared.hh" -#include "store-api.hh" -#include <sys/utsname.h> -#include <algorithm> -#include <iostream> -#include <fstream> -#include <sys/mman.h> -#include <fcntl.h> -#include <mach-o/loader.h> -#include <mach-o/swap.h> - -#define DO_SWAP(x, y) ((x) ? OSSwapInt32(y) : (y)) - -using namespace nix; - -static auto cacheDir = Path{}; - -Path resolveCacheFile(Path lib) -{ - std::replace(lib.begin(), lib.end(), '/', '%'); - return cacheDir + "/" + lib; -} - -std::set<std::string> readCacheFile(const Path & file) -{ - return tokenizeString<std::set<std::string>>(readFile(file), "\n"); -} - -std::set<std::string> runResolver(const Path & filename) -{ - AutoCloseFD fd{open(filename.c_str(), O_RDONLY)}; - if (!fd) - throw SysError("opening '%s'", filename); - - struct stat st; - if (fstat(fd.get(), &st)) - throw SysError("statting '%s'", filename); - - if (!S_ISREG(st.st_mode)) { - printError("file '%s' is not a regular MACH binary", filename); - return {}; - } - - if (st.st_size < sizeof(mach_header_64)) { - printError("file '%s' is too short for a MACH binary", filename); - return {}; - } - - char* obj = (char*) mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0); - if (!obj) - throw SysError("mmapping '%s'", filename); - - ptrdiff_t mach64_offset = 0; - - uint32_t magic = ((mach_header_64*) obj)->magic; - if (magic == FAT_CIGAM || magic == FAT_MAGIC) { - bool should_swap = magic == FAT_CIGAM; - uint32_t narches = DO_SWAP(should_swap, ((fat_header *) obj)->nfat_arch); - for (uint32_t i = 0; i < narches; i++) { - fat_arch* arch = (fat_arch*) (obj + sizeof(fat_header) + sizeof(fat_arch) * i); - if (DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) { - mach64_offset = (ptrdiff_t) DO_SWAP(should_swap, arch->offset); - break; - } - } - if (mach64_offset == 0) { - printError("could not find any mach64 blobs in file '%1%', continuing...", filename); - return {}; - } - } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { - mach64_offset = 0; - } else { - printError("Object file has unknown magic number '%1%', skipping it...", magic); - return {}; - } - - mach_header_64 * m_header = (mach_header_64 *) (obj + mach64_offset); - - bool should_swap = magic == MH_CIGAM_64; - ptrdiff_t cmd_offset = mach64_offset + sizeof(mach_header_64); - - std::set<std::string> libs; - for (uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) { - load_command * cmd = (load_command *) (obj + cmd_offset); - switch(DO_SWAP(should_swap, cmd->cmd)) { - case LC_LOAD_UPWARD_DYLIB: - case LC_LOAD_DYLIB: - case LC_REEXPORT_DYLIB: - libs.insert(std::string((char *) cmd + ((dylib_command*) cmd)->dylib.name.offset)); - break; - } - cmd_offset += DO_SWAP(should_swap, cmd->cmdsize); - } - - return libs; -} - -bool isSymlink(const Path & path) -{ - return S_ISLNK(lstat(path).st_mode); -} - -Path resolveSymlink(const Path & path) -{ - auto target = readLink(path); - return target.starts_with("/") - ? target - : concatStrings(dirOf(path), "/", target); -} - -std::set<std::string> resolveTree(const Path & path, PathSet & deps) -{ - std::set<std::string> results; - if (!deps.insert(path).second) return {}; - for (auto & lib : runResolver(path)) { - results.insert(lib); - for (auto & p : resolveTree(lib, deps)) { - results.insert(p); - } - } - return results; -} - -std::set<std::string> getPath(const Path & path) -{ - if (path.starts_with("/dev")) return {}; - - Path cacheFile = resolveCacheFile(path); - if (pathExists(cacheFile)) - return readCacheFile(cacheFile); - - std::set<std::string> deps, paths; - paths.insert(path); - - Path nextPath(path); - while (isSymlink(nextPath)) { - nextPath = resolveSymlink(nextPath); - paths.insert(nextPath); - } - - for (auto & t : resolveTree(nextPath, deps)) - paths.insert(t); - - writeFile(cacheFile, concatStringsSep("\n", paths)); - - return paths; -} - -int main(int argc, char ** argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); - - struct utsname _uname; - - uname(&_uname); - - auto cacheParentDir = fmt("%1%/dependency-maps", settings.nixStateDir); - - cacheDir = fmt("%1%/%2%-%3%-%4%", cacheParentDir, _uname.machine, _uname.sysname, _uname.release); - - mkdir(cacheParentDir.c_str(), 0755); - mkdir(cacheDir.c_str(), 0755); - - auto store = openStore(); - - StringSet impurePaths; - - if (std::string(argv[1]) == "--test") - impurePaths.insert(argv[2]); - else { - auto drv = store->derivationFromPath(store->parseStorePath(argv[1])); - impurePaths = tokenizeString<StringSet>(getOr(drv.env, "__impureHostDeps", "")); - impurePaths.insert("/usr/lib/libSystem.dylib"); - } - - std::set<std::string> allPaths; - - for (auto & path : impurePaths) - for (auto & p : getPath(path)) - allPaths.insert(p); - - std::cout << "extra-chroot-dirs" << std::endl; - for (auto & path : allPaths) - std::cout << path << std::endl; - std::cout << std::endl; - }); -} diff --git a/subprojects/lix-clang-tidy/CharPtrCast.cc b/subprojects/lix-clang-tidy/CharPtrCast.cc new file mode 100644 index 000000000..e76797f5d --- /dev/null +++ b/subprojects/lix-clang-tidy/CharPtrCast.cc @@ -0,0 +1,45 @@ +#include "CharPtrCast.hh" +#include <clang/AST/ExprCXX.h> +#include <clang/Basic/Diagnostic.h> +#include <clang/Tooling/Transformer/SourceCode.h> + +namespace nix::clang_tidy { +using namespace clang::ast_matchers; +using namespace clang; + +void CharPtrCastCheck::registerMatchers(ast_matchers::MatchFinder *Finder) { + Finder->addMatcher( + traverse(clang::TK_IgnoreUnlessSpelledInSource, + cxxReinterpretCastExpr(allOf( + hasDestinationType(qualType(pointsTo(isAnyCharacter()))), + has(expr(hasType(qualType(pointsTo(isAnyCharacter())))))))) + .bind("reinterpret-cast-expr"), + this); +} + +void CharPtrCastCheck::check( + const ast_matchers::MatchFinder::MatchResult &Result) { + const auto ReinterpretCastExpr = + Result.Nodes.getNodeAs<CXXReinterpretCastExpr>("reinterpret-cast-expr"); + const auto ToTypeSpan = ReinterpretCastExpr->getAngleBrackets(); + const auto & SM = Result.Context->getSourceManager(); + + auto Diag = + diag(ReinterpretCastExpr->getExprLoc(), + "reinterpret_cast used for trivially safe character pointer cast"); + Diag << ReinterpretCastExpr->getSourceRange(); + + auto Inside = tooling::getText(*ReinterpretCastExpr->getSubExprAsWritten(), + *Result.Context); + + Diag << Inserter.createIncludeInsertion(SM.getFileID(ReinterpretCastExpr->getExprLoc()), "charptr-cast.hh"); + + llvm::Twine Replacement = + "charptr_cast" + + tooling::getText(CharSourceRange(ToTypeSpan, true), *Result.Context) + + "(" + Inside + ")"; + Diag << FixItHint::CreateReplacement(ReinterpretCastExpr->getSourceRange(), + Replacement.str()); +} + +} // namespace nix::clang_tidy diff --git a/subprojects/lix-clang-tidy/CharPtrCast.hh b/subprojects/lix-clang-tidy/CharPtrCast.hh new file mode 100644 index 000000000..66883d055 --- /dev/null +++ b/subprojects/lix-clang-tidy/CharPtrCast.hh @@ -0,0 +1,32 @@ +#pragma once +///@file + +#include <clang-tidy/ClangTidyCheck.h> +#include <clang-tidy/utils/IncludeInserter.h> +#include <clang/ASTMatchers/ASTMatchFinder.h> +#include <llvm/ADT/StringRef.h> + +namespace nix::clang_tidy { + +using namespace clang; +using namespace clang::tidy; + +class CharPtrCastCheck : public ClangTidyCheck { + tidy::utils::IncludeInserter Inserter{ + Options.getLocalOrGlobal("IncludeStyle", + tidy::utils::IncludeSorter::IS_Google), + false}; + +public: + CharPtrCastCheck(StringRef Name, ClangTidyContext *Context) + : ClangTidyCheck(Name, Context) {} + + void registerPPCallbacks(const SourceManager &, Preprocessor *PP, + Preprocessor *) override { + Inserter.registerPreprocessor(PP); + } + + void registerMatchers(ast_matchers::MatchFinder *Finder) override; + void check(const ast_matchers::MatchFinder::MatchResult &Result) override; +}; +} // namespace nix::clang_tidy diff --git a/subprojects/lix-clang-tidy/LixClangTidyChecks.cc b/subprojects/lix-clang-tidy/LixClangTidyChecks.cc index b3503dd3a..283e0e7c8 100644 --- a/subprojects/lix-clang-tidy/LixClangTidyChecks.cc +++ b/subprojects/lix-clang-tidy/LixClangTidyChecks.cc @@ -2,6 +2,7 @@ #include <clang-tidy/ClangTidyModuleRegistry.h> #include "FixIncludes.hh" #include "HasPrefixSuffix.hh" +#include "CharPtrCast.hh" namespace nix::clang_tidy { using namespace clang; @@ -12,6 +13,7 @@ class NixClangTidyChecks : public ClangTidyModule { void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override { CheckFactories.registerCheck<HasPrefixSuffixCheck>("lix-hasprefixsuffix"); CheckFactories.registerCheck<FixIncludesCheck>("lix-fixincludes"); + CheckFactories.registerCheck<CharPtrCastCheck>("lix-charptrcast"); } }; diff --git a/subprojects/lix-clang-tidy/meson.build b/subprojects/lix-clang-tidy/meson.build index ef0226420..43648a1c8 100644 --- a/subprojects/lix-clang-tidy/meson.build +++ b/subprojects/lix-clang-tidy/meson.build @@ -5,9 +5,10 @@ project('lix-clang-tidy', ['cpp', 'c'], llvm = dependency('Clang', version: '>= 17', modules: ['libclang']) sources = files( + 'CharPtrCast.cc', + 'FixIncludes.cc', 'HasPrefixSuffix.cc', 'LixClangTidyChecks.cc', - 'FixIncludes.cc', ) lix_clang_tidy = shared_module('lix-clang-tidy', sources, diff --git a/tests/functional/lang/eval-okay-derivation-legacy.err.exp b/tests/functional/lang/eval-okay-derivation-legacy.err.exp new file mode 100644 index 000000000..94f0854dd --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.err.exp @@ -0,0 +1,6 @@ +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks.<output>.allowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks.<output>.allowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks.<output>.disallowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks.<output>.disallowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks.<output>.maxClosureSize' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks.<output>.maxSize' instead diff --git a/tests/functional/lang/eval-okay-derivation-legacy.exp b/tests/functional/lang/eval-okay-derivation-legacy.exp new file mode 100644 index 000000000..4f374a1aa --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.exp @@ -0,0 +1 @@ +"/nix/store/mzgwvrjjir216ra58mwwizi8wj6y9ddr-eval-okay-derivation-legacy" diff --git a/tests/functional/lang/eval-okay-derivation-legacy.nix b/tests/functional/lang/eval-okay-derivation-legacy.nix new file mode 100644 index 000000000..b529cdf90 --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.nix @@ -0,0 +1,12 @@ +(builtins.derivationStrict { + name = "eval-okay-derivation-legacy"; + system = "x86_64-linux"; + builder = "/dontcare"; + __structuredAttrs = true; + allowedReferences = [ ]; + disallowedReferences = [ ]; + allowedRequisites = [ ]; + disallowedRequisites = [ ]; + maxSize = 1234; + maxClosureSize = 12345; +}).out diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 22c69e20b..1f2c2e93f 100644 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -271,3 +271,8 @@ a = ''test string that we'll grep later'' :e identity a " "undefined variable" + +# Test :log with derivation paths. +simple_path="$(nix-instantiate "$testDir/simple.nix")" +# `PATH=` is a part of build log. +testReplResponseNoRegex ":log ${simple_path}" "PATH=" diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index 01dcbb34c..745aa168d 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -26,9 +26,9 @@ namespace nix { , state({}, store) { } - Value eval(std::string input, bool forceValue = true) { + Value eval(std::string input, bool forceValue = true, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { Value v; - Expr & e = state.parseExprFromString(input, state.rootPath(CanonPath::root)); + Expr & e = state.parseExprFromString(input, state.rootPath(CanonPath::root), xpSettings); state.eval(e, v); if (forceValue) state.forceValue(v, noPos); diff --git a/tests/unit/libexpr/trivial.cc b/tests/unit/libexpr/trivial.cc index 171727ac7..c984657fd 100644 --- a/tests/unit/libexpr/trivial.cc +++ b/tests/unit/libexpr/trivial.cc @@ -59,6 +59,11 @@ namespace nix { ASSERT_THAT(v, IsFloatEq(1.234)); } + TEST_F(TrivialExpressionTest, pointfloat) { + auto v = eval(".234"); + ASSERT_THAT(v, IsFloatEq(0.234)); + } + TEST_F(TrivialExpressionTest, updateAttrs) { auto v = eval("{ a = 1; } // { b = 2; a = 3; }"); ASSERT_THAT(v, IsAttrsOfSize(2)); @@ -81,6 +86,18 @@ namespace nix { ASSERT_THAT(v, IsTrue()); } + TEST_F(TrivialExpressionTest, urlLiteral) { + auto v = eval("https://nixos.org"); + ASSERT_THAT(v, IsStringEq("https://nixos.org")); + } + + TEST_F(TrivialExpressionTest, noUrlLiteral) { + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "no-url-literals"); + + ASSERT_THROW(eval("https://nixos.org", true, mockXpSettings), Error); + } + TEST_F(TrivialExpressionTest, withFound) { auto v = eval("with { a = 23; }; a"); ASSERT_THAT(v, IsIntEq(23)); @@ -193,4 +210,40 @@ namespace nix { TEST_F(TrivialExpressionTest, orCantBeUsed) { ASSERT_THROW(eval("let or = 1; in or"), Error); } + + // pipes are gated behind an experimental feature flag + TEST_F(TrivialExpressionTest, pipeDisabled) { + ASSERT_THROW(eval("let add = l: r: l + r; in ''a'' |> add ''b''"), Error); + ASSERT_THROW(eval("let add = l: r: l + r; in ''a'' <| add ''b''"), Error); + } + + TEST_F(TrivialExpressionTest, pipeRight) { + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "pipe-operator"); + + auto v = eval("let add = l: r: l + r; in ''a'' |> add ''b''", true, mockXpSettings); + ASSERT_THAT(v, IsStringEq("ba")); + v = eval("let add = l: r: l + r; in ''a'' |> add ''b'' |> add ''c''", true, mockXpSettings); + ASSERT_THAT(v, IsStringEq("cba")); + } + + TEST_F(TrivialExpressionTest, pipeLeft) { + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "pipe-operator"); + + auto v = eval("let add = l: r: l + r; in add ''a'' <| ''b''", true, mockXpSettings); + ASSERT_THAT(v, IsStringEq("ab")); + v = eval("let add = l: r: l + r; in add ''a'' <| add ''b'' <| ''c''", true, mockXpSettings); + ASSERT_THAT(v, IsStringEq("abc")); + } + + TEST_F(TrivialExpressionTest, pipeMixed) { + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "pipe-operator"); + + auto v = eval("let add = l: r: l + r; in add ''a'' <| ''b'' |> add ''c''", true, mockXpSettings); + ASSERT_THAT(v, IsStringEq("acb")); + v = eval("let add = l: r: l + r; in ''a'' |> add <| ''c''", true, mockXpSettings); + ASSERT_THAT(v, IsStringEq("ac")); + } } /* namespace nix */ diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc index 86ba537db..9523c9acb 100644 --- a/tests/unit/libstore/common-protocol.cc +++ b/tests/unit/libstore/common-protocol.cc @@ -102,15 +102,15 @@ CHARACTERIZATION_TEST( (std::tuple<ContentAddress, ContentAddress, ContentAddress> { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashType::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashType::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashType::SHA256, "(...)"), }, })) @@ -197,7 +197,7 @@ CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashType::SHA1, "blob blob..."), }, }, })) diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc index ca0cdff71..f6401a095 100644 --- a/tests/unit/libstore/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -148,7 +148,7 @@ TEST_JSON(DynDerivationTest, caFixedText, TEST_JSON(CaDerivationTest, caFloating, (DerivationOutput::CAFloating { .method = FileIngestionMethod::Recursive, - .hashType = htSHA256, + .hashType = HashType::SHA256, }), "drv-name", "output-name") @@ -159,7 +159,7 @@ TEST_JSON(DerivationTest, deferred, TEST_JSON(ImpureDerivationTest, impure, (DerivationOutput::Impure { .method = FileIngestionMethod::Recursive, - .hashType = htSHA256, + .hashType = HashType::SHA256, }), "drv-name", "output-name") diff --git a/tests/unit/libstore/filetransfer.cc b/tests/unit/libstore/filetransfer.cc index 6e8cf3bbe..71e7392fc 100644 --- a/tests/unit/libstore/filetransfer.cc +++ b/tests/unit/libstore/filetransfer.cc @@ -6,7 +6,6 @@ #include <future> #include <gtest/gtest.h> #include <netinet/in.h> -#include <stdexcept> #include <string> #include <string_view> #include <sys/poll.h> @@ -130,7 +129,7 @@ serveHTTP(std::string status, std::string headers, std::function<std::string()> TEST(FileTransfer, exceptionAbortsDownload) { - struct Done + struct Done : std::exception {}; auto ft = makeFileTransfer(); diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index 6cc60ca3e..c6793be74 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -53,15 +53,15 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple<ContentAddress, ContentAddress, ContentAddress> { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashType::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashType::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashType::SHA256, "(...)"), }, })) @@ -278,7 +278,7 @@ VERSIONED_CHARACTERIZATION_TEST( "foo", FixedOutputInfo { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashType::SHA256, "(...)"), .references = { .others = { StorePath { @@ -348,7 +348,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashType::SHA1, "blob blob..."), }, }, })) diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index f36de9cf9..92a33f595 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -55,15 +55,15 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple<ContentAddress, ContentAddress, ContentAddress> { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashType::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashType::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashType::SHA256, "(...)"), }, })) @@ -417,7 +417,7 @@ VERSIONED_CHARACTERIZATION_TEST( "foo", FixedOutputInfo { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashType::SHA256, "(...)"), .references = { .others = { StorePath { @@ -492,7 +492,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashType::SHA1, "blob blob..."), }, }, })) diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc index 7cc994b40..1ff31ae6a 100644 --- a/tests/unit/libutil-support/tests/hash.cc +++ b/tests/unit/libutil-support/tests/hash.cc @@ -11,11 +11,11 @@ using namespace nix; Gen<Hash> Arbitrary<Hash>::arbitrary() { - Hash prototype(htSHA1); + Hash prototype(HashType::SHA1); return gen::apply( [](const std::vector<uint8_t> & v) { - Hash hash(htSHA1); + Hash hash(HashType::SHA1); assert(v.size() == hash.hashSize); std::copy(v.begin(), v.end(), hash.hash); return hash; diff --git a/tests/unit/libutil/closure.cc b/tests/unit/libutil/closure.cc index b4eaad6f9..8cac9a9fd 100644 --- a/tests/unit/libutil/closure.cc +++ b/tests/unit/libutil/closure.cc @@ -28,7 +28,7 @@ TEST(closure, correctClosure) { } TEST(closure, properlyHandlesDirectExceptions) { - struct TestExn {}; + struct TestExn : std::exception {}; EXPECT_THROW( computeClosure<string>( {"A"}, diff --git a/tests/unit/libutil/generator.cc b/tests/unit/libutil/generator.cc index 800e6ca8a..22cc085f9 100644 --- a/tests/unit/libutil/generator.cc +++ b/tests/unit/libutil/generator.cc @@ -85,6 +85,7 @@ TEST(Generator, nestsExceptions) co_yield 1; co_yield []() -> Generator<int> { co_yield 9; + // NOLINTNEXTLINE(hicpp-exception-baseclass) throw 1; co_yield 10; }(); @@ -101,6 +102,7 @@ TEST(Generator, exception) { auto g = []() -> Generator<int> { co_yield 1; + // NOLINTNEXTLINE(hicpp-exception-baseclass) throw 1; }(); @@ -110,6 +112,7 @@ TEST(Generator, exception) } { auto g = []() -> Generator<int> { + // NOLINTNEXTLINE(hicpp-exception-baseclass) throw 1; co_return; }(); @@ -173,11 +176,13 @@ struct ThrowTransform int operator()(bool) { + // NOLINTNEXTLINE(hicpp-exception-baseclass) throw 2; } Generator<int, void> operator()(Generator<int> && inner) { + // NOLINTNEXTLINE(hicpp-exception-baseclass) throw false; } }; diff --git a/tests/unit/libutil/hash.cc b/tests/unit/libutil/hash.cc index 1f40edc95..6c24eda10 100644 --- a/tests/unit/libutil/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -13,28 +13,28 @@ namespace nix { TEST(hashString, testKnownMD5Hashes1) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; - auto hash = hashString(HashType::htMD5, s1); + auto hash = hashString(HashType::MD5, s1); ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; - auto hash = hashString(HashType::htMD5, s2); + auto hash = hashString(HashType::MD5, s2); ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; - auto hash = hashString(HashType::htSHA1, s); + auto hash = hashString(HashType::SHA1, s); ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; - auto hash = hashString(HashType::htSHA1, s); + auto hash = hashString(HashType::SHA1, s); ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } @@ -42,7 +42,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; - auto hash = hashString(HashType::htSHA256, s); + auto hash = hashString(HashType::SHA256, s); ASSERT_EQ(hash.to_string(Base::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -50,7 +50,7 @@ namespace nix { TEST(hashString, testKnownSHA256Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; - auto hash = hashString(HashType::htSHA256, s); + auto hash = hashString(HashType::SHA256, s); ASSERT_EQ(hash.to_string(Base::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -58,7 +58,7 @@ namespace nix { TEST(hashString, testKnownSHA512Hashes1) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; - auto hash = hashString(HashType::htSHA512, s); + auto hash = hashString(HashType::SHA512, s); ASSERT_EQ(hash.to_string(Base::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" @@ -68,7 +68,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; - auto hash = hashString(HashType::htSHA512, s); + auto hash = hashString(HashType::SHA512, s); ASSERT_EQ(hash.to_string(Base::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 9a44ad59b..29a5f7164 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -29,12 +29,12 @@ namespace nix { char cwd[PATH_MAX+1]; auto p = absPath(""); - ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX)); + ASSERT_EQ(p, getcwd(cwd, PATH_MAX)); } TEST(absPath, usesOptionalBasePathWhenGiven) { char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); + char* cwd = getcwd(_cwd, PATH_MAX); auto p = absPath("", cwd); @@ -43,7 +43,7 @@ namespace nix { TEST(absPath, isIdempotent) { char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); + char* cwd = getcwd(_cwd, PATH_MAX); auto p1 = absPath(cwd); auto p2 = absPath(p1); |