diff options
154 files changed, 3543 insertions, 2401 deletions
diff --git a/.editorconfig b/.editorconfig index bcee9cfce..bbc2a4908 100644 --- a/.editorconfig +++ b/.editorconfig @@ -29,3 +29,7 @@ trim_trailing_whitespace = false indent_style = space indent_size = 2 max_line_length = 0 + +[meson.build] +indent_style = space +indent_size = 2 diff --git a/bench/bench.sh b/bench/bench.sh index 70acd4640..15d8af05a 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -1,4 +1,5 @@ -#!/usr/bin/env bash +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p bash -p hyperfine set -euo pipefail shopt -s inherit_errexit @@ -21,16 +22,21 @@ fi _exit="" trap "$_exit" EXIT +flake_args=("--extra-experimental-features" "nix-command flakes") + # XXX: yes this is very silly. flakes~!! -nix build --impure --expr '(builtins.getFlake "git+file:.").inputs.nixpkgs.outPath' -o bench/nixpkgs +nix build "${flake_args[@]}" --impure --expr '(builtins.getFlake "git+file:.").inputs.nixpkgs.outPath' -o bench/nixpkgs +# We must ignore the global config, or else NIX_PATH won't work reliably. +# See https://github.com/NixOS/nix/issues/9574 +export NIX_CONF_DIR='/var/empty' export NIX_REMOTE="$(mktemp -d)" _exit='rm -rfv "$NIX_REMOTE"; $_exit' export NIX_PATH="nixpkgs=bench/nixpkgs:nixos-config=bench/configuration.nix" builds=("$@") -flake_args="--extra-experimental-features 'nix-command flakes'" +flake_args="${flake_args[*]@Q}" hyperfineArgs=( --parameter-list BUILD "$(IFS=,; echo "${builds[*]}")" diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 73fba6948..662fb4333 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -33,32 +33,7 @@ GENERATE_LATEX = NO # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -# FIXME Make this list more maintainable somehow. We could maybe generate this -# in the Makefile, but we would need to change how `.in` files are preprocessed -# so they can expand variables despite configure variables. - -INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store +INPUT = @INPUT_PATHS@ # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be @@ -97,3 +72,15 @@ EXPAND_AS_DEFINED = \ DECLARE_WORKER_SERIALISER \ DECLARE_SERVE_SERIALISER \ LENGTH_PREFIXED_PROTO_HELPER + +# The STRIP_FROM_PATH tag can be used to strip a user-defined part of the path. +# Stripping is only done if one of the specified strings matches the left-hand +# part of the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the path to +# strip. +# +# Note that you can specify absolute paths here, but also relative paths, which +# will be relative from the directory where doxygen is started. +# This tag requires that the tag FULL_PATH_NAMES is set to YES. + +STRIP_FROM_PATH = "@PROJECT_SOURCE_ROOT@" diff --git a/doc/internal-api/meson.build b/doc/internal-api/meson.build index faa30f194..af93b6943 100644 --- a/doc/internal-api/meson.build +++ b/doc/internal-api/meson.build @@ -1,3 +1,35 @@ +internal_api_sources = [ + 'src/libcmd', + 'src/libexpr', + 'src/libexpr/flake', + 'tests/unit/libexpr', + 'tests/unit/libexpr/value', + 'tests/unit/libexpr/test', + 'tests/unit/libexpr/test/value', + 'src/libexpr/value', + 'src/libfetchers', + 'src/libmain', + 'src/libstore', + 'src/libstore/build', + 'src/libstore/builtins', + 'tests/unit/libstore', + 'tests/unit/libstore/test', + 'src/libutil', + 'tests/unit/libutil', + 'tests/unit/libutil/test', + 'src/nix', + 'src/nix-env', + 'src/nix-store', +] + +# We feed Doxygen absolute paths so it can be invoked from any working directory. +internal_api_sources_absolute = [] +foreach src : internal_api_sources + internal_api_sources_absolute += '"' + (meson.project_source_root() / src) + '"' +endforeach + +internal_api_sources_oneline = ' \\\n '.join(internal_api_sources_absolute) + doxygen_cfg = configure_file( input : 'doxygen.cfg.in', output : 'doxygen.cfg', @@ -5,22 +37,16 @@ doxygen_cfg = configure_file( 'PACKAGE_VERSION': meson.project_version(), 'RAPIDCHECK_HEADERS': rapidcheck_meson.get_variable('includedir'), 'docdir' : meson.current_build_dir(), + 'INPUT_PATHS' : internal_api_sources_oneline, + 'PROJECT_SOURCE_ROOT' : meson.project_source_root(), }, ) internal_api_docs = custom_target( 'internal-api-docs', command : [ - bash, - # Meson can you please just give us a `workdir` argument to custom targets... - '-c', - # We have to prefix the doxygen_cfg path with the project build root - # because of the cd in front. - 'cd @0@ && @1@ @2@/@INPUT0@'.format( - meson.project_source_root(), - doxygen.full_path(), - meson.project_build_root(), - ), + doxygen.full_path(), + '@INPUT0@', ], input : [ doxygen_cfg, diff --git a/doc/manual/change-authors.yml b/doc/manual/change-authors.yml index e18abada1..60c0924c7 100644 --- a/doc/manual/change-authors.yml +++ b/doc/manual/change-authors.yml @@ -147,3 +147,6 @@ winter: yshui: github: yshui + +zimbatm: + github: zimbatm diff --git a/doc/manual/meson.build b/doc/manual/meson.build index f53d41b5d..35d94740c 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -126,20 +126,19 @@ manual = custom_target( 'manual', 'markdown', ], + install : true, + install_dir : [ + datadir / 'doc/nix', + false, + ], depfile : 'manual.d', env : { 'RUST_LOG': 'info', 'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'src', }, ) -manual_html = manual[0] manual_md = manual[1] -install_subdir( - manual_html.full_path(), - install_dir : datadir / 'doc/nix', -) - nix_nested_manpages = [ [ 'nix-env', [ diff --git a/doc/manual/rl-next/ctrl-c-improved.md b/doc/manual/rl-next/ctrl-c-improved.md new file mode 100644 index 000000000..c27a0edbb --- /dev/null +++ b/doc/manual/rl-next/ctrl-c-improved.md @@ -0,0 +1,13 @@ +--- +synopsis: Ctrl-C stops Nix commands much more reliably and responsively +issues: [7245, fj#393] +cls: [2016] +prs: [11618] +category: Fixes +credits: [roberth, 9999years] +--- + +CTRL-C will now stop Nix commands much more reliably and responsively. While +there are still some cases where a Nix command can be slow or unresponsive +following a `SIGINT` (please report these as issues!), the vast majority of +signals will now cause the Nix command to quit quickly and consistently. diff --git a/doc/manual/rl-next/fetchGit-regression.md b/doc/manual/rl-next/fetchGit-regression.md new file mode 100644 index 000000000..f6b4fb9e5 --- /dev/null +++ b/doc/manual/rl-next/fetchGit-regression.md @@ -0,0 +1,23 @@ +--- +synopsis: restore backwards-compatibility of `builtins.fetchGit` with Nix 2.3 +issues: [5291, 5128] +credits: [ma27] +category: Fixes +--- + +Compatibility with `builtins.fetchGit` from Nix 2.3 has been restored as follows: + +* Until now, each `ref` was prefixed with `refs/heads` unless it starts with `refs/` itself. + + Now, this is not done if the `ref` looks like a commit hash. + +* Specifying `builtins.fetchGit { ref = "a-tag"; /* … */ }` was broken because `refs/heads` was appended. + + Now, the fetcher doesn't turn a ref into `refs/heads/ref`, but into `refs/*/ref`. That way, + the value in `ref` can be either a tag or a branch. + +* The ref resolution happens the same way as in git: + + * If `refs/ref` exists, it's used. + * If a tag `refs/tags/ref` exists, it's used. + * If a branch `refs/heads/ref` exists, it's used. diff --git a/doc/manual/rl-next/nix-fmt-default-argument.md b/doc/manual/rl-next/nix-fmt-default-argument.md new file mode 100644 index 000000000..41b8f85bd --- /dev/null +++ b/doc/manual/rl-next/nix-fmt-default-argument.md @@ -0,0 +1,38 @@ +--- +synopsis: Removing the `.` default argument passed to the `nix fmt` formatter +issues: [] +prs: [11438] +cls: [1902] +category: Breaking Changes +credits: zimbatm +--- + +The underlying formatter no longer receives the ". " default argument when `nix fmt` is called with no arguments. + +This change was necessary as the formatter wasn't able to distinguish between +a user wanting to format the current folder with `nix fmt .` or the generic +`nix fmt`. + +The default behaviour is now the responsibility of the formatter itself, and +allows tools such as treefmt to format the whole tree instead of only the +current directory and below. + +This may cause issues with some formatters: nixfmt, nixpkgs-fmt and alejandra currently format stdin when no arguments are passed. + +Here is a small wrapper example that will restore the previous behaviour for such a formatter: + +```nix +{ + outputs = { self, nixpkgs, systems }: + let + eachSystem = nixpkgs.lib.genAttrs (import systems) (system: nixpkgs.legacyPackages.${system}); + in + { + formatter = eachSystem (pkgs: + pkgs.writeShellScriptBin "formatter" '' + if [[ $# = 0 ]]; set -- .; fi + exec "${pkgs.nixfmt-rfc-style}/bin/nixfmt "$@" + ''); + }; +} +``` diff --git a/doc/manual/rl-next/pytest-suite.md b/doc/manual/rl-next/pytest-suite.md new file mode 100644 index 000000000..f4dbda1e8 --- /dev/null +++ b/doc/manual/rl-next/pytest-suite.md @@ -0,0 +1,10 @@ +--- +synopsis: "The beginnings of a new pytest-based functional test suite" +category: Development +cls: [2036, 2037] +credits: jade +--- + +The existing integration/functional test suite is based on a large volume of shell scripts. +This often makes it somewhat challenging to debug at the best of times. +The goal of the pytest test suite is to make tests have more obvious dependencies on files and to make tests more concise and easier to write, as well as making new testing methods like snapshot testing easy. diff --git a/doc/manual/rl-next/stack-traces.md b/doc/manual/rl-next/stack-traces.md new file mode 100644 index 000000000..e16d6c886 --- /dev/null +++ b/doc/manual/rl-next/stack-traces.md @@ -0,0 +1,26 @@ +--- +synopsis: "Some Lix crashes now produce reporting instructions and a stack trace, then abort" +cls: [1854] +category: Improvements +credits: jade +--- + +Lix, being a C++ program, can crash in a few kinds of ways. +It can obviously do a memory access violation, which will generate a core dump and thus be relatively debuggable. +But, worse, it could throw an unhandled exception, and, in the past, we would just show the message but not where it comes from, in spite of this always being a bug, since we expect all such errors to be translated to a Lix specific error. +Now the latter kind of bug should print reporting instructions, a rudimentary stack trace and (depending on system configuration) generate a core dump. + +Sample output: + +``` +Lix crashed. This is a bug. We would appreciate if you report it along with what caused it at https://git.lix.systems/lix-project/lix/issues with the following information included: + +Exception: std::runtime_error: test exception +Stack trace: + 0# nix::printStackTrace() in /home/jade/lix/lix3/build/src/nix/../libutil/liblixutil.so + 1# 0x000073C9862331F2 in /home/jade/lix/lix3/build/src/nix/../libmain/liblixmain.so + 2# 0x000073C985F2E21A in /nix/store/p44qan69linp3ii0xrviypsw2j4qdcp2-gcc-13.2.0-lib/lib/libstdc++.so.6 + 3# 0x000073C985F2E285 in /nix/store/p44qan69linp3ii0xrviypsw2j4qdcp2-gcc-13.2.0-lib/lib/libstdc++.so.6 + 4# nix::handleExceptions(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::function<void ()>) in /home/jade/lix/lix3/build/src/nix/../libmain/liblixmain.so + ... +``` @@ -217,7 +217,7 @@ # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. - overlays.default = overlayFor (p: p.stdenv); + overlays.default = overlayFor (p: p.clangStdenv); hydraJobs = { # Binary package for various platforms. @@ -269,6 +269,8 @@ nix = pkgs.callPackage ./package.nix { inherit versionSuffix officialRelease buildUnreleasedNotes; inherit (pkgs) build-release-notes; + # Required since we don't support gcc stdenv + stdenv = pkgs.clangStdenv; internalApiDocs = true; busybox-sandbox-shell = pkgs.busybox-sandbox-shell; }; @@ -326,6 +328,8 @@ inherit (nixpkgs) pkgs; in pkgs.callPackage ./package.nix { + # Required since we don't support gcc stdenv + stdenv = pkgs.clangStdenv; versionSuffix = ""; lintInsteadOfBuild = true; }; diff --git a/meson.build b/meson.build index f89f5a016..7a5f28d6b 100644 --- a/meson.build +++ b/meson.build @@ -47,12 +47,12 @@ # in the build directory. project('lix', 'cpp', 'rust', + meson_version : '>=1.4.0', version : run_command('bash', '-c', 'echo -n $(jq -r .version < ./version.json)$VERSION_SUFFIX', check : true).stdout().strip(), default_options : [ - 'cpp_std=c++2a', + 'cpp_std=c++23', 'rust_std=2021', - # TODO(Qyriad): increase the warning level - 'warning_level=1', + 'warning_level=2', 'debug=true', 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail @@ -484,6 +484,7 @@ add_project_arguments( # TODO(Qyriad): Yes this is how the autoconf+Make system did it. # It would be nice for our headers to be idempotent instead. '-include', 'config.h', + '-Wno-unused-parameter', '-Wno-deprecated-declarations', '-Wimplicit-fallthrough', '-Werror=switch', @@ -492,12 +493,6 @@ add_project_arguments( '-Wdeprecated-copy', '-Wignored-qualifiers', '-Werror=suggest-override', - # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked - # at ~1% overhead in `nix search`. - # - # FIXME: remove when we get meson 1.4.0 which will default this to on for us: - # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions - '-D_GLIBCXX_ASSERTIONS=1', language : 'cpp', ) @@ -593,10 +588,10 @@ run_command( ) if is_darwin - configure_file( - input : 'misc/launchd/org.nixos.nix-daemon.plist.in', - output : 'org.nixos.nix-daemon.plist', - copy : true, + fs.copyfile( + 'misc/launchd/org.nixos.nix-daemon.plist.in', + 'org.nixos.nix-daemon.plist', + install : true, install_dir : prefix / 'Library/LaunchDaemons', ) endif @@ -612,6 +607,7 @@ endif if enable_tests subdir('tests/unit') subdir('tests/functional') + subdir('tests/functional2') endif subdir('meson/clang-tidy') diff --git a/misc/bash/meson.build b/misc/bash/meson.build index 75acce2ea..178692536 100644 --- a/misc/bash/meson.build +++ b/misc/bash/meson.build @@ -1,8 +1,7 @@ -configure_file( - input : 'completion.sh', - output : 'nix', +fs.copyfile( + 'completion.sh', + 'nix', install : true, install_dir : datadir / 'bash-completion/completions', install_mode : 'rw-r--r--', - copy : true, ) diff --git a/misc/capnproto.nix b/misc/capnproto.nix new file mode 100644 index 000000000..0160050a0 --- /dev/null +++ b/misc/capnproto.nix @@ -0,0 +1,60 @@ +# FIXME: upstream to nixpkgs (do NOT build with gcc due to gcc coroutine bugs) +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + openssl, + zlib, +}: +assert stdenv.cc.isClang; +stdenv.mkDerivation rec { + pname = "capnproto"; + version = "1.0.2"; + + # release tarballs are missing some ekam rules + src = fetchFromGitHub { + owner = "capnproto"; + repo = "capnproto"; + rev = "v${version}"; + sha256 = "sha256-LVdkqVBTeh8JZ1McdVNtRcnFVwEJRNjt0JV2l7RkuO8="; + }; + + nativeBuildInputs = [ cmake ]; + propagatedBuildInputs = [ + openssl + zlib + ]; + + # FIXME: separate the binaries from the stuff that user systems actually use + # This runs into a terrible UX issue in Lix and I just don't want to debug it + # right now for the couple MB of closure size: + # https://git.lix.systems/lix-project/lix/issues/551 + # outputs = [ "bin" "dev" "out" ]; + + cmakeFlags = [ + (lib.cmakeBool "BUILD_SHARED_LIBS" true) + # Take optimization flags from CXXFLAGS rather than cmake injecting them + (lib.cmakeFeature "CMAKE_BUILD_TYPE" "None") + ]; + + env = { + # Required to build the coroutine library + CXXFLAGS = "-std=c++20"; + }; + + separateDebugInfo = true; + + meta = with lib; { + homepage = "https://capnproto.org/"; + description = "Cap'n Proto cerealization protocol"; + longDescription = '' + Cap’n Proto is an insanely fast data interchange format and + capability-based RPC system. Think JSON, except binary. Or think Protocol + Buffers, except faster. + ''; + license = licenses.mit; + platforms = platforms.all; + maintainers = lib.teams.lix.members; + }; +} diff --git a/misc/fish/meson.build b/misc/fish/meson.build index d54de9a13..7f9cd0896 100644 --- a/misc/fish/meson.build +++ b/misc/fish/meson.build @@ -1,8 +1,7 @@ -configure_file( - input : 'completion.fish', - output : 'nix.fish', +fs.copyfile( + 'completion.fish', + 'nix.fish', install : true, install_dir : datadir / 'fish/vendor_completions.d', install_mode : 'rw-r--r--', - copy : true, ) diff --git a/misc/meson.build b/misc/meson.build index bf3c157f7..4e2f6aacf 100644 --- a/misc/meson.build +++ b/misc/meson.build @@ -5,8 +5,4 @@ subdir('zsh') subdir('systemd') subdir('flake-registry') -runinpty = configure_file( - copy : true, - input : meson.current_source_dir() / 'runinpty.py', - output : 'runinpty.py', -) +runinpty = fs.copyfile('runinpty.py') diff --git a/misc/zsh/meson.build b/misc/zsh/meson.build index 8063a5cb8..bd388a31f 100644 --- a/misc/zsh/meson.build +++ b/misc/zsh/meson.build @@ -1,10 +1,9 @@ foreach script : [ [ 'completion.zsh', '_nix' ], [ 'run-help-nix' ] ] - configure_file( - input : script[0], - output : script.get(1, script[0]), + fs.copyfile( + script[0], + script.get(1, script[0]), install : true, install_dir : datadir / 'zsh/site-functions', install_mode : 'rw-r--r--', - copy : true, ) endforeach diff --git a/package.nix b/package.nix index d71037f54..39ecea714 100644 --- a/package.nix +++ b/package.nix @@ -16,7 +16,6 @@ bzip2, callPackage, capnproto-lix ? __forDefaults.capnproto-lix, - capnproto, cmake, curl, doxygen, @@ -32,6 +31,8 @@ lix-clang-tidy ? null, llvmPackages, lsof, + # FIXME: remove default after dropping NixOS 24.05 + lowdown-unsandboxed ? lowdown, lowdown, mdbook, mdbook-linkcheck, @@ -104,13 +105,14 @@ build-release-notes = callPackage ./maintainers/build-release-notes.nix { }; - # needs explicit c++20 to enable coroutine support - capnproto-lix = capnproto.overrideAttrs { CXXFLAGS = "-std=c++20"; }; + # needs derivation patching to add debuginfo and coroutine library support + # !! must build this with clang as it is affected by the gcc coroutine bugs + capnproto-lix = callPackage ./misc/capnproto.nix { inherit stdenv; }; }, }: # gcc miscompiles coroutines at least until 13.2, possibly longer -assert stdenv.cc.isClang || lintInsteadOfBuild; +assert stdenv.cc.isClang; let inherit (__forDefaults) canRunInstalled; @@ -170,6 +172,7 @@ let functionalTestFiles = fileset.unions [ ./tests/functional + ./tests/functional2 ./tests/unit (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) ]; @@ -243,6 +246,8 @@ stdenv.mkDerivation (finalAttrs: { nativeBuildInputs = [ python3 + python3.pkgs.pytest + python3.pkgs.pytest-xdist meson ninja cmake @@ -250,7 +255,7 @@ stdenv.mkDerivation (finalAttrs: { capnproto-lix ] ++ [ - (lib.getBin lowdown) + (lib.getBin lowdown-unsandboxed) mdbook mdbook-linkcheck ] @@ -269,6 +274,10 @@ stdenv.mkDerivation (finalAttrs: { ++ lib.optionals lintInsteadOfBuild [ # required for a wrapped clang-tidy llvmPackages.clang-tools + # load-bearing order (just as below); the actual stdenv wrapped clang + # needs to precede the unwrapped clang in PATH such that calling `clang` + # can compile things. + stdenv.cc # required for run-clang-tidy llvmPackages.clang-unwrapped ]; @@ -440,6 +449,7 @@ stdenv.mkDerivation (finalAttrs: { editline-lix build-release-notes pegtl + capnproto-lix ; # The collection of dependency logic for this derivation is complicated enough that @@ -474,6 +484,11 @@ stdenv.mkDerivation (finalAttrs: { pythonPackages = ( p: [ + # FIXME: these have to be added twice due to the nix shell using a + # wrapped python instead of build inputs for its python inputs + p.pytest + p.pytest-xdist + p.yapf p.python-frontmatter p.requests diff --git a/scripts/meson.build b/scripts/meson.build index c916c8efa..e35c6cbb0 100644 --- a/scripts/meson.build +++ b/scripts/meson.build @@ -8,12 +8,7 @@ configure_file( } ) -# https://github.com/mesonbuild/meson/issues/860 -configure_file( - input : 'nix-profile.sh.in', - output : 'nix-profile.sh.in', - copy : true, -) +fs.copyfile('nix-profile.sh.in') foreach rc : [ '.sh', '.fish', '-daemon.sh', '-daemon.fish' ] configure_file( diff --git a/src/build-remote/build-remote.cc b/src/legacy/build-remote.cc index 3c7af067b..62ceef283 100644 --- a/src/build-remote/build-remote.cc +++ b/src/legacy/build-remote.cc @@ -19,8 +19,9 @@ #include "legacy.hh" #include "experimental-features.hh" #include "hash.hh" +#include "build-remote.hh" -using namespace nix; +namespace nix { static void handleAlarm(int sig) { } @@ -388,4 +389,8 @@ connected: } } -static RegisterLegacyCommand r_build_remote("build-remote", main_build_remote); +void registerBuildRemote() { + LegacyCommands::add("build-remote", main_build_remote); +} + +} diff --git a/src/legacy/build-remote.hh b/src/legacy/build-remote.hh new file mode 100644 index 000000000..c4a35f706 --- /dev/null +++ b/src/legacy/build-remote.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerBuildRemote(); + +} diff --git a/src/nix-env/buildenv.nix b/src/legacy/buildenv.nix index c8955a94e..c8955a94e 100644 --- a/src/nix-env/buildenv.nix +++ b/src/legacy/buildenv.nix diff --git a/src/nix-store/dotgraph.cc b/src/legacy/dotgraph.cc index 2c530999b..2c530999b 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/legacy/dotgraph.cc diff --git a/src/nix-store/dotgraph.hh b/src/legacy/dotgraph.hh index 4fd944080..4fd944080 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/legacy/dotgraph.hh diff --git a/src/nix-store/graphml.cc b/src/legacy/graphml.cc index 3e789a2d8..3e789a2d8 100644 --- a/src/nix-store/graphml.cc +++ b/src/legacy/graphml.cc diff --git a/src/nix-store/graphml.hh b/src/legacy/graphml.hh index bd3a4a37c..bd3a4a37c 100644 --- a/src/nix-store/graphml.hh +++ b/src/legacy/graphml.hh diff --git a/src/legacy/meson.build b/src/legacy/meson.build new file mode 100644 index 000000000..13b90314c --- /dev/null +++ b/src/legacy/meson.build @@ -0,0 +1,35 @@ +legacy_include_directories = include_directories('.') + +legacy_sources = files( + # `build-remote` is not really legacy (it powers all remote builds), but it's + # not a `nix3` command. + 'build-remote.cc', + 'dotgraph.cc', + 'graphml.cc', + 'nix-build.cc', + 'nix-channel.cc', + 'nix-collect-garbage.cc', + 'nix-copy-closure.cc', + 'nix-env.cc', + 'nix-env.hh', + 'nix-instantiate.cc', + 'nix-store.cc', + 'user-env.cc', +) + +legacy_headers = files( + 'build-remote.hh', + 'nix-build.hh', + 'nix-channel.hh', + 'nix-collect-garbage.hh', + 'nix-copy-closure.hh', + 'nix-instantiate.hh', + 'nix-store.hh', +) + +legacy_generated_headers = [ + gen_header.process('buildenv.nix', preserve_path_from: meson.current_source_dir()), + gen_header.process('unpack-channel.nix', preserve_path_from: meson.current_source_dir()), +] + +fs.copyfile('unpack-channel.nix') diff --git a/src/nix-build/nix-build.cc b/src/legacy/nix-build.cc index 4b8d7a2fa..eb9b6cd8d 100644 --- a/src/nix-build/nix-build.cc +++ b/src/legacy/nix-build.cc @@ -24,11 +24,13 @@ #include "attr-path.hh" #include "legacy.hh" #include "shlex.hh" +#include "nix-build.hh" -using namespace nix; -using namespace std::string_literals; +extern char * * environ __attribute__((weak)); // Man what even is this + +namespace nix { -extern char * * environ __attribute__((weak)); +using namespace std::string_literals; static void main_nix_build(int argc, char * * argv) { @@ -613,5 +615,9 @@ static void main_nix_build(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_build("nix-build", main_nix_build); -static RegisterLegacyCommand r_nix_shell("nix-shell", main_nix_build); +void registerNixBuildAndNixShell() { + LegacyCommands::add("nix-build", main_nix_build); + LegacyCommands::add("nix-shell", main_nix_build); +} + +} diff --git a/src/legacy/nix-build.hh b/src/legacy/nix-build.hh new file mode 100644 index 000000000..945ac06e2 --- /dev/null +++ b/src/legacy/nix-build.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixBuildAndNixShell(); + +} diff --git a/src/nix-channel/nix-channel.cc b/src/legacy/nix-channel.cc index 971337b63..2f79919dd 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/legacy/nix-channel.cc @@ -7,12 +7,13 @@ #include "fetchers.hh" #include "eval-settings.hh" // for defexpr #include "users.hh" +#include "nix-channel.hh" #include <fcntl.h> #include <regex> #include <pwd.h> -using namespace nix; +namespace nix { typedef std::map<std::string, std::string> Channels; @@ -264,4 +265,8 @@ static int main_nix_channel(int argc, char ** argv) } } -static RegisterLegacyCommand r_nix_channel("nix-channel", main_nix_channel); +void registerNixChannel() { + LegacyCommands::add("nix-channel", main_nix_channel); +} + +} diff --git a/src/legacy/nix-channel.hh b/src/legacy/nix-channel.hh new file mode 100644 index 000000000..f1583767f --- /dev/null +++ b/src/legacy/nix-channel.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixChannel(); + +} diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/legacy/nix-collect-garbage.cc index c831f132f..7640100a0 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/legacy/nix-collect-garbage.cc @@ -7,11 +7,12 @@ #include "globals.hh" #include "legacy.hh" #include "signals.hh" +#include "nix-collect-garbage.hh" #include <iostream> #include <cerrno> -using namespace nix; +namespace nix { std::string deleteOlderThan; bool dryRun = false; @@ -110,4 +111,8 @@ static int main_nix_collect_garbage(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_collect_garbage("nix-collect-garbage", main_nix_collect_garbage); +void registerNixCollectGarbage() { + LegacyCommands::add("nix-collect-garbage", main_nix_collect_garbage); +} + +} diff --git a/src/legacy/nix-collect-garbage.hh b/src/legacy/nix-collect-garbage.hh new file mode 100644 index 000000000..68515b537 --- /dev/null +++ b/src/legacy/nix-collect-garbage.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixCollectGarbage(); + +} diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/legacy/nix-copy-closure.cc index 7f2bb93b6..1e61e3c48 100644 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/legacy/nix-copy-closure.cc @@ -1,8 +1,9 @@ #include "shared.hh" #include "store-api.hh" #include "legacy.hh" +#include "nix-copy-closure.hh" -using namespace nix; +namespace nix { static int main_nix_copy_closure(int argc, char ** argv) { @@ -60,4 +61,8 @@ static int main_nix_copy_closure(int argc, char ** argv) } } -static RegisterLegacyCommand r_nix_copy_closure("nix-copy-closure", main_nix_copy_closure); +void registerNixCopyClosure() { + LegacyCommands::add("nix-copy-closure", main_nix_copy_closure); +} + +} diff --git a/src/legacy/nix-copy-closure.hh b/src/legacy/nix-copy-closure.hh new file mode 100644 index 000000000..fb5d0fc6e --- /dev/null +++ b/src/legacy/nix-copy-closure.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixCopyClosure(); + +} diff --git a/src/nix-env/nix-env.cc b/src/legacy/nix-env.cc index 13fadb1d8..4674fd783 100644 --- a/src/nix-env/nix-env.cc +++ b/src/legacy/nix-env.cc @@ -17,6 +17,7 @@ #include "xml-writer.hh" #include "legacy.hh" #include "eval-settings.hh" // for defexpr +#include "nix-env.hh" #include <ctime> #include <algorithm> @@ -28,9 +29,10 @@ #include <unistd.h> #include <nlohmann/json.hpp> -using namespace nix; using std::cout; +namespace nix { + typedef enum { srcNixExprDrvs, @@ -1544,4 +1546,8 @@ static int main_nix_env(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_env("nix-env", main_nix_env); +void registerNixEnv() { + LegacyCommands::add("nix-env", main_nix_env); +} + +} diff --git a/src/legacy/nix-env.hh b/src/legacy/nix-env.hh new file mode 100644 index 000000000..47d62b8e6 --- /dev/null +++ b/src/legacy/nix-env.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixEnv(); + +} diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/legacy/nix-instantiate.cc index 7487af1c1..5d1da0d70 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/legacy/nix-instantiate.cc @@ -11,12 +11,13 @@ #include "local-fs-store.hh" #include "common-eval-args.hh" #include "legacy.hh" +#include "nix-instantiate.hh" #include <map> #include <iostream> -using namespace nix; +namespace nix { static Path gcRoot; @@ -195,4 +196,8 @@ static int main_nix_instantiate(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_instantiate("nix-instantiate", main_nix_instantiate); +void registerNixInstantiate() { + LegacyCommands::add("nix-instantiate", main_nix_instantiate); +} + +} diff --git a/src/legacy/nix-instantiate.hh b/src/legacy/nix-instantiate.hh new file mode 100644 index 000000000..f4c35a6b5 --- /dev/null +++ b/src/legacy/nix-instantiate.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixInstantiate(); + +} diff --git a/src/nix-store/nix-store.cc b/src/legacy/nix-store.cc index bc43aa7b1..e42aa4065 100644 --- a/src/nix-store/nix-store.cc +++ b/src/legacy/nix-store.cc @@ -15,6 +15,7 @@ #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" +#include "nix-store.hh" #include <iostream> #include <algorithm> @@ -24,10 +25,9 @@ #include <fcntl.h> -namespace nix_store { +namespace nix { -using namespace nix; using std::cin; using std::cout; @@ -1176,6 +1176,8 @@ static int main_nix_store(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_store("nix-store", main_nix_store); +void registerNixStore() { + LegacyCommands::add("nix-store", main_nix_store); +} } diff --git a/src/legacy/nix-store.hh b/src/legacy/nix-store.hh new file mode 100644 index 000000000..b010e7b19 --- /dev/null +++ b/src/legacy/nix-store.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixStore(); + +} diff --git a/src/nix-channel/unpack-channel.nix b/src/legacy/unpack-channel.nix index 84e324a4d..84e324a4d 100644 --- a/src/nix-channel/unpack-channel.nix +++ b/src/legacy/unpack-channel.nix diff --git a/src/nix-env/user-env.cc b/src/legacy/user-env.cc index f5dbd06ca..f5dbd06ca 100644 --- a/src/nix-env/user-env.cc +++ b/src/legacy/user-env.cc diff --git a/src/nix-env/user-env.hh b/src/legacy/user-env.hh index af45d2d85..af45d2d85 100644 --- a/src/nix-env/user-env.hh +++ b/src/legacy/user-env.hh diff --git a/src/libcmd/legacy.cc b/src/libcmd/legacy.cc index 6df09ee37..8bbe9b031 100644 --- a/src/libcmd/legacy.cc +++ b/src/libcmd/legacy.cc @@ -2,6 +2,6 @@ namespace nix { -RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0; +LegacyCommands::Commands * LegacyCommands::commands = 0; } diff --git a/src/libcmd/legacy.hh b/src/libcmd/legacy.hh index 357500a4d..45a231983 100644 --- a/src/libcmd/legacy.hh +++ b/src/libcmd/legacy.hh @@ -9,12 +9,12 @@ namespace nix { typedef std::function<void(int, char * *)> MainFunction; -struct RegisterLegacyCommand +struct LegacyCommands { typedef std::map<std::string, MainFunction> Commands; static Commands * commands; - RegisterLegacyCommand(const std::string & name, MainFunction fun) + static void add(const std::string & name, MainFunction fun) { if (!commands) commands = new Commands; (*commands)[name] = fun; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 83bfd4fb0..e7336c7e8 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -79,7 +79,7 @@ struct AttrDb state->txn->commit(); state->txn.reset(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -90,7 +90,7 @@ struct AttrDb try { return fun(); } catch (SQLiteError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); failed = true; return 0; } @@ -329,7 +329,7 @@ static std::shared_ptr<AttrDb> makeAttrDb( try { return std::make_shared<AttrDb>(cfg, fingerprint, symbols); } catch (SQLiteError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); return nullptr; } } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 68da254e2..0c1a1ec0e 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -21,6 +21,14 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) return printIdentifier(str, s); } +AttrName::AttrName(Symbol s) : symbol(s) +{ +} + +AttrName::AttrName(std::unique_ptr<Expr> e) : expr(std::move(e)) +{ +} + void Expr::show(const SymbolTable & symbols, std::ostream & str) const { abort(); @@ -239,9 +247,24 @@ void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) co { bool first = true; str << "("; - for (auto & i : es) { - if (first) first = false; else str << " + "; - i.second->show(symbols, str); + for (auto & [_pos, part] : es) { + if (first) + first = false; + else + str << " + "; + + if (forceString && !dynamic_cast<ExprString *>(part.get())) { + /* Print as a string with an interpolation, to preserve the + * semantics of the value having to be a string. + * Interpolations are weird and someone should eventually + * move them out into their own AST node please. + */ + str << "\"${"; + part->show(symbols, str); + str << "}\""; + } else { + part->show(symbols, str); + } } str << ")"; } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index d16281c39..4e857b321 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -30,8 +30,8 @@ struct AttrName { Symbol symbol; std::unique_ptr<Expr> expr; - AttrName(Symbol s) : symbol(s) {}; - AttrName(std::unique_ptr<Expr> e) : expr(std::move(e)) {}; + AttrName(Symbol s); + AttrName(std::unique_ptr<Expr> e); }; typedef std::vector<AttrName> AttrPath; diff --git a/src/libexpr/parser/grammar.hh b/src/libexpr/parser/grammar.hh index 2c5a3d1be..701b40505 100644 --- a/src/libexpr/parser/grammar.hh +++ b/src/libexpr/parser/grammar.hh @@ -12,7 +12,7 @@ // eolf rules in favor of reproducing the old flex lexer as faithfully as // possible, and deferring calculation of positions to downstream users. -namespace nix::parser::grammar { +namespace nix::parser::grammar::v1 { using namespace tao::pegtl; namespace p = tao::pegtl; @@ -225,7 +225,8 @@ struct string : _string, seq< > {}; struct _ind_string { - template<bool Indented, typename... Inner> + struct line_start : semantic, star<one<' '>> {}; + template<typename... Inner> struct literal : semantic, seq<Inner...> {}; struct interpolation : semantic, seq< p::string<'$', '{'>, seps, @@ -233,34 +234,53 @@ struct _ind_string { must<one<'}'>> > {}; struct escape : semantic, must<any> {}; + /* Marker for non-empty lines */ + struct has_content : semantic, seq<> {}; }; struct ind_string : _ind_string, seq< TAO_PEGTL_STRING("''"), + // Strip first line completely if empty opt<star<one<' '>>, one<'\n'>>, - star< - sor< - _ind_string::literal< - true, + list< + seq< + // Start a line with some indentation + // (we always match even the empty string if no indentation, as this creates the line) + _ind_string::line_start, + // The actual line + opt< plus< sor< - not_one<'$', '\''>, - seq<one<'$'>, not_one<'{', '\''>>, - seq<one<'\''>, not_one<'\'', '$'>> - > - > - >, - _ind_string::interpolation, - _ind_string::literal<false, one<'$'>>, - _ind_string::literal<false, one<'\''>, not_at<one<'\''>>>, - seq<one<'\''>, _ind_string::literal<false, p::string<'\'', '\''>>>, - seq< - p::string<'\'', '\''>, - sor< - _ind_string::literal<false, one<'$'>>, - seq<one<'\\'>, _ind_string::escape> + _ind_string::literal< + plus< + sor< + not_one<'$', '\'', '\n'>, + // TODO probably factor this out like the others for performance + seq<one<'$'>, not_one<'{', '\'', '\n'>>, + seq<one<'$'>, at<one<'\n'>>>, + seq<one<'\''>, not_one<'\'', '$', '\n'>>, + seq<one<'\''>, at<one<'\n'>>> + > + > + >, + _ind_string::interpolation, + _ind_string::literal<one<'$'>>, + _ind_string::literal<one<'\''>, not_at<one<'\''>>>, + seq<one<'\''>, _ind_string::literal<p::string<'\'', '\''>>>, + seq< + p::string<'\'', '\''>, + sor< + _ind_string::literal<one<'$'>>, + seq<one<'\\'>, _ind_string::escape> + > + > + >, + _ind_string::has_content > > - > + >, + // End of line, LF. CR is just ignored and not treated as ending a line + // (for the purpose of indentation stripping) + _ind_string::literal<one<'\n'>> >, must<TAO_PEGTL_STRING("''")> > {}; @@ -352,10 +372,10 @@ struct formals : semantic, _formals, seq< struct _attr { struct simple : semantic, sor<t::identifier, t::kw_or> {}; - struct string : semantic, seq<grammar::string> {}; + struct string : semantic, seq<grammar::v1::string> {}; struct expr : semantic, seq< TAO_PEGTL_STRING("${"), seps, - must<grammar::expr>, seps, + must<grammar::v1::expr>, seps, must<one<'}'>> > {}; }; @@ -452,9 +472,9 @@ struct _expr { struct id : semantic, t::identifier {}; struct int_ : semantic, t::integer {}; struct float_ : semantic, t::floating {}; - struct string : semantic, seq<grammar::string> {}; - struct ind_string : semantic, seq<grammar::ind_string> {}; - struct path : semantic, seq<grammar::path> {}; + struct string : semantic, seq<grammar::v1::string> {}; + struct ind_string : semantic, seq<grammar::v1::ind_string> {}; + struct path : semantic, seq<grammar::v1::path> {}; struct uri : semantic, t::uri {}; struct ancient_let : semantic, _attrset<must, t::kw_let, seps> {}; struct rec_set : semantic, _attrset<must, t::kw_rec, seps> {}; @@ -628,34 +648,34 @@ struct nothing : p::nothing<Rule> { template<typename Self, typename OpCtx, typename AttrPathT, typename ExprT> struct operator_semantics { - struct has_attr : grammar::op::has_attr { + struct has_attr : grammar::v1::op::has_attr { AttrPathT path; }; struct OpEntry { OpCtx ctx; uint8_t prec; - grammar::op::kind assoc; + grammar::v1::op::kind assoc; std::variant< - grammar::op::not_, - grammar::op::unary_minus, - grammar::op::implies, - grammar::op::or_, - grammar::op::and_, - grammar::op::equals, - grammar::op::not_equals, - grammar::op::less_eq, - grammar::op::greater_eq, - grammar::op::update, - grammar::op::concat, - grammar::op::less, - grammar::op::greater, - grammar::op::plus, - grammar::op::minus, - grammar::op::mul, - grammar::op::div, - grammar::op::pipe_right, - grammar::op::pipe_left, + grammar::v1::op::not_, + grammar::v1::op::unary_minus, + grammar::v1::op::implies, + grammar::v1::op::or_, + grammar::v1::op::and_, + grammar::v1::op::equals, + grammar::v1::op::not_equals, + grammar::v1::op::less_eq, + grammar::v1::op::greater_eq, + grammar::v1::op::update, + grammar::v1::op::concat, + grammar::v1::op::less, + grammar::v1::op::greater, + grammar::v1::op::plus, + grammar::v1::op::minus, + grammar::v1::op::mul, + grammar::v1::op::div, + grammar::v1::op::pipe_right, + grammar::v1::op::pipe_left, has_attr > op; }; @@ -676,7 +696,7 @@ struct operator_semantics { auto & [ctx, precedence, kind, op] = ops.back(); // NOTE this relies on associativity not being mixed within a precedence level. if ((precedence > toPrecedence) - || (kind != grammar::op::kind::leftAssoc && precedence == toPrecedence)) + || (kind != grammar::v1::op::kind::leftAssoc && precedence == toPrecedence)) break; std::visit([&, ctx=std::move(ctx)] (auto & op) { exprs.push_back(static_cast<Self &>(*this).applyOp(ctx, op, args...)); @@ -694,9 +714,9 @@ struct operator_semantics { void pushOp(OpCtx ctx, auto o, auto &... args) { - if (o.kind != grammar::op::kind::unary) + if (o.kind != grammar::v1::op::kind::unary) reduce(o.precedence, args...); - if (!ops.empty() && o.kind == grammar::op::kind::nonAssoc) { + if (!ops.empty() && o.kind == grammar::v1::op::kind::nonAssoc) { auto & [_pos, _prec, _kind, _o] = ops.back(); if (_kind == o.kind && _prec == o.precedence) Self::badOperator(ctx, args...); diff --git a/src/libexpr/parser/parser-impl1.inc.cc b/src/libexpr/parser/parser-impl1.inc.cc new file mode 100644 index 000000000..c937d17fd --- /dev/null +++ b/src/libexpr/parser/parser-impl1.inc.cc @@ -0,0 +1,863 @@ +// flip this define when doing parser development to enable some g checks. +#if 0 +#include <tao/pegtl/contrib/analyze.hpp> +#define ANALYZE_GRAMMAR \ + ([] { \ + const std::size_t issues = tao::pegtl::analyze<grammar::v1::root>(); \ + assert(issues == 0); \ + })() +#else +#define ANALYZE_GRAMMAR ((void) 0) +#endif + +namespace p = tao::pegtl; + +namespace nix::parser::v1 { +namespace { + +template<typename> +inline constexpr const char * error_message = nullptr; + +#define error_message_for(...) \ + template<> inline constexpr auto error_message<__VA_ARGS__> + +error_message_for(p::one<'{'>) = "expecting '{'"; +error_message_for(p::one<'}'>) = "expecting '}'"; +error_message_for(p::one<'"'>) = "expecting '\"'"; +error_message_for(p::one<';'>) = "expecting ';'"; +error_message_for(p::one<')'>) = "expecting ')'"; +error_message_for(p::one<']'>) = "expecting ']'"; +error_message_for(p::one<':'>) = "expecting ':'"; +error_message_for(p::string<'\'', '\''>) = "expecting \"''\""; +error_message_for(p::any) = "expecting any character"; +error_message_for(grammar::v1::eof) = "expecting end of file"; +error_message_for(grammar::v1::seps) = "expecting separators"; +error_message_for(grammar::v1::path::forbid_prefix_triple_slash) = "too many slashes in path"; +error_message_for(grammar::v1::path::forbid_prefix_double_slash_no_interp) = "path has a trailing slash"; +error_message_for(grammar::v1::expr) = "expecting expression"; +error_message_for(grammar::v1::expr::unary) = "expecting expression"; +error_message_for(grammar::v1::binding::equal) = "expecting '='"; +error_message_for(grammar::v1::expr::lambda::arg) = "expecting identifier"; +error_message_for(grammar::v1::formals) = "expecting formals"; +error_message_for(grammar::v1::attrpath) = "expecting attribute path"; +error_message_for(grammar::v1::expr::select) = "expecting selection expression"; +error_message_for(grammar::v1::t::kw_then) = "expecting 'then'"; +error_message_for(grammar::v1::t::kw_else) = "expecting 'else'"; +error_message_for(grammar::v1::t::kw_in) = "expecting 'in'"; + +struct SyntaxErrors +{ + template<typename Rule> + static constexpr auto message = error_message<Rule>; + + template<typename Rule> + static constexpr bool raise_on_failure = false; +}; + +template<typename Rule> +struct Control : p::must_if<SyntaxErrors>::control<Rule> +{ + template<typename ParseInput, typename... States> + [[noreturn]] static void raise(const ParseInput & in, States &&... st) + { + if (in.empty()) { + std::string expected; + if constexpr (constexpr auto msg = error_message<Rule>) + expected = fmt(", %s", msg); + throw p::parse_error("unexpected end of file" + expected, in); + } + p::must_if<SyntaxErrors>::control<Rule>::raise(in, st...); + } +}; + +struct ExprState + : grammar::v1:: + operator_semantics<ExprState, PosIdx, AttrPath, std::pair<PosIdx, std::unique_ptr<Expr>>> +{ + std::unique_ptr<Expr> popExprOnly() { + return std::move(popExpr().second); + } + + template<typename Op, typename... Args> + std::unique_ptr<Expr> applyUnary(Args &&... args) { + return std::make_unique<Op>(popExprOnly(), std::forward<Args>(args)...); + } + + template<typename Op> + std::unique_ptr<Expr> applyBinary(PosIdx pos) { + auto right = popExprOnly(), left = popExprOnly(); + return std::make_unique<Op>(pos, std::move(left), std::move(right)); + } + + std::unique_ptr<Expr> call(PosIdx pos, Symbol fn, bool flip = false) + { + std::vector<std::unique_ptr<Expr>> args(2); + args[flip ? 0 : 1] = popExprOnly(); + args[flip ? 1 : 0] = popExprOnly(); + return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args)); + } + + std::unique_ptr<Expr> pipe(PosIdx pos, State & state, bool flip = false) + { + if (!state.featureSettings.isEnabled(Xp::PipeOperator)) + throw ParseError({ + .msg = HintFmt("Pipe operator is disabled"), + .pos = state.positions[pos] + }); + + // Reverse the order compared to normal function application: arg |> fn + std::unique_ptr<Expr> fn, arg; + if (flip) { + fn = popExprOnly(); + arg = popExprOnly(); + } else { + arg = popExprOnly(); + fn = popExprOnly(); + } + std::vector<std::unique_ptr<Expr>> args{1}; + args[0] = std::move(arg); + + return std::make_unique<ExprCall>(pos, std::move(fn), std::move(args)); + } + + std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state) + { + return call(pos, state.s.lessThan, !less); + } + + std::unique_ptr<Expr> concatStrings(PosIdx pos) + { + std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> args(2); + args[1] = popExpr(); + args[0] = popExpr(); + return std::make_unique<ExprConcatStrings>(pos, false, std::move(args)); + } + + std::unique_ptr<Expr> negate(PosIdx pos, State & state) + { + std::vector<std::unique_ptr<Expr>> args(2); + args[0] = std::make_unique<ExprInt>(0); + args[1] = popExprOnly(); + return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(state.s.sub), std::move(args)); + } + + std::pair<PosIdx, std::unique_ptr<Expr>> applyOp(PosIdx pos, auto & op, State & state) { + using Op = grammar::v1::op; + + auto not_ = [] (auto e) { + return std::make_unique<ExprOpNot>(std::move(e)); + }; + + return { + pos, + (overloaded { + [&] (Op::implies) { return applyBinary<ExprOpImpl>(pos); }, + [&] (Op::or_) { return applyBinary<ExprOpOr>(pos); }, + [&] (Op::and_) { return applyBinary<ExprOpAnd>(pos); }, + [&] (Op::equals) { return applyBinary<ExprOpEq>(pos); }, + [&] (Op::not_equals) { return applyBinary<ExprOpNEq>(pos); }, + [&] (Op::less) { return order(pos, true, state); }, + [&] (Op::greater_eq) { return not_(order(pos, true, state)); }, + [&] (Op::greater) { return order(pos, false, state); }, + [&] (Op::less_eq) { return not_(order(pos, false, state)); }, + [&] (Op::update) { return applyBinary<ExprOpUpdate>(pos); }, + [&] (Op::not_) { return applyUnary<ExprOpNot>(); }, + [&] (Op::plus) { return concatStrings(pos); }, + [&] (Op::minus) { return call(pos, state.s.sub); }, + [&] (Op::mul) { return call(pos, state.s.mul); }, + [&] (Op::div) { return call(pos, state.s.div); }, + [&] (Op::concat) { return applyBinary<ExprOpConcatLists>(pos); }, + [&] (has_attr & a) { return applyUnary<ExprOpHasAttr>(std::move(a.path)); }, + [&] (Op::unary_minus) { return negate(pos, state); }, + [&] (Op::pipe_right) { return pipe(pos, state, true); }, + [&] (Op::pipe_left) { return pipe(pos, state); }, + })(op) + }; + } + + // always_inline is needed, otherwise pushOp slows down considerably + [[noreturn, gnu::always_inline]] + static void badOperator(PosIdx pos, State & state) + { + throw ParseError({ + .msg = HintFmt("syntax error, unexpected operator"), + .pos = state.positions[pos] + }); + } + + template<typename Expr, typename... Args> + Expr & pushExpr(PosIdx pos, Args && ... args) + { + auto p = std::make_unique<Expr>(std::forward<Args>(args)...); + auto & result = *p; + exprs.emplace_back(pos, std::move(p)); + return result; + } +}; + +struct SubexprState { +private: + ExprState * up; + +public: + explicit SubexprState(ExprState & up, auto &...) : up(&up) {} + operator ExprState &() { return *up; } + ExprState * operator->() { return up; } +}; + + + +template<typename Rule> +struct BuildAST : grammar::v1::nothing<Rule> {}; + +struct LambdaState : SubexprState { + using SubexprState::SubexprState; + + Symbol arg; + std::unique_ptr<Formals> formals; +}; + +struct FormalsState : SubexprState { + using SubexprState::SubexprState; + + Formals formals{}; + Formal formal{}; +}; + +template<> struct BuildAST<grammar::v1::formal::name> { + static void apply(const auto & in, FormalsState & s, State & ps) { + s.formal = { + .pos = ps.at(in), + .name = ps.symbols.create(in.string_view()), + }; + } +}; + +template<> struct BuildAST<grammar::v1::formal> { + static void apply0(FormalsState & s, State &) { + s.formals.formals.emplace_back(std::move(s.formal)); + } +}; + +template<> struct BuildAST<grammar::v1::formal::default_value> { + static void apply0(FormalsState & s, State & ps) { + s.formal.def = s->popExprOnly(); + } +}; + +template<> struct BuildAST<grammar::v1::formals::ellipsis> { + static void apply0(FormalsState & s, State &) { + s.formals.ellipsis = true; + } +}; + +template<> struct BuildAST<grammar::v1::formals> : change_head<FormalsState> { + static void success0(FormalsState & f, LambdaState & s, State &) { + s.formals = std::make_unique<Formals>(std::move(f.formals)); + } +}; + +struct AttrState : SubexprState { + using SubexprState::SubexprState; + + std::vector<AttrName> attrs; + + template <typename T> + void pushAttr(T && attr, PosIdx) { attrs.emplace_back(std::forward<T>(attr)); } +}; + +template<> struct BuildAST<grammar::v1::attr::simple> { + static void apply(const auto & in, auto & s, State & ps) { + s.pushAttr(ps.symbols.create(in.string_view()), ps.at(in)); + } +}; + +template<> struct BuildAST<grammar::v1::attr::string> { + static void apply(const auto & in, auto & s, State & ps) { + auto e = s->popExprOnly(); + if (auto str = dynamic_cast<ExprString *>(e.get())) + s.pushAttr(ps.symbols.create(str->s), ps.at(in)); + else + s.pushAttr(std::move(e), ps.at(in)); + } +}; + +template<> struct BuildAST<grammar::v1::attr::expr> : BuildAST<grammar::v1::attr::string> {}; + +struct BindingsState : SubexprState { + using SubexprState::SubexprState; + + ExprAttrs attrs; + AttrPath path; + std::unique_ptr<Expr> value; +}; + +struct InheritState : SubexprState { + using SubexprState::SubexprState; + + std::vector<std::pair<AttrName, PosIdx>> attrs; + std::unique_ptr<Expr> from; + PosIdx fromPos; + + template <typename T> + void pushAttr(T && attr, PosIdx pos) { attrs.emplace_back(std::forward<T>(attr), pos); } +}; + +template<> struct BuildAST<grammar::v1::inherit::from> { + static void apply(const auto & in, InheritState & s, State & ps) { + s.from = s->popExprOnly(); + s.fromPos = ps.at(in); + } +}; + +template<> struct BuildAST<grammar::v1::inherit> : change_head<InheritState> { + static void success0(InheritState & s, BindingsState & b, State & ps) { + auto & attrs = b.attrs.attrs; + // TODO this should not reuse generic attrpath rules. + for (auto & [i, iPos] : s.attrs) { + if (i.symbol) + continue; + if (auto str = dynamic_cast<ExprString *>(i.expr.get())) + i = AttrName(ps.symbols.create(str->s)); + else { + throw ParseError({ + .msg = HintFmt("dynamic attributes not allowed in inherit"), + .pos = ps.positions[iPos] + }); + } + } + if (s.from != nullptr) { + if (!b.attrs.inheritFromExprs) + b.attrs.inheritFromExprs = std::make_unique<std::vector<ref<Expr>>>(); + auto fromExpr = ref<Expr>(std::move(s.from)); + b.attrs.inheritFromExprs->push_back(fromExpr); + for (auto & [i, iPos] : s.attrs) { + if (attrs.find(i.symbol) != attrs.end()) + ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); + auto inheritFrom = std::make_unique<ExprInheritFrom>( + s.fromPos, + b.attrs.inheritFromExprs->size() - 1, + fromExpr + ); + attrs.emplace( + i.symbol, + ExprAttrs::AttrDef( + std::make_unique<ExprSelect>(iPos, std::move(inheritFrom), i.symbol), + iPos, + ExprAttrs::AttrDef::Kind::InheritedFrom)); + } + } else { + for (auto & [i, iPos] : s.attrs) { + if (attrs.find(i.symbol) != attrs.end()) + ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); + attrs.emplace( + i.symbol, + ExprAttrs::AttrDef( + std::make_unique<ExprVar>(iPos, i.symbol), + iPos, + ExprAttrs::AttrDef::Kind::Inherited)); + } + } + } +}; + +template<> struct BuildAST<grammar::v1::binding::path> : change_head<AttrState> { + static void success0(AttrState & a, BindingsState & s, State & ps) { + s.path = std::move(a.attrs); + } +}; + +template<> struct BuildAST<grammar::v1::binding::value> { + static void apply0(BindingsState & s, State & ps) { + s.value = s->popExprOnly(); + } +}; + +template<> struct BuildAST<grammar::v1::binding> { + static void apply(const auto & in, BindingsState & s, State & ps) { + ps.addAttr(&s.attrs, std::move(s.path), std::move(s.value), ps.at(in)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::id> { + static void apply(const auto & in, ExprState & s, State & ps) { + if (in.string_view() == "__curPos") + s.pushExpr<ExprPos>(ps.at(in), ps.at(in)); + else + s.pushExpr<ExprVar>(ps.at(in), ps.at(in), ps.symbols.create(in.string_view())); + } +}; + +template<> struct BuildAST<grammar::v1::expr::int_> { + static void apply(const auto & in, ExprState & s, State & ps) { + int64_t v; + if (std::from_chars(in.begin(), in.end(), v).ec != std::errc{}) { + throw ParseError({ + .msg = HintFmt("invalid integer '%1%'", in.string_view()), + .pos = ps.positions[ps.at(in)], + }); + } + s.pushExpr<ExprInt>(noPos, v); + } +}; + +template<> struct BuildAST<grammar::v1::expr::float_> { + static void apply(const auto & in, ExprState & s, State & ps) { + // copy the input into a temporary string so we can call stod. + // can't use from_chars because libc++ (thus darwin) does not have it, + // and floats are not performance-sensitive anyway. if they were you'd + // be in much bigger trouble than this. + // + // we also get to do a locale-save dance because stod is locale-aware and + // something (a plugin?) may have called setlocale or uselocale. + static struct locale_hack { + locale_t posix; + locale_hack(): posix(newlocale(LC_ALL_MASK, "POSIX", 0)) + { + if (posix == 0) + throw SysError("could not get POSIX locale"); + } + } locale; + + auto tmp = in.string(); + double v = [&] { + auto oldLocale = uselocale(locale.posix); + Finally resetLocale([=] { uselocale(oldLocale); }); + try { + return std::stod(tmp); + } catch (...) { + throw ParseError({ + .msg = HintFmt("invalid float '%1%'", in.string_view()), + .pos = ps.positions[ps.at(in)], + }); + } + }(); + s.pushExpr<ExprFloat>(noPos, v); + } +}; + +struct StringState : SubexprState { + using SubexprState::SubexprState; + + std::string currentLiteral; + PosIdx currentPos; + std::vector<std::pair<nix::PosIdx, std::unique_ptr<Expr>>> parts; + + void append(PosIdx pos, std::string_view s) + { + if (currentLiteral.empty()) + currentPos = pos; + currentLiteral += s; + } + + // FIXME this truncates strings on NUL for compat with the old parser. ideally + // we should use the decomposition the g gives us instead of iterating over + // the entire string again. + static void unescapeStr(std::string & str) + { + char * s = str.data(); + char * t = s; + char c; + while ((c = *s++)) { + if (c == '\\') { + c = *s++; + if (c == 'n') *t = '\n'; + else if (c == 'r') *t = '\r'; + else if (c == 't') *t = '\t'; + else *t = c; + } + else if (c == '\r') { + /* Normalise CR and CR/LF into LF. */ + *t = '\n'; + if (*s == '\n') s++; /* cr/lf */ + } + else *t = c; + t++; + } + str.resize(t - str.data()); + } + + void endLiteral() + { + if (!currentLiteral.empty()) { + unescapeStr(currentLiteral); + parts.emplace_back(currentPos, std::make_unique<ExprString>(std::move(currentLiteral))); + } + } + + std::unique_ptr<Expr> finish() + { + if (parts.empty()) { + unescapeStr(currentLiteral); + return std::make_unique<ExprString>(std::move(currentLiteral)); + } else { + endLiteral(); + auto pos = parts[0].first; + return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts)); + } + } +}; + +template<typename... Content> struct BuildAST<grammar::v1::string::literal<Content...>> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::string::cr_lf> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), in.string_view()); // FIXME compat with old parser + } +}; + +template<> struct BuildAST<grammar::v1::string::interpolation> { + static void apply(const auto & in, StringState & s, State & ps) { + s.endLiteral(); + s.parts.emplace_back(ps.at(in), s->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::string::escape> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), "\\"); // FIXME compat with old parser + s.append(ps.at(in), in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::string> : change_head<StringState> { + static void success0(StringState & s, ExprState & e, State &) { + e.exprs.emplace_back(noPos, s.finish()); + } +}; + +struct IndStringState : SubexprState { + using SubexprState::SubexprState; + + std::vector<IndStringLine> lines; +}; + +template<> struct BuildAST<grammar::v1::ind_string::line_start> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.push_back(IndStringLine { in.string_view(), ps.at(in) }); + } +}; + +template<typename... Content> +struct BuildAST<grammar::v1::ind_string::literal<Content...>> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.back().parts.emplace_back(ps.at(in), in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::ind_string::interpolation> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.back().parts.emplace_back(ps.at(in), s->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::ind_string::escape> { + static void apply(const auto & in, IndStringState & s, State & ps) { + switch (*in.begin()) { + case 'n': s.lines.back().parts.emplace_back(ps.at(in), "\n"); break; + case 'r': s.lines.back().parts.emplace_back(ps.at(in), "\r"); break; + case 't': s.lines.back().parts.emplace_back(ps.at(in), "\t"); break; + default: s.lines.back().parts.emplace_back(ps.at(in), in.string_view()); break; + } + } +}; + +template<> struct BuildAST<grammar::v1::ind_string::has_content> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.back().hasContent = true; + } +}; + +template<> struct BuildAST<grammar::v1::ind_string> : change_head<IndStringState> { + static void success(const auto & in, IndStringState & s, ExprState & e, State & ps) { + e.exprs.emplace_back(noPos, ps.stripIndentation(ps.at(in), std::move(s.lines))); + } +}; + +template<typename... Content> struct BuildAST<grammar::v1::path::literal<Content...>> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), in.string_view()); + s.endLiteral(); + } +}; + +template<> struct BuildAST<grammar::v1::path::interpolation> : BuildAST<grammar::v1::string::interpolation> {}; + +template<> struct BuildAST<grammar::v1::path::anchor> { + static void apply(const auto & in, StringState & s, State & ps) { + Path path(absPath(in.string(), ps.basePath.path.abs())); + /* add back in the trailing '/' to the first segment */ + if (in.string_view().ends_with('/') && in.size() > 1) + path += "/"; + s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); + } +}; + +template<> struct BuildAST<grammar::v1::path::home_anchor> { + static void apply(const auto & in, StringState & s, State & ps) { + if (evalSettings.pureEval) + throw Error("the path '%s' can not be resolved in pure mode", in.string_view()); + Path path(getHome() + in.string_view().substr(1)); + s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); + } +}; + +template<> struct BuildAST<grammar::v1::path::searched_path> { + static void apply(const auto & in, StringState & s, State & ps) { + std::vector<std::unique_ptr<Expr>> args{2}; + args[0] = std::make_unique<ExprVar>(ps.s.nixPath); + args[1] = std::make_unique<ExprString>(in.string()); + s.parts.emplace_back( + ps.at(in), + std::make_unique<ExprCall>( + ps.at(in), + std::make_unique<ExprVar>(ps.s.findFile), + std::move(args))); + } +}; + +template<> struct BuildAST<grammar::v1::path> : change_head<StringState> { + template<typename E> + static void check_slash(PosIdx end, StringState & s, State & ps) { + auto e = dynamic_cast<E *>(s.parts.back().second.get()); + if (!e || !e->s.ends_with('/')) + return; + if (s.parts.size() > 1 || e->s != "/") + throw ParseError({ + .msg = HintFmt("path has a trailing slash"), + .pos = ps.positions[end], + }); + } + + static void success(const auto & in, StringState & s, ExprState & e, State & ps) { + s.endLiteral(); + check_slash<ExprPath>(ps.atEnd(in), s, ps); + check_slash<ExprString>(ps.atEnd(in), s, ps); + if (s.parts.size() == 1) { + e.exprs.emplace_back(noPos, std::move(s.parts.back().second)); + } else { + e.pushExpr<ExprConcatStrings>(ps.at(in), ps.at(in), false, std::move(s.parts)); + } + } +}; + +// strings and paths sare handled fully by the grammar-level rule for now +template<> struct BuildAST<grammar::v1::expr::string> : p::maybe_nothing {}; +template<> struct BuildAST<grammar::v1::expr::ind_string> : p::maybe_nothing {}; +template<> struct BuildAST<grammar::v1::expr::path> : p::maybe_nothing {}; + +template<> struct BuildAST<grammar::v1::expr::uri> { + static void apply(const auto & in, ExprState & s, State & ps) { + bool URLLiterals = ps.featureSettings.isEnabled(Dep::UrlLiterals); + if (!URLLiterals) + throw ParseError({ + .msg = HintFmt("URL literals are deprecated, allow using them with %s", "--extra-deprecated-features url-literals"), + .pos = ps.positions[ps.at(in)] + }); + s.pushExpr<ExprString>(ps.at(in), in.string()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::ancient_let> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + // Added 2024-09-18. Turn into an error at some point in the future. + // See the documentation on deprecated features for more details. + if (!ps.featureSettings.isEnabled(Dep::AncientLet)) + warn( + "%s found at %s. This feature is deprecated and will be removed in the future. Use %s to silence this warning.", + "let {", + ps.positions[ps.at(in)], + "--extra-deprecated-features ancient-let" + ); + + b.attrs.pos = ps.at(in); + b.attrs.recursive = true; + s.pushExpr<ExprSelect>(b.attrs.pos, b.attrs.pos, std::make_unique<ExprAttrs>(std::move(b.attrs)), ps.s.body); + } +}; + +template<> struct BuildAST<grammar::v1::expr::rec_set> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + // Before inserting new attrs, check for __override and throw an error + // (the error will initially be a warning to ease migration) + if (!featureSettings.isEnabled(Dep::RecSetOverrides) && b.attrs.attrs.contains(ps.s.overrides)) { + ps.overridesFound(ps.at(in)); + } + + b.attrs.pos = ps.at(in); + b.attrs.recursive = true; + s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::set> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + b.attrs.pos = ps.at(in); + s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); + } +}; + +using ListState = std::vector<std::unique_ptr<Expr>>; + +template<> struct BuildAST<grammar::v1::expr::list> : change_head<ListState> { + static void success(const auto & in, ListState & ls, ExprState & s, State & ps) { + auto e = std::make_unique<ExprList>(); + e->elems = std::move(ls); + s.exprs.emplace_back(ps.at(in), std::move(e)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::list::entry> : change_head<ExprState> { + static void success0(ExprState & e, ListState & s, State & ps) { + s.emplace_back(e.finish(ps).second); + } +}; + +struct SelectState : SubexprState { + using SubexprState::SubexprState; + + PosIdx pos; + ExprSelect * e = nullptr; +}; + +template<> struct BuildAST<grammar::v1::expr::select::head> { + static void apply(const auto & in, SelectState & s, State & ps) { + s.pos = ps.at(in); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select::attr> : change_head<AttrState> { + static void success0(AttrState & a, SelectState & s, State &) { + s.e = &s->pushExpr<ExprSelect>(s.pos, s.pos, s->popExprOnly(), std::move(a.attrs), nullptr); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select::attr_or> { + static void apply0(SelectState & s, State &) { + s.e->def = s->popExprOnly(); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select::as_app_or> { + static void apply(const auto & in, SelectState & s, State & ps) { + std::vector<std::unique_ptr<Expr>> args(1); + args[0] = std::make_unique<ExprVar>(ps.at(in), ps.s.or_); + s->pushExpr<ExprCall>(s.pos, s.pos, s->popExprOnly(), std::move(args)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select> : change_head<SelectState> { + static void success0(const auto &...) {} +}; + +struct AppState : SubexprState { + using SubexprState::SubexprState; + + PosIdx pos; + ExprCall * e = nullptr; +}; + +template<> struct BuildAST<grammar::v1::expr::app::select_or_fn> { + static void apply(const auto & in, AppState & s, State & ps) { + s.pos = ps.at(in); + } +}; + +template<> struct BuildAST<grammar::v1::expr::app::first_arg> { + static void apply(auto & in, AppState & s, State & ps) { + auto arg = s->popExprOnly(), fn = s->popExprOnly(); + if ((s.e = dynamic_cast<ExprCall *>(fn.get()))) { + // TODO remove. + // AST compat with old parser, semantics are the same. + // this can happen on occasions such as `<p> <p>` or `a or b or`, + // neither of which are super worth optimizing. + s.e->args.push_back(std::move(arg)); + s->exprs.emplace_back(noPos, std::move(fn)); + } else { + std::vector<std::unique_ptr<Expr>> args{1}; + args[0] = std::move(arg); + s.e = &s->pushExpr<ExprCall>(s.pos, s.pos, std::move(fn), std::move(args)); + } + } +}; + +template<> struct BuildAST<grammar::v1::expr::app::another_arg> { + static void apply0(AppState & s, State & ps) { + s.e->args.push_back(s->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::app> : change_head<AppState> { + static void success0(const auto &...) {} +}; + +template<typename Op> struct BuildAST<grammar::v1::expr::operator_<Op>> { + static void apply(const auto & in, ExprState & s, State & ps) { + s.pushOp(ps.at(in), Op{}, ps); + } +}; +template<> struct BuildAST<grammar::v1::expr::operator_<grammar::v1::op::has_attr>> : change_head<AttrState> { + static void success(const auto & in, AttrState & a, ExprState & s, State & ps) { + s.pushOp(ps.at(in), ExprState::has_attr{{}, std::move(a.attrs)}, ps); + } +}; + +template<> struct BuildAST<grammar::v1::expr::lambda::arg> { + static void apply(const auto & in, LambdaState & s, State & ps) { + s.arg = ps.symbols.create(in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::lambda> : change_head<LambdaState> { + static void success(const auto & in, LambdaState & l, ExprState & s, State & ps) { + if (l.formals) + l.formals = ps.validateFormals(std::move(l.formals), ps.at(in), l.arg); + s.pushExpr<ExprLambda>(ps.at(in), ps.at(in), l.arg, std::move(l.formals), l->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::assert_> { + static void apply(const auto & in, ExprState & s, State & ps) { + auto body = s.popExprOnly(), cond = s.popExprOnly(); + s.pushExpr<ExprAssert>(ps.at(in), ps.at(in), std::move(cond), std::move(body)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::with> { + static void apply(const auto & in, ExprState & s, State & ps) { + auto body = s.popExprOnly(), scope = s.popExprOnly(); + s.pushExpr<ExprWith>(ps.at(in), ps.at(in), std::move(scope), std::move(body)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::let> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + if (!b.attrs.dynamicAttrs.empty()) + throw ParseError({ + .msg = HintFmt("dynamic attributes not allowed in let"), + .pos = ps.positions[ps.at(in)] + }); + + s.pushExpr<ExprLet>(ps.at(in), std::make_unique<ExprAttrs>(std::move(b.attrs)), b->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::if_> { + static void apply(const auto & in, ExprState & s, State & ps) { + auto else_ = s.popExprOnly(), then = s.popExprOnly(), cond = s.popExprOnly(); + s.pushExpr<ExprIf>(ps.at(in), ps.at(in), std::move(cond), std::move(then), std::move(else_)); + } +}; + +template<> struct BuildAST<grammar::v1::expr> : change_head<ExprState> { + static void success0(ExprState & inner, ExprState & outer, State & ps) { + outer.exprs.push_back(inner.finish(ps)); + } +}; + +} +} diff --git a/src/libexpr/parser/parser.cc b/src/libexpr/parser/parser.cc index 17463056c..896a54981 100644 --- a/src/libexpr/parser/parser.cc +++ b/src/libexpr/parser/parser.cc @@ -14,857 +14,11 @@ #include <charconv> #include <memory> -// flip this define when doing parser development to enable some g checks. -#if 0 -#include <tao/pegtl/contrib/analyze.hpp> -#define ANALYZE_GRAMMAR \ - ([] { \ - const std::size_t issues = tao::pegtl::analyze<grammar::root>(); \ - assert(issues == 0); \ - })() -#else -#define ANALYZE_GRAMMAR ((void) 0) -#endif - -namespace p = tao::pegtl; - -namespace nix::parser { -namespace { - -template<typename> -inline constexpr const char * error_message = nullptr; - -#define error_message_for(...) \ - template<> inline constexpr auto error_message<__VA_ARGS__> - -error_message_for(p::one<'{'>) = "expecting '{'"; -error_message_for(p::one<'}'>) = "expecting '}'"; -error_message_for(p::one<'"'>) = "expecting '\"'"; -error_message_for(p::one<';'>) = "expecting ';'"; -error_message_for(p::one<')'>) = "expecting ')'"; -error_message_for(p::one<']'>) = "expecting ']'"; -error_message_for(p::one<':'>) = "expecting ':'"; -error_message_for(p::string<'\'', '\''>) = "expecting \"''\""; -error_message_for(p::any) = "expecting any character"; -error_message_for(grammar::eof) = "expecting end of file"; -error_message_for(grammar::seps) = "expecting separators"; -error_message_for(grammar::path::forbid_prefix_triple_slash) = "too many slashes in path"; -error_message_for(grammar::path::forbid_prefix_double_slash_no_interp) = "path has a trailing slash"; -error_message_for(grammar::expr) = "expecting expression"; -error_message_for(grammar::expr::unary) = "expecting expression"; -error_message_for(grammar::binding::equal) = "expecting '='"; -error_message_for(grammar::expr::lambda::arg) = "expecting identifier"; -error_message_for(grammar::formals) = "expecting formals"; -error_message_for(grammar::attrpath) = "expecting attribute path"; -error_message_for(grammar::expr::select) = "expecting selection expression"; -error_message_for(grammar::t::kw_then) = "expecting 'then'"; -error_message_for(grammar::t::kw_else) = "expecting 'else'"; -error_message_for(grammar::t::kw_in) = "expecting 'in'"; - -struct SyntaxErrors -{ - template<typename Rule> - static constexpr auto message = error_message<Rule>; - - template<typename Rule> - static constexpr bool raise_on_failure = false; -}; - -template<typename Rule> -struct Control : p::must_if<SyntaxErrors>::control<Rule> -{ - template<typename ParseInput, typename... States> - [[noreturn]] static void raise(const ParseInput & in, States &&... st) - { - if (in.empty()) { - std::string expected; - if constexpr (constexpr auto msg = error_message<Rule>) - expected = fmt(", %s", msg); - throw p::parse_error("unexpected end of file" + expected, in); - } - p::must_if<SyntaxErrors>::control<Rule>::raise(in, st...); - } -}; - -struct ExprState - : grammar:: - operator_semantics<ExprState, PosIdx, AttrPath, std::pair<PosIdx, std::unique_ptr<Expr>>> -{ - std::unique_ptr<Expr> popExprOnly() { - return std::move(popExpr().second); - } - - template<typename Op, typename... Args> - std::unique_ptr<Expr> applyUnary(Args &&... args) { - return std::make_unique<Op>(popExprOnly(), std::forward<Args>(args)...); - } - - template<typename Op> - std::unique_ptr<Expr> applyBinary(PosIdx pos) { - auto right = popExprOnly(), left = popExprOnly(); - return std::make_unique<Op>(pos, std::move(left), std::move(right)); - } - - std::unique_ptr<Expr> call(PosIdx pos, Symbol fn, bool flip = false) - { - std::vector<std::unique_ptr<Expr>> args(2); - args[flip ? 0 : 1] = popExprOnly(); - args[flip ? 1 : 0] = popExprOnly(); - return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args)); - } - - std::unique_ptr<Expr> pipe(PosIdx pos, State & state, bool flip = false) - { - if (!state.featureSettings.isEnabled(Xp::PipeOperator)) - throw ParseError({ - .msg = HintFmt("Pipe operator is disabled"), - .pos = state.positions[pos] - }); - - // Reverse the order compared to normal function application: arg |> fn - std::unique_ptr<Expr> fn, arg; - if (flip) { - fn = popExprOnly(); - arg = popExprOnly(); - } else { - arg = popExprOnly(); - fn = popExprOnly(); - } - std::vector<std::unique_ptr<Expr>> args{1}; - args[0] = std::move(arg); - - return std::make_unique<ExprCall>(pos, std::move(fn), std::move(args)); - } - - std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state) - { - return call(pos, state.s.lessThan, !less); - } - - std::unique_ptr<Expr> concatStrings(PosIdx pos) - { - std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> args(2); - args[1] = popExpr(); - args[0] = popExpr(); - return std::make_unique<ExprConcatStrings>(pos, false, std::move(args)); - } - - std::unique_ptr<Expr> negate(PosIdx pos, State & state) - { - std::vector<std::unique_ptr<Expr>> args(2); - args[0] = std::make_unique<ExprInt>(0); - args[1] = popExprOnly(); - return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(state.s.sub), std::move(args)); - } - - std::pair<PosIdx, std::unique_ptr<Expr>> applyOp(PosIdx pos, auto & op, State & state) { - using Op = grammar::op; - - auto not_ = [] (auto e) { - return std::make_unique<ExprOpNot>(std::move(e)); - }; - - return { - pos, - (overloaded { - [&] (Op::implies) { return applyBinary<ExprOpImpl>(pos); }, - [&] (Op::or_) { return applyBinary<ExprOpOr>(pos); }, - [&] (Op::and_) { return applyBinary<ExprOpAnd>(pos); }, - [&] (Op::equals) { return applyBinary<ExprOpEq>(pos); }, - [&] (Op::not_equals) { return applyBinary<ExprOpNEq>(pos); }, - [&] (Op::less) { return order(pos, true, state); }, - [&] (Op::greater_eq) { return not_(order(pos, true, state)); }, - [&] (Op::greater) { return order(pos, false, state); }, - [&] (Op::less_eq) { return not_(order(pos, false, state)); }, - [&] (Op::update) { return applyBinary<ExprOpUpdate>(pos); }, - [&] (Op::not_) { return applyUnary<ExprOpNot>(); }, - [&] (Op::plus) { return concatStrings(pos); }, - [&] (Op::minus) { return call(pos, state.s.sub); }, - [&] (Op::mul) { return call(pos, state.s.mul); }, - [&] (Op::div) { return call(pos, state.s.div); }, - [&] (Op::concat) { return applyBinary<ExprOpConcatLists>(pos); }, - [&] (has_attr & a) { return applyUnary<ExprOpHasAttr>(std::move(a.path)); }, - [&] (Op::unary_minus) { return negate(pos, state); }, - [&] (Op::pipe_right) { return pipe(pos, state, true); }, - [&] (Op::pipe_left) { return pipe(pos, state); }, - })(op) - }; - } - - // always_inline is needed, otherwise pushOp slows down considerably - [[noreturn, gnu::always_inline]] - static void badOperator(PosIdx pos, State & state) - { - throw ParseError({ - .msg = HintFmt("syntax error, unexpected operator"), - .pos = state.positions[pos] - }); - } - - template<typename Expr, typename... Args> - Expr & pushExpr(PosIdx pos, Args && ... args) - { - auto p = std::make_unique<Expr>(std::forward<Args>(args)...); - auto & result = *p; - exprs.emplace_back(pos, std::move(p)); - return result; - } -}; - -struct SubexprState { -private: - ExprState * up; - -public: - explicit SubexprState(ExprState & up, auto &...) : up(&up) {} - operator ExprState &() { return *up; } - ExprState * operator->() { return up; } -}; - - - -template<typename Rule> -struct BuildAST : grammar::nothing<Rule> {}; - -struct LambdaState : SubexprState { - using SubexprState::SubexprState; - - Symbol arg; - std::unique_ptr<Formals> formals; -}; - -struct FormalsState : SubexprState { - using SubexprState::SubexprState; - - Formals formals{}; - Formal formal{}; -}; - -template<> struct BuildAST<grammar::formal::name> { - static void apply(const auto & in, FormalsState & s, State & ps) { - s.formal = { - .pos = ps.at(in), - .name = ps.symbols.create(in.string_view()), - }; - } -}; - -template<> struct BuildAST<grammar::formal> { - static void apply0(FormalsState & s, State &) { - s.formals.formals.emplace_back(std::move(s.formal)); - } -}; - -template<> struct BuildAST<grammar::formal::default_value> { - static void apply0(FormalsState & s, State & ps) { - s.formal.def = s->popExprOnly(); - } -}; - -template<> struct BuildAST<grammar::formals::ellipsis> { - static void apply0(FormalsState & s, State &) { - s.formals.ellipsis = true; - } -}; - -template<> struct BuildAST<grammar::formals> : change_head<FormalsState> { - static void success0(FormalsState & f, LambdaState & s, State &) { - s.formals = std::make_unique<Formals>(std::move(f.formals)); - } -}; - -struct AttrState : SubexprState { - using SubexprState::SubexprState; - - std::vector<AttrName> attrs; - - template <typename T> - void pushAttr(T && attr, PosIdx) { attrs.emplace_back(std::forward<T>(attr)); } -}; - -template<> struct BuildAST<grammar::attr::simple> { - static void apply(const auto & in, auto & s, State & ps) { - s.pushAttr(ps.symbols.create(in.string_view()), ps.at(in)); - } -}; - -template<> struct BuildAST<grammar::attr::string> { - static void apply(const auto & in, auto & s, State & ps) { - auto e = s->popExprOnly(); - if (auto str = dynamic_cast<ExprString *>(e.get())) - s.pushAttr(ps.symbols.create(str->s), ps.at(in)); - else - s.pushAttr(std::move(e), ps.at(in)); - } -}; - -template<> struct BuildAST<grammar::attr::expr> : BuildAST<grammar::attr::string> {}; - -struct BindingsState : SubexprState { - using SubexprState::SubexprState; - - ExprAttrs attrs; - AttrPath path; - std::unique_ptr<Expr> value; -}; - -struct InheritState : SubexprState { - using SubexprState::SubexprState; - - std::vector<std::pair<AttrName, PosIdx>> attrs; - std::unique_ptr<Expr> from; - PosIdx fromPos; - - template <typename T> - void pushAttr(T && attr, PosIdx pos) { attrs.emplace_back(std::forward<T>(attr), pos); } -}; - -template<> struct BuildAST<grammar::inherit::from> { - static void apply(const auto & in, InheritState & s, State & ps) { - s.from = s->popExprOnly(); - s.fromPos = ps.at(in); - } -}; - -template<> struct BuildAST<grammar::inherit> : change_head<InheritState> { - static void success0(InheritState & s, BindingsState & b, State & ps) { - auto & attrs = b.attrs.attrs; - // TODO this should not reuse generic attrpath rules. - for (auto & [i, iPos] : s.attrs) { - if (i.symbol) - continue; - if (auto str = dynamic_cast<ExprString *>(i.expr.get())) - i = AttrName(ps.symbols.create(str->s)); - else { - throw ParseError({ - .msg = HintFmt("dynamic attributes not allowed in inherit"), - .pos = ps.positions[iPos] - }); - } - } - if (s.from != nullptr) { - if (!b.attrs.inheritFromExprs) - b.attrs.inheritFromExprs = std::make_unique<std::vector<ref<Expr>>>(); - auto fromExpr = ref<Expr>(std::move(s.from)); - b.attrs.inheritFromExprs->push_back(fromExpr); - for (auto & [i, iPos] : s.attrs) { - if (attrs.find(i.symbol) != attrs.end()) - ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); - auto inheritFrom = std::make_unique<ExprInheritFrom>( - s.fromPos, - b.attrs.inheritFromExprs->size() - 1, - fromExpr - ); - attrs.emplace( - i.symbol, - ExprAttrs::AttrDef( - std::make_unique<ExprSelect>(iPos, std::move(inheritFrom), i.symbol), - iPos, - ExprAttrs::AttrDef::Kind::InheritedFrom)); - } - } else { - for (auto & [i, iPos] : s.attrs) { - if (attrs.find(i.symbol) != attrs.end()) - ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); - attrs.emplace( - i.symbol, - ExprAttrs::AttrDef( - std::make_unique<ExprVar>(iPos, i.symbol), - iPos, - ExprAttrs::AttrDef::Kind::Inherited)); - } - } - } -}; - -template<> struct BuildAST<grammar::binding::path> : change_head<AttrState> { - static void success0(AttrState & a, BindingsState & s, State & ps) { - s.path = std::move(a.attrs); - } -}; - -template<> struct BuildAST<grammar::binding::value> { - static void apply0(BindingsState & s, State & ps) { - s.value = s->popExprOnly(); - } -}; - -template<> struct BuildAST<grammar::binding> { - static void apply(const auto & in, BindingsState & s, State & ps) { - ps.addAttr(&s.attrs, std::move(s.path), std::move(s.value), ps.at(in)); - } -}; - -template<> struct BuildAST<grammar::expr::id> { - static void apply(const auto & in, ExprState & s, State & ps) { - if (in.string_view() == "__curPos") - s.pushExpr<ExprPos>(ps.at(in), ps.at(in)); - else - s.pushExpr<ExprVar>(ps.at(in), ps.at(in), ps.symbols.create(in.string_view())); - } -}; - -template<> struct BuildAST<grammar::expr::int_> { - static void apply(const auto & in, ExprState & s, State & ps) { - int64_t v; - if (std::from_chars(in.begin(), in.end(), v).ec != std::errc{}) { - throw ParseError({ - .msg = HintFmt("invalid integer '%1%'", in.string_view()), - .pos = ps.positions[ps.at(in)], - }); - } - s.pushExpr<ExprInt>(noPos, v); - } -}; - -template<> struct BuildAST<grammar::expr::float_> { - static void apply(const auto & in, ExprState & s, State & ps) { - // copy the input into a temporary string so we can call stod. - // can't use from_chars because libc++ (thus darwin) does not have it, - // and floats are not performance-sensitive anyway. if they were you'd - // be in much bigger trouble than this. - // - // we also get to do a locale-save dance because stod is locale-aware and - // something (a plugin?) may have called setlocale or uselocale. - static struct locale_hack { - locale_t posix; - locale_hack(): posix(newlocale(LC_ALL_MASK, "POSIX", 0)) - { - if (posix == 0) - throw SysError("could not get POSIX locale"); - } - } locale; - - auto tmp = in.string(); - double v = [&] { - auto oldLocale = uselocale(locale.posix); - Finally resetLocale([=] { uselocale(oldLocale); }); - try { - return std::stod(tmp); - } catch (...) { - throw ParseError({ - .msg = HintFmt("invalid float '%1%'", in.string_view()), - .pos = ps.positions[ps.at(in)], - }); - } - }(); - s.pushExpr<ExprFloat>(noPos, v); - } -}; - -struct StringState : SubexprState { - using SubexprState::SubexprState; - - std::string currentLiteral; - PosIdx currentPos; - std::vector<std::pair<nix::PosIdx, std::unique_ptr<Expr>>> parts; - - void append(PosIdx pos, std::string_view s) - { - if (currentLiteral.empty()) - currentPos = pos; - currentLiteral += s; - } - - // FIXME this truncates strings on NUL for compat with the old parser. ideally - // we should use the decomposition the g gives us instead of iterating over - // the entire string again. - static void unescapeStr(std::string & str) - { - char * s = str.data(); - char * t = s; - char c; - while ((c = *s++)) { - if (c == '\\') { - c = *s++; - if (c == 'n') *t = '\n'; - else if (c == 'r') *t = '\r'; - else if (c == 't') *t = '\t'; - else *t = c; - } - else if (c == '\r') { - /* Normalise CR and CR/LF into LF. */ - *t = '\n'; - if (*s == '\n') s++; /* cr/lf */ - } - else *t = c; - t++; - } - str.resize(t - str.data()); - } - - void endLiteral() - { - if (!currentLiteral.empty()) { - unescapeStr(currentLiteral); - parts.emplace_back(currentPos, std::make_unique<ExprString>(std::move(currentLiteral))); - } - } - - std::unique_ptr<Expr> finish() - { - if (parts.empty()) { - unescapeStr(currentLiteral); - return std::make_unique<ExprString>(std::move(currentLiteral)); - } else { - endLiteral(); - auto pos = parts[0].first; - return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts)); - } - } -}; - -template<typename... Content> struct BuildAST<grammar::string::literal<Content...>> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), in.string_view()); - } -}; - -template<> struct BuildAST<grammar::string::cr_lf> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), in.string_view()); // FIXME compat with old parser - } -}; - -template<> struct BuildAST<grammar::string::interpolation> { - static void apply(const auto & in, StringState & s, State & ps) { - s.endLiteral(); - s.parts.emplace_back(ps.at(in), s->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::string::escape> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), "\\"); // FIXME compat with old parser - s.append(ps.at(in), in.string_view()); - } -}; - -template<> struct BuildAST<grammar::string> : change_head<StringState> { - static void success0(StringState & s, ExprState & e, State &) { - e.exprs.emplace_back(noPos, s.finish()); - } -}; - -struct IndStringState : SubexprState { - using SubexprState::SubexprState; - - std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> parts; -}; - -template<bool Indented, typename... Content> -struct BuildAST<grammar::ind_string::literal<Indented, Content...>> { - static void apply(const auto & in, IndStringState & s, State & ps) { - s.parts.emplace_back(ps.at(in), StringToken{in.string_view(), Indented}); - } -}; - -template<> struct BuildAST<grammar::ind_string::interpolation> { - static void apply(const auto & in, IndStringState & s, State & ps) { - s.parts.emplace_back(ps.at(in), s->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::ind_string::escape> { - static void apply(const auto & in, IndStringState & s, State & ps) { - switch (*in.begin()) { - case 'n': s.parts.emplace_back(ps.at(in), StringToken{"\n"}); break; - case 'r': s.parts.emplace_back(ps.at(in), StringToken{"\r"}); break; - case 't': s.parts.emplace_back(ps.at(in), StringToken{"\t"}); break; - default: s.parts.emplace_back(ps.at(in), StringToken{in.string_view()}); break; - } - } -}; - -template<> struct BuildAST<grammar::ind_string> : change_head<IndStringState> { - static void success(const auto & in, IndStringState & s, ExprState & e, State & ps) { - e.exprs.emplace_back(noPos, ps.stripIndentation(ps.at(in), std::move(s.parts))); - } -}; - -template<typename... Content> struct BuildAST<grammar::path::literal<Content...>> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), in.string_view()); - s.endLiteral(); - } -}; - -template<> struct BuildAST<grammar::path::interpolation> : BuildAST<grammar::string::interpolation> {}; - -template<> struct BuildAST<grammar::path::anchor> { - static void apply(const auto & in, StringState & s, State & ps) { - Path path(absPath(in.string(), ps.basePath.path.abs())); - /* add back in the trailing '/' to the first segment */ - if (in.string_view().ends_with('/') && in.size() > 1) - path += "/"; - s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); - } -}; - -template<> struct BuildAST<grammar::path::home_anchor> { - static void apply(const auto & in, StringState & s, State & ps) { - if (evalSettings.pureEval) - throw Error("the path '%s' can not be resolved in pure mode", in.string_view()); - Path path(getHome() + in.string_view().substr(1)); - s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); - } -}; - -template<> struct BuildAST<grammar::path::searched_path> { - static void apply(const auto & in, StringState & s, State & ps) { - std::vector<std::unique_ptr<Expr>> args{2}; - args[0] = std::make_unique<ExprVar>(ps.s.nixPath); - args[1] = std::make_unique<ExprString>(in.string()); - s.parts.emplace_back( - ps.at(in), - std::make_unique<ExprCall>( - ps.at(in), - std::make_unique<ExprVar>(ps.s.findFile), - std::move(args))); - } -}; - -template<> struct BuildAST<grammar::path> : change_head<StringState> { - template<typename E> - static void check_slash(PosIdx end, StringState & s, State & ps) { - auto e = dynamic_cast<E *>(s.parts.back().second.get()); - if (!e || !e->s.ends_with('/')) - return; - if (s.parts.size() > 1 || e->s != "/") - throw ParseError({ - .msg = HintFmt("path has a trailing slash"), - .pos = ps.positions[end], - }); - } - - static void success(const auto & in, StringState & s, ExprState & e, State & ps) { - s.endLiteral(); - check_slash<ExprPath>(ps.atEnd(in), s, ps); - check_slash<ExprString>(ps.atEnd(in), s, ps); - if (s.parts.size() == 1) { - e.exprs.emplace_back(noPos, std::move(s.parts.back().second)); - } else { - e.pushExpr<ExprConcatStrings>(ps.at(in), ps.at(in), false, std::move(s.parts)); - } - } -}; - -// strings and paths sare handled fully by the grammar-level rule for now -template<> struct BuildAST<grammar::expr::string> : p::maybe_nothing {}; -template<> struct BuildAST<grammar::expr::ind_string> : p::maybe_nothing {}; -template<> struct BuildAST<grammar::expr::path> : p::maybe_nothing {}; - -template<> struct BuildAST<grammar::expr::uri> { - static void apply(const auto & in, ExprState & s, State & ps) { - bool URLLiterals = ps.featureSettings.isEnabled(Dep::UrlLiterals); - if (!URLLiterals) - throw ParseError({ - .msg = HintFmt("URL literals are deprecated, allow using them with --extra-deprecated-features=url-literals"), - .pos = ps.positions[ps.at(in)] - }); - s.pushExpr<ExprString>(ps.at(in), in.string()); - } -}; - -template<> struct BuildAST<grammar::expr::ancient_let> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - // Added 2024-09-18. Turn into an error at some point in the future. - // See the documentation on deprecated features for more details. - if (!ps.featureSettings.isEnabled(Dep::AncientLet)) - warn( - "%s found at %s. This feature is deprecated and will be removed in the future. Use %s to silence this warning.", - "let {", - ps.positions[ps.at(in)], - "--extra-deprecated-features ancient-let" - ); - - b.attrs.pos = ps.at(in); - b.attrs.recursive = true; - s.pushExpr<ExprSelect>(b.attrs.pos, b.attrs.pos, std::make_unique<ExprAttrs>(std::move(b.attrs)), ps.s.body); - } -}; - -template<> struct BuildAST<grammar::expr::rec_set> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - // Before inserting new attrs, check for __override and throw an error - // (the error will initially be a warning to ease migration) - if (!featureSettings.isEnabled(Dep::RecSetOverrides) && b.attrs.attrs.contains(ps.s.overrides)) { - ps.overridesFound(ps.at(in)); - } - - b.attrs.pos = ps.at(in); - b.attrs.recursive = true; - s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); - } -}; - -template<> struct BuildAST<grammar::expr::set> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - b.attrs.pos = ps.at(in); - s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); - } -}; - -using ListState = std::vector<std::unique_ptr<Expr>>; - -template<> struct BuildAST<grammar::expr::list> : change_head<ListState> { - static void success(const auto & in, ListState & ls, ExprState & s, State & ps) { - auto e = std::make_unique<ExprList>(); - e->elems = std::move(ls); - s.exprs.emplace_back(ps.at(in), std::move(e)); - } -}; - -template<> struct BuildAST<grammar::expr::list::entry> : change_head<ExprState> { - static void success0(ExprState & e, ListState & s, State & ps) { - s.emplace_back(e.finish(ps).second); - } -}; - -struct SelectState : SubexprState { - using SubexprState::SubexprState; - - PosIdx pos; - ExprSelect * e = nullptr; -}; - -template<> struct BuildAST<grammar::expr::select::head> { - static void apply(const auto & in, SelectState & s, State & ps) { - s.pos = ps.at(in); - } -}; - -template<> struct BuildAST<grammar::expr::select::attr> : change_head<AttrState> { - static void success0(AttrState & a, SelectState & s, State &) { - s.e = &s->pushExpr<ExprSelect>(s.pos, s.pos, s->popExprOnly(), std::move(a.attrs), nullptr); - } -}; - -template<> struct BuildAST<grammar::expr::select::attr_or> { - static void apply0(SelectState & s, State &) { - s.e->def = s->popExprOnly(); - } -}; - -template<> struct BuildAST<grammar::expr::select::as_app_or> { - static void apply(const auto & in, SelectState & s, State & ps) { - std::vector<std::unique_ptr<Expr>> args(1); - args[0] = std::make_unique<ExprVar>(ps.at(in), ps.s.or_); - s->pushExpr<ExprCall>(s.pos, s.pos, s->popExprOnly(), std::move(args)); - } -}; - -template<> struct BuildAST<grammar::expr::select> : change_head<SelectState> { - static void success0(const auto &...) {} -}; - -struct AppState : SubexprState { - using SubexprState::SubexprState; - - PosIdx pos; - ExprCall * e = nullptr; -}; - -template<> struct BuildAST<grammar::expr::app::select_or_fn> { - static void apply(const auto & in, AppState & s, State & ps) { - s.pos = ps.at(in); - } -}; - -template<> struct BuildAST<grammar::expr::app::first_arg> { - static void apply(auto & in, AppState & s, State & ps) { - auto arg = s->popExprOnly(), fn = s->popExprOnly(); - if ((s.e = dynamic_cast<ExprCall *>(fn.get()))) { - // TODO remove. - // AST compat with old parser, semantics are the same. - // this can happen on occasions such as `<p> <p>` or `a or b or`, - // neither of which are super worth optimizing. - s.e->args.push_back(std::move(arg)); - s->exprs.emplace_back(noPos, std::move(fn)); - } else { - std::vector<std::unique_ptr<Expr>> args{1}; - args[0] = std::move(arg); - s.e = &s->pushExpr<ExprCall>(s.pos, s.pos, std::move(fn), std::move(args)); - } - } -}; - -template<> struct BuildAST<grammar::expr::app::another_arg> { - static void apply0(AppState & s, State & ps) { - s.e->args.push_back(s->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::expr::app> : change_head<AppState> { - static void success0(const auto &...) {} -}; - -template<typename Op> struct BuildAST<grammar::expr::operator_<Op>> { - static void apply(const auto & in, ExprState & s, State & ps) { - s.pushOp(ps.at(in), Op{}, ps); - } -}; -template<> struct BuildAST<grammar::expr::operator_<grammar::op::has_attr>> : change_head<AttrState> { - static void success(const auto & in, AttrState & a, ExprState & s, State & ps) { - s.pushOp(ps.at(in), ExprState::has_attr{{}, std::move(a.attrs)}, ps); - } -}; - -template<> struct BuildAST<grammar::expr::lambda::arg> { - static void apply(const auto & in, LambdaState & s, State & ps) { - s.arg = ps.symbols.create(in.string_view()); - } -}; - -template<> struct BuildAST<grammar::expr::lambda> : change_head<LambdaState> { - static void success(const auto & in, LambdaState & l, ExprState & s, State & ps) { - if (l.formals) - l.formals = ps.validateFormals(std::move(l.formals), ps.at(in), l.arg); - s.pushExpr<ExprLambda>(ps.at(in), ps.at(in), l.arg, std::move(l.formals), l->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::expr::assert_> { - static void apply(const auto & in, ExprState & s, State & ps) { - auto body = s.popExprOnly(), cond = s.popExprOnly(); - s.pushExpr<ExprAssert>(ps.at(in), ps.at(in), std::move(cond), std::move(body)); - } -}; - -template<> struct BuildAST<grammar::expr::with> { - static void apply(const auto & in, ExprState & s, State & ps) { - auto body = s.popExprOnly(), scope = s.popExprOnly(); - s.pushExpr<ExprWith>(ps.at(in), ps.at(in), std::move(scope), std::move(body)); - } -}; - -template<> struct BuildAST<grammar::expr::let> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - if (!b.attrs.dynamicAttrs.empty()) - throw ParseError({ - .msg = HintFmt("dynamic attributes not allowed in let"), - .pos = ps.positions[ps.at(in)] - }); - - s.pushExpr<ExprLet>(ps.at(in), std::make_unique<ExprAttrs>(std::move(b.attrs)), b->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::expr::if_> { - static void apply(const auto & in, ExprState & s, State & ps) { - auto else_ = s.popExprOnly(), then = s.popExprOnly(), cond = s.popExprOnly(); - s.pushExpr<ExprIf>(ps.at(in), ps.at(in), std::move(cond), std::move(then), std::move(else_)); - } -}; - -template<> struct BuildAST<grammar::expr> : change_head<ExprState> { - static void success0(ExprState & inner, ExprState & outer, State & ps) { - outer.exprs.push_back(inner.finish(ps)); - } -}; - -} -} +// Linter complains that this is a "suspicious include of file with '.cc' extension". +// While that is correct and generally not great, it is one of the less bad options to pick +// in terms of diff noise. +// NOLINTNEXTLINE(bugprone-suspicious-include) +#include "parser-impl1.inc.cc" namespace nix { @@ -884,7 +38,6 @@ Expr * EvalState::parse( exprSymbols, featureSettings, }; - parser::ExprState x; assert(length >= 2); assert(text[length - 1] == 0); @@ -893,7 +46,12 @@ Expr * EvalState::parse( p::string_input<p::tracking_mode::lazy> inp{std::string_view{text, length}, "input"}; try { - p::parse<parser::grammar::root, parser::BuildAST, parser::Control>(inp, x, s); + parser::v1::ExprState x; + p::parse<parser::grammar::v1::root, parser::v1::BuildAST, parser::v1::Control>(inp, x, s); + + auto [_pos, result] = x.finish(s); + result->bindVars(*this, staticEnv); + return result.release(); } catch (p::parse_error & e) { auto pos = e.positions().back(); throw ParseError({ @@ -901,10 +59,6 @@ Expr * EvalState::parse( .pos = positions[s.positions.add(s.origin, pos.byte)] }); } - - auto [_pos, result] = x.finish(s); - result->bindVars(*this, staticEnv); - return result.release(); } } diff --git a/src/libexpr/parser/state.hh b/src/libexpr/parser/state.hh index 1d57e2f5f..b969f73e4 100644 --- a/src/libexpr/parser/state.hh +++ b/src/libexpr/parser/state.hh @@ -6,11 +6,21 @@ namespace nix::parser { -struct StringToken -{ - std::string_view s; - bool hasIndentation; - operator std::string_view() const { return s; } +struct IndStringLine { + // String containing only the leading whitespace of the line. May be empty. + std::string_view indentation; + // Position of the line start (before the indentation) + PosIdx pos; + + // Whether the line contains anything besides indentation and line break + bool hasContent = false; + + std::vector< + std::pair< + PosIdx, + std::variant<std::unique_ptr<Expr>, std::string_view> + > + > parts = {}; }; struct State @@ -27,8 +37,7 @@ struct State void overridesFound(const PosIdx pos); void addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos); std::unique_ptr<Formals> validateFormals(std::unique_ptr<Formals> formals, PosIdx pos = noPos, Symbol arg = {}); - std::unique_ptr<Expr> stripIndentation(const PosIdx pos, - std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es); + std::unique_ptr<Expr> stripIndentation(const PosIdx pos, std::vector<IndStringLine> && line); // lazy positioning means we don't get byte offsets directly, in.position() would work // but also requires line and column (which is expensive) @@ -182,98 +191,87 @@ inline std::unique_ptr<Formals> State::validateFormals(std::unique_ptr<Formals> return formals; } -inline std::unique_ptr<Expr> State::stripIndentation(const PosIdx pos, - std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es) +inline std::unique_ptr<Expr> State::stripIndentation( + const PosIdx pos, + std::vector<IndStringLine> && lines) { - if (es.empty()) return std::make_unique<ExprString>(""); + /* If the only line is whitespace-only, directly return empty string. + * The rest of the code relies on the final string not being empty. + */ + if (lines.size() == 1 && lines.front().parts.empty()) { + return std::make_unique<ExprString>(""); + } - /* Figure out the minimum indentation. Note that by design - whitespace-only final lines are not taken into account. (So - the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ - bool atStartOfLine = true; /* = seen only whitespace in the current line */ + /* If the last line only contains whitespace, trim it to not cause excessive whitespace. + * (Other whitespace-only lines get stripped only of the common indentation, and excess + * whitespace becomes part of the string.) + */ + if (lines.back().parts.empty()) { + lines.back().indentation = {}; + } + + /* Figure out the minimum indentation. Note that by design + whitespace-only lines are not taken into account. */ size_t minIndent = 1000000; - size_t curIndent = 0; - for (auto & [i_pos, i] : es) { - auto * str = std::get_if<StringToken>(&i); - if (!str || !str->hasIndentation) { - /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ - if (atStartOfLine) { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - continue; - } - for (size_t j = 0; j < str->s.size(); ++j) { - if (atStartOfLine) { - if (str->s[j] == ' ') - curIndent++; - else if (str->s[j] == '\n') { - /* Empty line, doesn't influence minimum - indentation. */ - curIndent = 0; - } else { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - } else if (str->s[j] == '\n') { - atStartOfLine = true; - curIndent = 0; - } + for (auto & line : lines) { + if (line.hasContent) { + minIndent = std::min(minIndent, line.indentation.size()); } } /* Strip spaces from each line. */ - std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es2; - atStartOfLine = true; - size_t curDropped = 0; - size_t n = es.size(); - auto i = es.begin(); - const auto trimExpr = [&] (std::unique_ptr<Expr> e) { - atStartOfLine = false; - curDropped = 0; - es2.emplace_back(i->first, std::move(e)); - }; - const auto trimString = [&] (const StringToken t) { - std::string s2; - for (size_t j = 0; j < t.s.size(); ++j) { - if (atStartOfLine) { - if (t.s[j] == ' ') { - if (curDropped++ >= minIndent) - s2 += t.s[j]; - } - else if (t.s[j] == '\n') { - curDropped = 0; - s2 += t.s[j]; - } else { - atStartOfLine = false; - curDropped = 0; - s2 += t.s[j]; - } - } else { - s2 += t.s[j]; - if (t.s[j] == '\n') atStartOfLine = true; - } - } + for (auto & line : lines) { + line.indentation.remove_prefix(std::min(minIndent, line.indentation.size())); + } + + /* Concat the parts together again */ + + std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> parts; + /* Accumulator for merging intermediates */ + PosIdx merged_pos; + std::string merged = ""; - /* Remove the last line if it is empty and consists only of - spaces. */ - if (n == 1) { - std::string::size_type p = s2.find_last_of('\n'); - if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) - s2 = std::string(s2, 0, p + 1); + auto push_merged = [&] (PosIdx i_pos, std::string_view str) { + if (merged.empty()) { + merged_pos = i_pos; } + merged += str; + }; - es2.emplace_back(i->first, std::make_unique<ExprString>(std::move(s2))); + auto flush_merged = [&] () { + if (!merged.empty()) { + parts.emplace_back(merged_pos, std::make_unique<ExprString>(std::string(merged))); + merged.clear(); + } }; - for (; i != es.end(); ++i, --n) { - std::visit(overloaded { trimExpr, trimString }, std::move(i->second)); + + for (auto && [li, line] : enumerate(lines)) { + push_merged(line.pos, line.indentation); + + for (auto & val : line.parts) { + auto &[i_pos, item] = val; + + std::visit(overloaded{ + [&](std::string_view str) { + push_merged(i_pos, str); + }, + [&](std::unique_ptr<Expr> expr) { + flush_merged(); + parts.emplace_back(i_pos, std::move(expr)); + }, + }, std::move(item)); + } } - /* If this is a single string, then don't do a concatenation. */ - if (es2.size() == 1 && dynamic_cast<ExprString *>(es2[0].second.get())) { - return std::move(es2[0].second); + flush_merged(); + + /* If this is a single string, then don't do a concatenation. + * (If it's a single expression, still do the ConcatStrings to properly force it being a string.) + */ + if (parts.size() == 1 && dynamic_cast<ExprString *>(parts[0].second.get())) { + return std::move(parts[0].second); } - return std::make_unique<ExprConcatStrings>(pos, true, std::move(es2)); + return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts)); } } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index b0e14a26e..c98fe2a03 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -394,7 +394,8 @@ static RegisterPrimOp primop_fetchGit({ [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References By default, the `ref` value is prefixed with `refs/heads/`. - As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`. + As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/` or + if `ref` looks like a commit hash for backwards compatibility with CppNix 2.3. - `submodules` (default: `false`) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7d16d3f57..da60bf331 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,3 +1,4 @@ +#include "error.hh" #include "fetchers.hh" #include "cache.hh" #include "globals.hh" @@ -257,6 +258,28 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co } } // end namespace +static std::optional<Path> resolveRefToCachePath( + Input & input, + const Path & cacheDir, + std::vector<Path> & gitRefFileCandidates, + std::function<bool(const Path&)> condition) +{ + if (input.getRef()->starts_with("refs/")) { + Path fullpath = cacheDir + "/" + *input.getRef(); + if (condition(fullpath)) { + return fullpath; + } + } + + for (auto & candidate : gitRefFileCandidates) { + if (condition(candidate)) { + return candidate; + } + } + + return std::nullopt; +} + struct GitInputScheme : InputScheme { std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override @@ -539,10 +562,13 @@ struct GitInputScheme : InputScheme runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); } - Path localRefFile = - input.getRef()->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input.getRef() - : cacheDir + "/refs/heads/" + *input.getRef(); + std::vector<Path> gitRefFileCandidates; + for (auto & infix : {"", "tags/", "heads/"}) { + Path p = cacheDir + "/refs/" + infix + *input.getRef(); + gitRefFileCandidates.push_back(p); + } + + Path localRefFile; bool doFetch; time_t now = time(0); @@ -564,29 +590,70 @@ struct GitInputScheme : InputScheme if (allRefs) { doFetch = true; } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - !isCacheFileWithinTtl(now, st); + std::function<bool(const Path&)> condition; + condition = [&now](const Path & path) { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + return stat(path.c_str(), &st) == 0 && + isCacheFileWithinTtl(now, st); + }; + if (auto result = resolveRefToCachePath( + input, + cacheDir, + gitRefFileCandidates, + condition + )) { + localRefFile = *result; + doFetch = false; + } else { + doFetch = true; + } } } + // When having to fetch, we don't know `localRefFile` yet. + // Because git needs to figure out what we're fetching + // (i.e. is it a rev? a branch? a tag?) if (doFetch) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. + auto ref = input.getRef(); + std::string fetchRef; + if (allRefs) { + fetchRef = "refs/*"; + } else if ( + ref->starts_with("refs/") + || *ref == "HEAD" + || std::regex_match(*ref, revRegex)) + { + fetchRef = *ref; + } else { + fetchRef = "refs/*/" + *ref; + } + try { - auto ref = input.getRef(); - auto fetchRef = allRefs - ? "refs/*" - : ref->compare(0, 5, "refs/") == 0 - ? *ref - : ref == "HEAD" - ? *ref - : "refs/heads/" + *ref; - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, true); + Finally finally([&]() { + if (auto p = resolveRefToCachePath( + input, + cacheDir, + gitRefFileCandidates, + pathExists + )) { + localRefFile = *p; + } + }); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, { + "-C", repoDir, + "--git-dir", gitDir, + "fetch", + "--quiet", + "--force", + "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) + }, true); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); diff --git a/src/libmain/crash-handler.cc b/src/libmain/crash-handler.cc new file mode 100644 index 000000000..3f1b9f7d8 --- /dev/null +++ b/src/libmain/crash-handler.cc @@ -0,0 +1,41 @@ +#include "crash-handler.hh" +#include "fmt.hh" + +#include <boost/core/demangle.hpp> +#include <exception> + +namespace nix { + +namespace { +void onTerminate() +{ + std::cerr << "Lix crashed. This is a bug. We would appreciate if you report it along with what caused it at https://git.lix.systems/lix-project/lix/issues with the following information included:\n\n"; + try { + std::exception_ptr eptr = std::current_exception(); + if (eptr) { + std::rethrow_exception(eptr); + } else { + std::cerr << "std::terminate() called without exception\n"; + } + } catch (const std::exception & ex) { + std::cerr << "Exception: " << boost::core::demangle(typeid(ex).name()) << ": " << ex.what() << "\n"; + } catch (...) { + std::cerr << "Unknown exception! Spooky.\n"; + } + + std::cerr << "Stack trace:\n"; + nix::printStackTrace(); + + std::abort(); +} +} + +void registerCrashHandler() +{ + // DO NOT use this for signals. Boost stacktrace is very much not + // async-signal-safe, and in a world with ASLR, addr2line is pointless. + // + // If you want signals, set up a minidump system and do it out-of-process. + std::set_terminate(onTerminate); +} +} diff --git a/src/libmain/crash-handler.hh b/src/libmain/crash-handler.hh new file mode 100644 index 000000000..4c5641b8c --- /dev/null +++ b/src/libmain/crash-handler.hh @@ -0,0 +1,21 @@ +#pragma once +/// @file Crash handler for Lix that prints back traces (hopefully in instances where it is not just going to crash the process itself). +/* + * Author's note: This will probably be partially/fully supplanted by a + * minidump writer like the following once we get our act together on crashes a + * little bit more: + * https://github.com/rust-minidump/minidump-writer + * https://github.com/EmbarkStudios/crash-handling + * (out of process implementation *should* be able to be done on-demand) + * + * Such an out-of-process implementation could then both make minidumps and + * print stack traces for arbitrarily messed-up process states such that we can + * safely give out backtraces for SIGSEGV and other deadly signals. + */ + +namespace nix { + +/** Registers the Lix crash handler for std::terminate (currently; will support more crashes later). See also detectStackOverflow(). */ +void registerCrashHandler(); + +} diff --git a/src/libmain/meson.build b/src/libmain/meson.build index a7cce287c..a1a888c16 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -1,5 +1,6 @@ libmain_sources = files( 'common-args.cc', + 'crash-handler.cc', 'loggers.cc', 'progress-bar.cc', 'shared.cc', @@ -8,6 +9,7 @@ libmain_sources = files( libmain_headers = files( 'common-args.hh', + 'crash-handler.hh', 'loggers.hh', 'progress-bar.hh', 'shared.hh', diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index bc9548e09..029b457b1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,3 +1,4 @@ +#include "crash-handler.hh" #include "globals.hh" #include "shared.hh" #include "store-api.hh" @@ -118,6 +119,8 @@ static void sigHandler(int signo) { } void initNix() { + registerCrashHandler(); + /* Turn on buffering for cerr. */ static char buf[1024]; std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); @@ -335,12 +338,15 @@ int handleExceptions(const std::string & programName, std::function<void()> fun) } catch (BaseError & e) { logError(e.info()); return e.info().status; - } catch (std::bad_alloc & e) { + } catch (const std::bad_alloc & e) { printError(error + "out of memory"); return 1; - } catch (std::exception & e) { - printError(error + e.what()); - return 1; + } catch (const std::exception & e) { + // Random exceptions bubbling into main are cause for bug reports, crash + std::terminate(); + } catch (...) { + // Explicitly do not tolerate non-std exceptions escaping. + std::terminate(); } return 0; @@ -389,7 +395,7 @@ RunPager::~RunPager() pid.wait(); } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index b41efe567..49b72a54e 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -111,7 +111,7 @@ struct PrintFreed /** - * Install a SIGSEGV handler to detect stack overflows. + * Install a SIGSEGV handler to detect stack overflows. See also registerCrashHandler(). */ void detectStackOverflow(); diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 9634fb944..846c6c9b9 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -47,7 +47,7 @@ struct BuildResult * @todo This should be an entire ErrorInfo object, not just a * string, for richer information. */ - std::string errorMsg; + std::string errorMsg = {}; std::string toString() const { auto strStatus = [&]() { @@ -90,7 +90,7 @@ struct BuildResult * For derivations, a mapping from the names of the wanted outputs * to actual paths. */ - SingleDrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs = {}; /** * The start/stop times of the build (or one of the rounds, if it diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 827c9f541..96140e10b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -11,7 +11,13 @@ #include "drv-output-substitution-goal.hh" #include "strings.hh" +#include <boost/outcome/try.hpp> #include <fstream> +#include <kj/array.h> +#include <kj/async-unix.h> +#include <kj/async.h> +#include <kj/debug.h> +#include <kj/vector.h> #include <sys/types.h> #include <sys/socket.h> #include <sys/un.h> @@ -65,7 +71,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -85,7 +90,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique<Derivation>(drv); - state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); @@ -103,17 +107,7 @@ DerivationGoal::~DerivationGoal() noexcept(false) { /* Careful: we should never ever throw an exception from a destructor. */ - try { closeLogFile(); } catch (...) { ignoreException(); } -} - - -std::string DerivationGoal::key() -{ - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); + try { closeLogFile(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -124,20 +118,24 @@ void DerivationGoal::killChild() } -Goal::Finished DerivationGoal::timedOut(Error && ex) +Goal::WorkResult DerivationGoal::timedOut(Error && ex) { killChild(); return done(BuildResult::TimedOut, {}, std::move(ex)); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::work(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::workImpl() noexcept { - return (this->*state)(inBuildSlot); + return useDerivation ? getDerivation() : haveDerivation(); } -void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) +bool DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) { + if (isDone) { + return false; + } + auto newWanted = wantedOutputs.union_(outputs); switch (needRestart) { case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed: @@ -154,10 +152,11 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) break; }; wantedOutputs = newWanted; + return true; } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::getDerivation(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::getDerivation() noexcept try { trace("init"); @@ -165,18 +164,17 @@ try { exists. If it doesn't, it may be created through a substitute. */ if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - return loadDerivation(inBuildSlot); + co_return co_await loadDerivation(); } - - state = &DerivationGoal::loadDerivation; - return {WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}}; + (co_await waitForGoals(worker.goalFactory().makePathSubstitutionGoal(drvPath))).value(); + co_return co_await loadDerivation(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::loadDerivation(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::loadDerivation() noexcept try { trace("loading derivation"); @@ -207,13 +205,13 @@ try { } assert(drv); - return haveDerivation(inBuildSlot); + return haveDerivation(); } catch (...) { return {std::current_exception()}; } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::haveDerivation(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::haveDerivation() noexcept try { trace("have derivation"); @@ -241,7 +239,7 @@ try { }); } - return gaveUpOnSubstitution(inBuildSlot); + co_return co_await gaveUpOnSubstitution(); } for (auto & i : drv->outputsAndOptPaths(worker.store)) @@ -263,19 +261,19 @@ try { /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - return {done(BuildResult::AlreadyValid, std::move(validOutputs))}; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - WaitForGoals result; + kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies; if (settings.useSubstitutes) { if (parsedDrv->substitutesAllowed()) { for (auto & [outputName, status] : initialOutputs) { if (!status.wanted) continue; if (!status.known) - result.goals.insert( + dependencies.add( worker.goalFactory().makeDrvOutputSubstitutionGoal( DrvOutput{status.outputHash, outputName}, buildMode == bmRepair ? Repair : NoRepair @@ -283,7 +281,7 @@ try { ); else { auto * cap = getDerivationCA(*drv); - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal( + dependencies.add(worker.goalFactory().makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, cap ? std::optional { *cap } : std::nullopt)); @@ -294,17 +292,15 @@ try { } } - if (result.goals.empty()) { /* to prevent hang (no wake-up event) */ - return outputsSubstitutionTried(inBuildSlot); - } else { - state = &DerivationGoal::outputsSubstitutionTried; - return {std::move(result)}; + if (!dependencies.empty()) { /* to prevent hang (no wake-up event) */ + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await outputsSubstitutionTried(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::outputsSubstitutionTried(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::outputsSubstitutionTried() noexcept try { trace("all outputs substituted (maybe)"); @@ -354,7 +350,7 @@ try { if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; - return haveDerivation(inBuildSlot); + return haveDerivation(); } auto [allValid, validOutputs] = checkPathValidity(); @@ -370,7 +366,7 @@ try { worker.store.printStorePath(drvPath)); /* Nothing to wait for; tail call */ - return gaveUpOnSubstitution(inBuildSlot); + return gaveUpOnSubstitution(); } catch (...) { return {std::current_exception()}; } @@ -378,9 +374,9 @@ try { /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::gaveUpOnSubstitution() noexcept try { - WaitForGoals result; + kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies; /* At this point we are building all outputs, so if more are wanted there is no need to restart. */ @@ -393,7 +389,7 @@ try { addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) { if (!inputNode.value.empty()) - result.goals.insert(worker.goalFactory().makeGoal( + dependencies.add(worker.goalFactory().makeGoal( DerivedPath::Built { .drvPath = inputDrv, .outputs = inputNode.value, @@ -438,17 +434,15 @@ try { if (!settings.useSubstitutes) throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); } - if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ - return inputsRealised(inBuildSlot); - } else { - state = &DerivationGoal::inputsRealised; - return {result}; + if (!dependencies.empty()) {/* to prevent hang (no wake-up event) */ + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await inputsRealised(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } @@ -488,7 +482,7 @@ try { } /* Check each path (slow!). */ - WaitForGoals result; + kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies; for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; printError( @@ -496,9 +490,9 @@ try { worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i, Repair)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i, Repair)); else - result.goals.insert(worker.goalFactory().makeGoal( + dependencies.add(worker.goalFactory().makeGoal( DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath2->second), .outputs = OutputsSpec::All { }, @@ -506,18 +500,18 @@ try { bmRepair)); } - if (result.goals.empty()) { - return {done(BuildResult::AlreadyValid, assertPathValidity())}; + if (dependencies.empty()) { + co_return done(BuildResult::AlreadyValid, assertPathValidity()); } - state = &DerivationGoal::closureRepaired; - return {result}; + (co_await waitForGoals(dependencies.releaseAsArray())).value(); + co_return co_await closureRepaired(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::closureRepaired(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::closureRepaired() noexcept try { trace("closure repaired"); if (nrFailed > 0) @@ -529,14 +523,14 @@ try { } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::inputsRealised(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::inputsRealised() noexcept try { trace("all inputs realised"); if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - return {done( + co_return done( BuildResult::DependencyFailed, {}, Error( @@ -544,12 +538,12 @@ try { nrFailed, worker.store.printStorePath(drvPath) ) - )}; + ); } if (retrySubstitution == RetrySubstitution::YesNeed) { retrySubstitution = RetrySubstitution::AlreadyRetried; - return haveDerivation(inBuildSlot); + co_return co_await haveDerivation(); } /* Gather information necessary for computing the closure and/or @@ -611,11 +605,12 @@ try { worker.store.printStorePath(pathResolved), }); - resolvedDrvGoal = worker.goalFactory().makeDerivationGoal( + auto dependency = worker.goalFactory().makeDerivationGoal( pathResolved, wantedOutputs, buildMode); + resolvedDrvGoal = dependency.first; - state = &DerivationGoal::resolvedFinished; - return {WaitForGoals{{resolvedDrvGoal}}}; + (co_await waitForGoals(std::move(dependency))).value(); + co_return co_await resolvedFinished(); } std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths; @@ -679,10 +674,9 @@ try { /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ - state = &DerivationGoal::tryToBuild; - return tryToBuild(inBuildSlot); + co_return co_await tryToBuild(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } void DerivationGoal::started() @@ -698,8 +692,9 @@ void DerivationGoal::started() mcRunningBuilds = worker.runningBuilds.addTemporarily(1); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::tryToBuild(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::tryToBuild() noexcept try { +retry: trace("trying to build"); /* Obtain locks on all output paths, if the paths are known a priori. @@ -733,7 +728,9 @@ try { if (!actLock) actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); - return {WaitForAWhile{}}; + co_await waitForAWhile(); + // we can loop very often, and `co_return co_await` always allocates a new frame + goto retry; } actLock.reset(); @@ -750,7 +747,7 @@ try { if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - return {done(BuildResult::AlreadyValid, std::move(validOutputs))}; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* If any of the outputs already exist but are not valid, delete @@ -770,47 +767,56 @@ try { && settings.maxBuildJobs.get() != 0; if (!buildLocally) { - auto hookReply = tryBuildHook(inBuildSlot); - auto result = std::visit( - overloaded{ - [&](HookReply::Accept & a) -> std::optional<WorkResult> { - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - buildResult.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return WaitForWorld{std::move(a.fds), false}; - }, - [&](HookReply::Postpone) -> std::optional<WorkResult> { - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique<Activity>(*logger, lvlTalkative, actBuildWaiting, - fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); - outputLocks.unlock(); - return WaitForAWhile{}; - }, - [&](HookReply::Decline) -> std::optional<WorkResult> { - /* We should do it ourselves. */ - return std::nullopt; - }, - }, - hookReply); - if (result) { - return {std::move(*result)}; + auto hookReply = tryBuildHook(); + switch (hookReply.index()) { + case 0: { + HookReply::Accept & a = std::get<0>(hookReply); + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + actLock.reset(); + buildResult.startTime = time(0); // inexact + started(); + auto r = co_await a.promise; + if (r.has_value()) { + co_return co_await buildDone(); + } else if (r.has_error()) { + co_return r.assume_error(); + } else { + co_return r.assume_exception(); + } + } + + case 1: { + HookReply::Decline _ [[gnu::unused]] = std::get<1>(hookReply); + break; + } + + case 2: { + HookReply::Postpone _ [[gnu::unused]] = std::get<2>(hookReply); + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlTalkative, actBuildWaiting, + fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); + outputLocks.unlock(); + co_await waitForAWhile(); + goto retry; + } + + default: + // can't static_assert this because HookReply *subclasses* variant and std::variant_size breaks + assert(false && "unexpected hook reply"); } } actLock.reset(); - state = &DerivationGoal::tryLocalBuild; - return tryLocalBuild(inBuildSlot); + co_return co_await tryLocalBuild(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::tryLocalBuild(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::tryLocalBuild() noexcept try { throw Error( "unable to build with a primary store that isn't a local store; " @@ -857,7 +863,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) // attempt to recover movePath(oldPath, storePath); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } throw; } @@ -973,10 +979,11 @@ void runPostBuildHook( proc.getStdout()->drainInto(sink); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::buildDone(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::buildDone() noexcept try { trace("build done"); + slotToken = {}; Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); cleanupPreChildKill(); @@ -992,9 +999,6 @@ try { buildResult.timesBuilt++; buildResult.stopTime = time(0); - /* So the child is gone now. */ - worker.childTerminated(this); - /* Close the read side of the logger pipe. */ closeReadPipes(); @@ -1095,7 +1099,7 @@ try { return {std::current_exception()}; } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::resolvedFinished(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::resolvedFinished() noexcept try { trace("resolved derivation finished"); @@ -1168,7 +1172,7 @@ try { return {std::current_exception()}; } -HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) +HookReply DerivationGoal::tryBuildHook() { if (!worker.hook.available || !useDerivation) return HookReply::Decline{}; @@ -1180,7 +1184,7 @@ HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) /* Send the request to the hook. */ worker.hook.instance->sink << "try" - << (inBuildSlot ? 1 : 0) + << (slotToken.valid() ? 1 : 0) << drv->platform << worker.store.printStorePath(drvPath) << parsedDrv->getRequiredSystemFeatures(); @@ -1207,6 +1211,7 @@ HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) else { s += "\n"; writeLogsToStderr(s); + logger->log(lvlInfo, s); } } @@ -1266,12 +1271,8 @@ HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) /* Create the log file and pipe. */ Path logFile = openLogFile(); - std::set<int> fds; - fds.insert(hook->fromHook.get()); - fds.insert(hook->builderOut.get()); builderOutFD = &hook->builderOut; - - return HookReply::Accept{std::move(fds)}; + return HookReply::Accept{handleChildOutput()}; } @@ -1331,23 +1332,69 @@ void DerivationGoal::closeLogFile() } -Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data) +Goal::WorkResult DerivationGoal::tooMuchLogs() { - assert(builderOutFD); + killChild(); + return done( + BuildResult::LogLimitExceeded, {}, + Error("%s killed after writing more than %d bytes of log output", + getName(), settings.maxLogSize)); +} - auto tooMuchLogs = [&] { - killChild(); - return done( - BuildResult::LogLimitExceeded, {}, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - }; +struct DerivationGoal::InputStream final : private kj::AsyncObject +{ + int fd; + kj::UnixEventPort::FdObserver observer; + + InputStream(kj::UnixEventPort & ep, int fd) + : fd(fd) + , observer(ep, fd, kj::UnixEventPort::FdObserver::OBSERVE_READ) + { + int flags = fcntl(fd, F_GETFL); + if (flags < 0) { + throw SysError("fcntl(F_GETFL) failed on fd %i", fd); + } + if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { + throw SysError("fcntl(F_SETFL) failed on fd %i", fd); + } + } + + kj::Promise<std::string_view> read(kj::ArrayPtr<char> buffer) + { + const auto res = ::read(fd, buffer.begin(), buffer.size()); + // closing a pty endpoint causes EIO on the other endpoint. stock kj streams + // do not handle this and throw exceptions we can't ask for errno instead :( + // (we can't use `errno` either because kj may well have mangled it by now.) + if (res == 0 || (res == -1 && errno == EIO)) { + return std::string_view{}; + } + + KJ_NONBLOCKING_SYSCALL(res) {} + + if (res > 0) { + return std::string_view{buffer.begin(), static_cast<size_t>(res)}; + } + + return observer.whenBecomesReadable().then([this, buffer] { + return read(buffer); + }); + } +}; + +kj::Promise<Outcome<void, Goal::WorkResult>> DerivationGoal::handleBuilderOutput(InputStream & in) noexcept +try { + auto buf = kj::heapArray<char>(4096); + while (true) { + auto data = co_await in.read(buf); + lastChildActivity = worker.aio.provider->getTimer().now(); + + if (data.empty()) { + co_return result::success(); + } - // local & `ssh://`-builds are dealt with here. - if (fd == builderOutFD->get()) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - return tooMuchLogs(); + co_return tooMuchLogs(); } for (auto c : data) @@ -1362,10 +1409,22 @@ Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data } if (logSink) (*logSink)(data); - return StillAlive{}; } +} catch (...) { + co_return std::current_exception(); +} + +kj::Promise<Outcome<void, Goal::WorkResult>> DerivationGoal::handleHookOutput(InputStream & in) noexcept +try { + auto buf = kj::heapArray<char>(4096); + while (true) { + auto data = co_await in.read(buf); + lastChildActivity = worker.aio.provider->getTimer().now(); + + if (data.empty()) { + co_return result::success(); + } - if (hook && fd == hook->fromHook.get()) { for (auto c : data) if (c == '\n') { auto json = parseJSONMessage(currentHookLine); @@ -1381,7 +1440,7 @@ Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data (fields.size() > 0 ? fields[0].get<std::string>() : "") + "\n"; logSize += logLine.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - return tooMuchLogs(); + co_return tooMuchLogs(); } (*logSink)(logLine); } else if (type == resSetPhase && ! fields.is_null()) { @@ -1405,16 +1464,83 @@ Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data } else currentHookLine += c; } - - return StillAlive{}; +} catch (...) { + co_return std::current_exception(); } +kj::Promise<Outcome<void, Goal::WorkResult>> DerivationGoal::handleChildOutput() noexcept +try { + assert(builderOutFD); + + auto builderIn = kj::heap<InputStream>(worker.aio.unixEventPort, builderOutFD->get()); + kj::Own<InputStream> hookIn; + if (hook) { + hookIn = kj::heap<InputStream>(worker.aio.unixEventPort, hook->fromHook.get()); + } -void DerivationGoal::handleEOF(int fd) + auto handlers = handleChildStreams(*builderIn, hookIn.get()).attach(std::move(builderIn), std::move(hookIn)); + + if (respectsTimeouts() && settings.buildTimeout != 0) { + handlers = handlers.exclusiveJoin( + worker.aio.provider->getTimer() + .afterDelay(settings.buildTimeout.get() * kj::SECONDS) + .then([this]() -> Outcome<void, WorkResult> { + return timedOut( + Error("%1% timed out after %2% seconds", name, settings.buildTimeout) + ); + }) + ); + } + + return handlers.then([this](auto r) -> Outcome<void, WorkResult> { + if (!currentLogLine.empty()) flushLine(); + return r; + }); +} catch (...) { + return {std::current_exception()}; +} + +kj::Promise<Outcome<void, Goal::WorkResult>> DerivationGoal::monitorForSilence() noexcept { - if (!currentLogLine.empty()) flushLine(); + while (true) { + const auto stash = lastChildActivity; + auto waitUntil = lastChildActivity + settings.maxSilentTime.get() * kj::SECONDS; + co_await worker.aio.provider->getTimer().atTime(waitUntil); + if (lastChildActivity == stash) { + co_return timedOut( + Error("%1% timed out after %2% seconds of silence", name, settings.maxSilentTime) + ); + } + } } +kj::Promise<Outcome<void, Goal::WorkResult>> +DerivationGoal::handleChildStreams(InputStream & builderIn, InputStream * hookIn) noexcept +{ + lastChildActivity = worker.aio.provider->getTimer().now(); + + auto handlers = kj::joinPromisesFailFast([&] { + kj::Vector<kj::Promise<Outcome<void, WorkResult>>> parts{2}; + + parts.add(handleBuilderOutput(builderIn)); + if (hookIn) { + parts.add(handleHookOutput(*hookIn)); + } + + return parts.releaseAsArray(); + }()); + + if (respectsTimeouts() && settings.maxSilentTime != 0) { + handlers = handlers.exclusiveJoin(monitorForSilence().then([](auto r) { + return kj::arr(std::move(r)); + })); + } + + for (auto r : co_await handlers) { + BOOST_OUTCOME_CO_TRYV(r); + } + co_return result::success(); +} void DerivationGoal::flushLine() { @@ -1555,11 +1681,13 @@ SingleDrvOutputs DerivationGoal::assertPathValidity() } -Goal::Finished DerivationGoal::done( +Goal::WorkResult DerivationGoal::done( BuildResult::Status status, SingleDrvOutputs builtOutputs, std::optional<Error> ex) { + isDone = true; + outputLocks.unlock(); buildResult.status = status; if (ex) @@ -1590,7 +1718,7 @@ Goal::Finished DerivationGoal::done( logError(ex->info()); } - return Finished{ + return WorkResult{ .exitCode = buildResult.success() ? ecSuccess : ecFailed, .result = buildResult, .ex = ex ? std::make_shared<Error>(std::move(*ex)) : nullptr, @@ -1629,5 +1757,4 @@ void DerivationGoal::waiteeDone(GoalPtr waitee) } } } - } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 020388d5a..6dd58afd2 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -8,6 +8,7 @@ #include "store-api.hh" #include "pathlocks.hh" #include "goal.hh" +#include <kj/time.h> namespace nix { @@ -17,7 +18,7 @@ struct HookInstance; struct HookReplyBase { struct [[nodiscard]] Accept { - std::set<int> fds; + kj::Promise<Outcome<void, Goal::WorkResult>> promise; }; struct [[nodiscard]] Decline {}; struct [[nodiscard]] Postpone {}; @@ -62,7 +63,7 @@ struct InitialOutputStatus { struct InitialOutput { bool wanted; Hash outputHash; - std::optional<InitialOutputStatus> known; + std::optional<InitialOutputStatus> known = {}; }; /** @@ -70,6 +71,14 @@ struct InitialOutput { */ struct DerivationGoal : public Goal { + struct InputStream; + + /** + * Whether this goal has completed. Completed goals can not be + * asked for more outputs, a new goal must be created instead. + */ + bool isDone = false; + /** * Whether to use an on-disk .drv file. */ @@ -176,6 +185,11 @@ struct DerivationGoal : public Goal std::map<std::string, InitialOutput> initialOutputs; /** + * Build result. + */ + BuildResult buildResult; + + /** * File descriptor for the log file. */ AutoCloseFD fdLogFile; @@ -213,9 +227,6 @@ struct DerivationGoal : public Goal */ std::optional<DerivationType> derivationType; - typedef kj::Promise<Result<WorkResult>> (DerivationGoal::*GoalState)(bool inBuildSlot) noexcept; - GoalState state; - BuildMode buildMode; NotifyingCounter<uint64_t>::Bump mcExpectedBuilds, mcRunningBuilds; @@ -242,37 +253,35 @@ struct DerivationGoal : public Goal BuildMode buildMode = bmNormal); virtual ~DerivationGoal() noexcept(false); - Finished timedOut(Error && ex) override; - - std::string key() override; + WorkResult timedOut(Error && ex); - kj::Promise<Result<WorkResult>> work(bool inBuildSlot) noexcept override; + kj::Promise<Result<WorkResult>> workImpl() noexcept override; /** * Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const OutputsSpec & outputs); + bool addWantedOutputs(const OutputsSpec & outputs); /** * The states. */ - kj::Promise<Result<WorkResult>> getDerivation(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> loadDerivation(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> haveDerivation(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> outputsSubstitutionTried(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> gaveUpOnSubstitution(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> closureRepaired(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> inputsRealised(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> tryToBuild(bool inBuildSlot) noexcept; - virtual kj::Promise<Result<WorkResult>> tryLocalBuild(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> buildDone(bool inBuildSlot) noexcept; + kj::Promise<Result<WorkResult>> getDerivation() noexcept; + kj::Promise<Result<WorkResult>> loadDerivation() noexcept; + kj::Promise<Result<WorkResult>> haveDerivation() noexcept; + kj::Promise<Result<WorkResult>> outputsSubstitutionTried() noexcept; + kj::Promise<Result<WorkResult>> gaveUpOnSubstitution() noexcept; + kj::Promise<Result<WorkResult>> closureRepaired() noexcept; + kj::Promise<Result<WorkResult>> inputsRealised() noexcept; + kj::Promise<Result<WorkResult>> tryToBuild() noexcept; + virtual kj::Promise<Result<WorkResult>> tryLocalBuild() noexcept; + kj::Promise<Result<WorkResult>> buildDone() noexcept; - kj::Promise<Result<WorkResult>> resolvedFinished(bool inBuildSlot) noexcept; + kj::Promise<Result<WorkResult>> resolvedFinished() noexcept; /** * Is the build hook willing to perform the build? */ - HookReply tryBuildHook(bool inBuildSlot); + HookReply tryBuildHook(); virtual int getChildStatus(); @@ -312,13 +321,19 @@ struct DerivationGoal : public Goal virtual void cleanupPostOutputsRegisteredModeCheck(); virtual void cleanupPostOutputsRegisteredModeNonCheck(); - /** - * Callback used by the worker to write to the log. - */ - WorkResult handleChildOutput(int fd, std::string_view data) override; - void handleEOF(int fd) override; +protected: + kj::TimePoint lastChildActivity = kj::minValue; + + kj::Promise<Outcome<void, WorkResult>> handleChildOutput() noexcept; + kj::Promise<Outcome<void, WorkResult>> + handleChildStreams(InputStream & builderIn, InputStream * hookIn) noexcept; + kj::Promise<Outcome<void, WorkResult>> handleBuilderOutput(InputStream & in) noexcept; + kj::Promise<Outcome<void, WorkResult>> handleHookOutput(InputStream & in) noexcept; + kj::Promise<Outcome<void, WorkResult>> monitorForSilence() noexcept; + WorkResult tooMuchLogs(); void flushLine(); +public: /** * Wrappers around the corresponding Store methods that first consult the * derivation. This is currently needed because when there is no drv file @@ -350,13 +365,18 @@ struct DerivationGoal : public Goal void started(); - Finished done( + WorkResult done( BuildResult::Status status, SingleDrvOutputs builtOutputs = {}, std::optional<Error> ex = {}); void waiteeDone(GoalPtr waitee) override; + virtual bool respectsTimeouts() + { + return false; + } + StorePathSet exportReferences(const StorePathSet & storePaths); JobCategory jobCategory() const override { diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 7986123cc..f04beb884 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -4,6 +4,9 @@ #include "worker.hh" #include "substitution-goal.hh" #include "signals.hh" +#include <kj/array.h> +#include <kj/async.h> +#include <kj/vector.h> namespace nix { @@ -16,33 +19,32 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( : Goal(worker, isDependency) , id(id) { - state = &DrvOutputSubstitutionGoal::init; name = fmt("substitution of '%s'", id.to_string()); trace("created"); } -kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::init(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::workImpl() noexcept try { trace("init"); /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - return {Finished{ecSuccess, std::move(buildResult)}}; + co_return WorkResult{ecSuccess}; } subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>(); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::tryNext(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::tryNext() noexcept try { trace("trying next substituter"); - if (!inBuildSlot) { - return {WaitForSlot{}}; + if (!slotToken.valid()) { + slotToken = co_await worker.substitutions.acquire(); } maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); @@ -59,7 +61,7 @@ try { /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - return {Finished{substituterFailed ? ecFailed : ecNoSubstituters, std::move(buildResult)}}; + co_return WorkResult{substituterFailed ? ecFailed : ecNoSubstituters}; } sub = subs.front(); @@ -69,25 +71,26 @@ try { some other error occurs), so it must not touch `this`. So put the shared state in a separate refcounted object. */ downloadState = std::make_shared<DownloadState>(); - downloadState->outPipe.create(); + auto pipe = kj::newPromiseAndCrossThreadFulfiller<void>(); + downloadState->outPipe = kj::mv(pipe.fulfiller); downloadState->result = std::async(std::launch::async, [downloadState{downloadState}, id{id}, sub{sub}] { + Finally updateStats([&]() { downloadState->outPipe->fulfill(); }); ReceiveInterrupts receiveInterrupts; - Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); }); return sub->queryRealisation(id); }); - state = &DrvOutputSubstitutionGoal::realisationFetched; - return {WaitForWorld{{downloadState->outPipe.readSide.get()}, true}}; + co_await pipe.promise; + co_return co_await realisationFetched(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::realisationFetched(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::realisationFetched() noexcept try { - worker.childTerminated(this); maintainRunningSubstitutions.reset(); + slotToken = {}; try { outputInfo = downloadState->result.get(); @@ -97,10 +100,10 @@ try { } if (!outputInfo) { - return tryNext(inBuildSlot); + co_return co_await tryNext(); } - WaitForGoals result; + kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies; for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { if (depId != id) { if (auto localOutputInfo = worker.store.queryRealisation(depId); @@ -114,34 +117,31 @@ try { worker.store.printStorePath(localOutputInfo->outPath), worker.store.printStorePath(depPath) ); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } - result.goals.insert(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); + dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); } } - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); - if (result.goals.empty()) { - return outPathValid(inBuildSlot); - } else { - state = &DrvOutputSubstitutionGoal::outPathValid; - return {std::move(result)}; + if (!dependencies.empty()) { + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await outPathValid(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::outPathValid(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::outPathValid() noexcept try { assert(outputInfo); trace("output path substituted"); if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - return {Finished{ + return {WorkResult{ nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, - std::move(buildResult), }}; } @@ -154,22 +154,9 @@ try { kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::finished() noexcept try { trace("finished"); - return {Finished{ecSuccess, std::move(buildResult)}}; + return {WorkResult{ecSuccess}}; } catch (...) { return {std::current_exception()}; } -std::string DrvOutputSubstitutionGoal::key() -{ - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(id.to_string()); -} - -kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::work(bool inBuildSlot) noexcept -{ - return (this->*state)(inBuildSlot); -} - - } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index f33196665..f959e2a7b 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -45,7 +45,7 @@ class DrvOutputSubstitutionGoal : public Goal { struct DownloadState { - Pipe outPipe; + kj::Own<kj::CrossThreadPromiseFulfiller<void>> outPipe; std::future<std::shared_ptr<const Realisation>> result; }; @@ -65,20 +65,12 @@ public: std::optional<ContentAddress> ca = std::nullopt ); - typedef kj::Promise<Result<WorkResult>> (DrvOutputSubstitutionGoal::*GoalState)(bool inBuildSlot) noexcept; - GoalState state; - - kj::Promise<Result<WorkResult>> init(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> tryNext(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> realisationFetched(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> outPathValid(bool inBuildSlot) noexcept; + kj::Promise<Result<WorkResult>> tryNext() noexcept; + kj::Promise<Result<WorkResult>> realisationFetched() noexcept; + kj::Promise<Result<WorkResult>> outPathValid() noexcept; kj::Promise<Result<WorkResult>> finished() noexcept; - Finished timedOut(Error && ex) override { abort(); }; - - std::string key() override; - - kj::Promise<Result<WorkResult>> work(bool inBuildSlot) noexcept override; + kj::Promise<Result<WorkResult>> workImpl() noexcept override; JobCategory jobCategory() const override { return JobCategory::Substitution; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index a0f18a02c..808179a4d 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -17,22 +17,22 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod Worker worker(*this, evalStore ? *evalStore : *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - Goals goals; + Worker::Targets goals; for (auto & br : reqs) - goals.insert(gf.makeGoal(br, buildMode)); + goals.emplace(gf.makeGoal(br, buildMode)); return goals; }); StringSet failed; std::shared_ptr<Error> ex; - for (auto & i : goals) { - if (i->ex) { + for (auto & [i, result] : goals) { + if (result.ex) { if (ex) - logError(i->ex->info()); + logError(result.ex->info()); else - ex = i->ex; + ex = result.ex; } - if (i->exitCode != Goal::ecSuccess) { + if (result.exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(printStorePath(i2->drvPath)); else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) @@ -60,11 +60,11 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults( std::vector<std::pair<const DerivedPath &, GoalPtr>> state; auto goals = runWorker(worker, [&](GoalFactory & gf) { - Goals goals; + Worker::Targets goals; for (const auto & req : reqs) { auto goal = gf.makeGoal(req, buildMode); - goals.insert(goal); - state.push_back({req, goal}); + state.push_back({req, goal.first}); + goals.emplace(std::move(goal)); } return goals; }); @@ -72,7 +72,7 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults( std::vector<KeyedBuildResult> results; for (auto & [req, goalPtr] : state) - results.emplace_back(goalPtr->buildResult.restrictTo(req)); + results.emplace_back(goals[goalPtr].result.restrictTo(req)); return results; } @@ -84,11 +84,13 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat Worker worker(*this, *this, aio); try { - auto goals = runWorker(worker, [&](GoalFactory & gf) -> Goals { - return Goals{gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)}; + auto goals = runWorker(worker, [&](GoalFactory & gf) { + Worker::Targets goals; + goals.emplace(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)); + return goals; }); - auto goal = *goals.begin(); - return goal->buildResult.restrictTo(DerivedPath::Built { + auto [goal, result] = *goals.begin(); + return result.result.restrictTo(DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); @@ -110,14 +112,16 @@ void Store::ensurePath(const StorePath & path) Worker worker(*this, *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - return Goals{gf.makePathSubstitutionGoal(path)}; + Worker::Targets goals; + goals.emplace(gf.makePathSubstitutionGoal(path)); + return goals; }); - auto goal = *goals.begin(); + auto [goal, result] = *goals.begin(); - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->withExitStatus(worker.failingExitStatus()); - throw std::move(*goal->ex); + if (result.exitCode != Goal::ecSuccess) { + if (result.ex) { + result.ex->withExitStatus(worker.failingExitStatus()); + throw std::move(*result.ex); } else throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } @@ -130,24 +134,28 @@ void Store::repairPath(const StorePath & path) Worker worker(*this, *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - return Goals{gf.makePathSubstitutionGoal(path, Repair)}; + Worker::Targets goals; + goals.emplace(gf.makePathSubstitutionGoal(path, Repair)); + return goals; }); - auto goal = *goals.begin(); + auto [goal, result] = *goals.begin(); - if (goal->exitCode != Goal::ecSuccess) { + if (result.exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { worker.run([&](GoalFactory & gf) { - return Goals{gf.makeGoal( + Worker::Targets goals; + goals.emplace(gf.makeGoal( DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*info->deriver), // FIXME: Should just build the specific output we need. .outputs = OutputsSpec::All{}, }, bmRepair - )}; + )); + return goals; }); } else throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 82861ad2b..02b22b8ad 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -1,18 +1,73 @@ #include "goal.hh" +#include "async-collect.hh" +#include "worker.hh" +#include <boost/outcome/try.hpp> +#include <kj/time.h> namespace nix { -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - std::string s1 = a->key(); - std::string s2 = b->key(); - return s1 < s2; -} - - void Goal::trace(std::string_view s) { debug("%1%: %2%", name, s); } +kj::Promise<void> Goal::waitForAWhile() +{ + trace("wait for a while"); + /* If we are polling goals that are waiting for a lock, then wake + up after a few seconds at most. */ + return worker.aio.provider->getTimer().afterDelay(settings.pollInterval.get() * kj::SECONDS); +} + +kj::Promise<Result<Goal::WorkResult>> Goal::work() noexcept +try { + BOOST_OUTCOME_CO_TRY(auto result, co_await workImpl()); + + trace("done"); + + cleanup(); + + co_return std::move(result); +} catch (...) { + co_return result::failure(std::current_exception()); +} + +kj::Promise<Result<void>> +Goal::waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies) noexcept +try { + auto left = dependencies.size(); + for (auto & [dep, p] : dependencies) { + p = p.then([this, dep, &left](auto _result) -> Result<WorkResult> { + BOOST_OUTCOME_TRY(auto result, _result); + + left--; + trace(fmt("waitee '%s' done; %d left", dep->name, left)); + + if (result.exitCode != Goal::ecSuccess) ++nrFailed; + if (result.exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters; + if (result.exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure; + + return std::move(result); + }).eagerlyEvaluate(nullptr); + } + + auto collectDeps = asyncCollect(std::move(dependencies)); + + while (auto item = co_await collectDeps.next()) { + auto & [dep, _result] = *item; + BOOST_OUTCOME_CO_TRY(auto result, _result); + + waiteeDone(dep); + + if (result.exitCode == ecFailed && !settings.keepGoing) { + co_return result::success(); + } + } + + co_return result::success(); +} catch (...) { + co_return result::failure(std::current_exception()); +} + } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 189505308..29540dcd3 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,10 +1,12 @@ #pragma once ///@file +#include "async-semaphore.hh" #include "result.hh" #include "types.hh" #include "store-api.hh" #include "build-result.hh" +#include <concepts> // IWYU pragma: keep #include <kj/async.h> namespace nix { @@ -19,22 +21,11 @@ class Worker; * A pointer to a goal. */ typedef std::shared_ptr<Goal> GoalPtr; -typedef std::weak_ptr<Goal> WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; /** * Set of goals. */ -typedef std::set<GoalPtr, CompareGoalPtrs> Goals; -typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals; - -/** - * A map of paths to goals (and the other way around). - */ -typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; +typedef std::set<GoalPtr> Goals; /** * Used as a hint to the worker on how to schedule a particular goal. For example, @@ -70,17 +61,6 @@ struct Goal const bool isDependency; /** - * Goals that this goal is waiting for. - */ - Goals waitees; - - /** - * Goals waiting for this one to finish. Must use weak pointers - * here to prevent cycles. - */ - WeakGoals waiters; - - /** * Number of goals we are/were waiting for that have failed. */ size_t nrFailed = 0; @@ -102,57 +82,37 @@ struct Goal */ std::string name; - /** - * Whether the goal is finished. - */ - std::optional<ExitCode> exitCode; - - /** - * Build result. - */ - BuildResult buildResult; +protected: + AsyncSemaphore::Token slotToken; public: - - struct [[nodiscard]] StillAlive {}; - struct [[nodiscard]] WaitForSlot {}; - struct [[nodiscard]] WaitForAWhile {}; - struct [[nodiscard]] ContinueImmediately {}; - struct [[nodiscard]] WaitForGoals { - Goals goals; - }; - struct [[nodiscard]] WaitForWorld { - std::set<int> fds; - bool inBuildSlot; - }; - struct [[nodiscard]] Finished { + struct [[nodiscard]] WorkResult { ExitCode exitCode; - BuildResult result; - std::shared_ptr<Error> ex; + BuildResult result = {}; + std::shared_ptr<Error> ex = {}; bool permanentFailure = false; bool timedOut = false; bool hashMismatch = false; bool checkMismatch = false; }; - struct [[nodiscard]] WorkResult : std::variant< - StillAlive, - WaitForSlot, - WaitForAWhile, - ContinueImmediately, - WaitForGoals, - WaitForWorld, - Finished> +protected: + kj::Promise<void> waitForAWhile(); + kj::Promise<Result<void>> + waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies) noexcept; + + template<std::derived_from<Goal>... G> + kj::Promise<Result<void>> + waitForGoals(std::pair<std::shared_ptr<G>, kj::Promise<Result<WorkResult>>>... goals) noexcept { - WorkResult() = delete; - using variant::variant; - }; + return waitForGoals( + kj::arrOf<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>>(std::move(goals)...) + ); + } - /** - * Exception containing an error message, if any. - */ - std::shared_ptr<Error> ex; + virtual kj::Promise<Result<WorkResult>> workImpl() noexcept = 0; +public: explicit Goal(Worker & worker, bool isDependency) : worker(worker) , isDependency(isDependency) @@ -163,24 +123,10 @@ public: trace("goal destroyed"); } - virtual kj::Promise<Result<WorkResult>> work(bool inBuildSlot) noexcept = 0; + kj::Promise<Result<WorkResult>> work() noexcept; virtual void waiteeDone(GoalPtr waitee) { } - virtual WorkResult handleChildOutput(int fd, std::string_view data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - } - - virtual bool respectsTimeouts() - { - return false; - } - void trace(std::string_view s); std::string getName() const @@ -188,15 +134,6 @@ public: return name; } - /** - * Callback in case of a timeout. It should wake up its waiters, - * get rid of any running child processes that are being monitored - * by the worker (important!), etc. - */ - virtual Finished timedOut(Error && ex) = 0; - - virtual std::string key() = 0; - virtual void cleanup() { } /** diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index f91a904cc..521f34917 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,4 +1,5 @@ #include "child.hh" +#include "error.hh" #include "file-system.hh" #include "globals.hh" #include "hook-instance.hh" @@ -86,7 +87,7 @@ HookInstance::~HookInstance() toHook.reset(); if (pid) pid.kill(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4baa525d9..c8c68f99f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,4 +1,5 @@ #include "local-derivation-goal.hh" +#include "error.hh" #include "indirect-root-store.hh" #include "machines.hh" #include "store-api.hh" @@ -98,9 +99,9 @@ LocalDerivationGoal::~LocalDerivationGoal() noexcept(false) { /* Careful: we should never ever throw an exception from a destructor. */ - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } + try { deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } + try { stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -121,8 +122,6 @@ LocalStore & LocalDerivationGoal::getLocalStore() void LocalDerivationGoal::killChild() { if (pid) { - worker.childTerminated(this); - /* If we're using a build user, then there is a tricky race condition: if we kill the build user before the child has done its setuid() to the build user uid, then it won't be @@ -149,17 +148,18 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -kj::Promise<Result<Goal::WorkResult>> LocalDerivationGoal::tryLocalBuild(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> LocalDerivationGoal::tryLocalBuild() noexcept try { +retry: #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif - if (!inBuildSlot) { - state = &DerivationGoal::tryToBuild; + if (!slotToken.valid()) { outputLocks.unlock(); - if (0U != settings.maxBuildJobs) { - return {WaitForSlot{}}; + if (worker.localBuilds.capacity() > 0) { + slotToken = co_await worker.localBuilds.acquire(); + co_return co_await tryToBuild(); } if (getMachines().empty()) { throw Error( @@ -214,7 +214,9 @@ try { if (!actLock) actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - return {WaitForAWhile{}}; + co_await waitForAWhile(); + // we can loop very often, and `co_return co_await` always allocates a new frame + goto retry; } } @@ -243,24 +245,29 @@ try { try { /* Okay, we have to build. */ - auto fds = startBuilder(); - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; + auto promise = startBuilder(); started(); - return {WaitForWorld{std::move(fds), true}}; + auto r = co_await promise; + if (r.has_value()) { + // all good so far + } else if (r.has_error()) { + co_return r.assume_error(); + } else { + co_return r.assume_exception(); + } } catch (BuildError & e) { outputLocks.unlock(); buildUser.reset(); auto report = done(BuildResult::InputRejected, {}, std::move(e)); report.permanentFailure = true; - return {std::move(report)}; + co_return report; } + + co_return co_await buildDone(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } @@ -390,7 +397,9 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() cleanupPostOutputsRegisteredModeCheck(); } -std::set<int> LocalDerivationGoal::startBuilder() +// NOTE this one isn't noexcept because it's called from places that expect +// exceptions to signal failure to launch. we should change this some time. +kj::Promise<Outcome<void, Goal::WorkResult>> LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) #if __linux__ @@ -779,7 +788,7 @@ std::set<int> LocalDerivationGoal::startBuilder() msgs.push_back(std::move(msg)); } - return {builderOutPTY.get()}; + return handleChildOutput(); } @@ -1241,7 +1250,7 @@ void LocalDerivationGoal::startDaemon() NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } }); @@ -1361,13 +1370,20 @@ void LocalDerivationGoal::runChild() bool setUser = true; - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ + /* Make the contents of netrc and the CA certificate bundle + available to builtin:fetchurl (which may run under a + different uid and/or in a sandbox). */ std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl" && !derivationType->isSandboxed()) + std::string caFileData; + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl" && !derivationType->isSandboxed()) { + try { netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } + } catch (SysError &) { } + + try { + caFileData = readFile(settings.caFile); + } catch (SysError &) { } + } #if __linux__ if (useChroot) { @@ -1802,7 +1818,7 @@ void LocalDerivationGoal::runChild() e.second = rewriteStrings(e.second, inputRewrites); if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); + builtinFetchurl(drv2, netrcData, caFileData); else if (drv->builder == "builtin:buildenv") builtinBuildenv(drv2); else if (drv->builder == "builtin:unpack-channel") diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index cd040bc15..cd6ea2b55 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -182,7 +182,7 @@ struct LocalDerivationGoal : public DerivationGoal * Create a LocalDerivationGoal without an on-disk .drv file, * possibly a platform-specific subclass */ - static std::shared_ptr<LocalDerivationGoal> makeLocalDerivationGoal( + static std::unique_ptr<LocalDerivationGoal> makeLocalDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, @@ -194,7 +194,7 @@ struct LocalDerivationGoal : public DerivationGoal * Create a LocalDerivationGoal for an on-disk .drv file, * possibly a platform-specific subclass */ - static std::shared_ptr<LocalDerivationGoal> makeLocalDerivationGoal( + static std::unique_ptr<LocalDerivationGoal> makeLocalDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -213,12 +213,12 @@ struct LocalDerivationGoal : public DerivationGoal /** * The additional states. */ - kj::Promise<Result<WorkResult>> tryLocalBuild(bool inBuildSlot) noexcept override; + kj::Promise<Result<WorkResult>> tryLocalBuild() noexcept override; /** * Start building a derivation. */ - std::set<int> startBuilder(); + kj::Promise<Outcome<void, WorkResult>> startBuilder(); /** * Fill in the environment for the builder. diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index bd0ffcb9b..e0ca23a86 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -3,6 +3,8 @@ #include "nar-info.hh" #include "signals.hh" #include "finally.hh" +#include <kj/array.h> +#include <kj/vector.h> namespace nix { @@ -18,7 +20,6 @@ PathSubstitutionGoal::PathSubstitutionGoal( , repair(repair) , ca(ca) { - state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = worker.expectedSubstitutions.addTemporarily(1); @@ -31,27 +32,21 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } -Goal::Finished PathSubstitutionGoal::done( +Goal::WorkResult PathSubstitutionGoal::done( ExitCode result, BuildResult::Status status, std::optional<std::string> errorMsg) { - buildResult.status = status; + BuildResult buildResult{.status = status}; if (errorMsg) { debug(*errorMsg); buildResult.errorMsg = *errorMsg; } - return Finished{result, std::move(buildResult)}; + return WorkResult{result, std::move(buildResult)}; } -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::work(bool inBuildSlot) noexcept -{ - return (this->*state)(inBuildSlot); -} - - -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::init(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::workImpl() noexcept try { trace("init"); @@ -67,13 +62,13 @@ try { subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>(); - return tryNext(inBuildSlot); + return tryNext(); } catch (...) { return {std::current_exception()}; } -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::tryNext(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::tryNext() noexcept try { trace("trying next substituter"); @@ -89,10 +84,10 @@ try { /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - return {done( + co_return done( substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters, - fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)))}; + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); } sub = subs.front(); @@ -105,26 +100,28 @@ try { if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { - return tryNext(inBuildSlot); + co_return co_await tryNext(); } - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - return tryNext(inBuildSlot); - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - return tryNext(inBuildSlot); - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - return tryNext(inBuildSlot); + do { + try { + // FIXME: make async + info = sub->queryPathInfo(subPath ? *subPath : storePath); + break; + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + if (!settings.tryFallback) { + throw; + } + } catch (Error & e) { + if (settings.tryFallback) { + logError(e.info()); + } else { + throw; + } } - throw; - } + co_return co_await tryNext(); + } while (false); if (info->path != storePath) { if (info->isContentAddressed(*sub) && info->references.empty()) { @@ -134,7 +131,7 @@ try { } else { printError("asked '%s' for '%s' but got '%s'", sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } } @@ -155,28 +152,26 @@ try { { warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - WaitForGoals result; + kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies; for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); - if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ - return referencesValid(inBuildSlot); - } else { - state = &PathSubstitutionGoal::referencesValid; - return {std::move(result)}; + if (!dependencies.empty()) {/* to prevent hang (no wake-up event) */ + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await referencesValid(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::referencesValid(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::referencesValid() noexcept try { trace("all references realised"); @@ -191,33 +186,33 @@ try { if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); - state = &PathSubstitutionGoal::tryToRun; - return tryToRun(inBuildSlot); + return tryToRun(); } catch (...) { return {std::current_exception()}; } -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::tryToRun(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::tryToRun() noexcept try { trace("trying to run"); - if (!inBuildSlot) { - return {WaitForSlot{}}; + if (!slotToken.valid()) { + slotToken = co_await worker.substitutions.acquire(); } maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); - outPipe.create(); + auto pipe = kj::newPromiseAndCrossThreadFulfiller<void>(); + outPipe = kj::mv(pipe.fulfiller); thr = std::async(std::launch::async, [this]() { + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe->fulfill(); }); + auto & fetchPath = subPath ? *subPath : storePath; try { ReceiveInterrupts receiveInterrupts; - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide.close(); }); - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); @@ -233,39 +228,39 @@ try { } }); - state = &PathSubstitutionGoal::finished; - return {WaitForWorld{{outPipe.readSide.get()}, true}}; + co_await pipe.promise; + co_return co_await finished(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::finished(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::finished() noexcept try { trace("substitute finished"); - worker.childTerminated(this); - - try { - thr.get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ + do { try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; + slotToken = {}; + thr.get(); + break; + } catch (std::exception & e) { + printError(e.what()); + + /* Cause the parent build to fail unless --fallback is given, + or the substitute has disappeared. The latter case behaves + the same as the substitute never having existed in the + first place. */ + try { + throw; + } catch (SubstituteGone &) { + } catch (...) { + substituterFailed = true; + } } - /* Try the next substitute. */ - state = &PathSubstitutionGoal::tryNext; - return tryNext(inBuildSlot); - } + co_return co_await tryNext(); + } while (false); worker.markContentsGood(storePath); @@ -282,15 +277,9 @@ try { worker.doneNarSize += maintainExpectedNar.delta(); maintainExpectedNar.reset(); - return {done(ecSuccess, BuildResult::Substituted)}; + co_return done(ecSuccess, BuildResult::Substituted); } catch (...) { - return {std::current_exception()}; -} - - -Goal::WorkResult PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data) -{ - return StillAlive{}; + co_return result::failure(std::current_exception()); } @@ -300,12 +289,9 @@ void PathSubstitutionGoal::cleanup() if (thr.valid()) { // FIXME: signal worker thread to quit. thr.get(); - worker.childTerminated(this); } - - outPipe.close(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 3c97b19fd..18b4262a4 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -46,7 +46,7 @@ struct PathSubstitutionGoal : public Goal /** * Pipe for the substituter's standard output. */ - Pipe outPipe; + kj::Own<kj::CrossThreadPromiseFulfiller<void>> outPipe; /** * The substituter thread. @@ -67,15 +67,12 @@ struct PathSubstitutionGoal : public Goal NotifyingCounter<uint64_t>::Bump maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef kj::Promise<Result<WorkResult>> (PathSubstitutionGoal::*GoalState)(bool inBuildSlot) noexcept; - GoalState state; - /** * Content address for recomputing store path */ std::optional<ContentAddress> ca; - Finished done( + WorkResult done( ExitCode result, BuildResult::Status status, std::optional<std::string> errorMsg = {}); @@ -90,32 +87,15 @@ public: ); ~PathSubstitutionGoal(); - Finished timedOut(Error && ex) override { abort(); }; - - /** - * We prepend "a$" to the key name to ensure substitution goals - * happen before derivation goals. - */ - std::string key() override - { - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - kj::Promise<Result<WorkResult>> work(bool inBuildSlot) noexcept override; + kj::Promise<Result<WorkResult>> workImpl() noexcept override; /** * The states. */ - kj::Promise<Result<WorkResult>> init(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> tryNext(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> referencesValid(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> tryToRun(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> finished(bool inBuildSlot) noexcept; - - /** - * Callback used by the worker to write to the log. - */ - WorkResult handleChildOutput(int fd, std::string_view data) override; + kj::Promise<Result<WorkResult>> tryNext() noexcept; + kj::Promise<Result<WorkResult>> referencesValid() noexcept; + kj::Promise<Result<WorkResult>> tryToRun() noexcept; + kj::Promise<Result<WorkResult>> finished() noexcept; /* Called by destructor, can't be overridden */ void cleanup() override final; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index ee45c7e3f..10f58f5d3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,3 +1,4 @@ +#include "async-collect.hh" #include "charptr-cast.hh" #include "worker.hh" #include "finally.hh" @@ -6,11 +7,22 @@ #include "local-derivation-goal.hh" #include "signals.hh" #include "hook-instance.hh" // IWYU pragma: keep - -#include <poll.h> +#include <boost/outcome/try.hpp> +#include <kj/vector.h> namespace nix { +namespace { +struct ErrorHandler : kj::TaskSet::ErrorHandler +{ + void taskFailed(kj::Exception && e) override + { + printError("unexpected async failure in Worker: %s", kj::str(e).cStr()); + abort(); + } +} errorHandler; +} + Worker::Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) @@ -18,11 +30,13 @@ Worker::Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio) , store(store) , evalStore(evalStore) , aio(aio) + /* Make sure that we are always allowed to run at least one substitution. + This prevents infinite waiting. */ + , substitutions(std::max<unsigned>(1, settings.maxSubstitutionJobs)) + , localBuilds(settings.maxBuildJobs) + , children(errorHandler) { /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - nrSubstitutions = 0; - lastWokenUp = steady_time_point::min(); } @@ -32,7 +46,11 @@ Worker::~Worker() goals that refer to this worker should be gone. (Otherwise we are in trouble, since goals may call childTerminated() etc. in their destructors). */ - topGoals.clear(); + children.clear(); + + derivationGoals.clear(); + drvOutputSubstitutionGoals.clear(); + substitutionGoals.clear(); assert(expectedSubstitutions == 0); assert(expectedDownloadSize == 0); @@ -40,292 +58,158 @@ Worker::~Worker() } -std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const OutputsSpec & wantedOutputs, - std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal) +template<typename ID, std::derived_from<Goal> G> +std::pair<std::shared_ptr<G>, kj::Promise<Result<Goal::WorkResult>>> Worker::makeGoalCommon( + std::map<ID, CachedGoal<G>> & map, + const ID & key, + InvocableR<std::unique_ptr<G>> auto create, + InvocableR<bool, G &> auto modify +) { - std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath]; - std::shared_ptr<DerivationGoal> goal = goal_weak.lock(); - if (!goal) { - goal = mkDrvGoal(); - goal_weak = goal; - wakeUp(goal); - } else { - goal->addWantedOutputs(wantedOutputs); + auto [it, _inserted] = map.try_emplace(key); + // try twice to create the goal. we can only loop if we hit the continue, + // and then we only want to recreate the goal *once*. concurrent accesses + // to the worker are not sound, we want to catch them if at all possible. + for ([[maybe_unused]] auto _attempt : {1, 2}) { + auto & cachedGoal = it->second; + auto & goal = cachedGoal.goal; + if (!goal) { + goal = create(); + // do not start working immediately. if we are not yet running we + // may create dependencies as though they were toplevel goals, in + // which case the dependencies will not report build errors. when + // we are running we may be called for this same goal more times, + // and then we want to modify rather than recreate when possible. + auto removeWhenDone = [goal, &map, it] { + // c++ lambda coroutine capture semantics are *so* fucked up. + return [](auto goal, auto & map, auto it) -> kj::Promise<Result<Goal::WorkResult>> { + auto result = co_await goal->work(); + // a concurrent call to makeGoalCommon may have reset our + // cached goal and replaced it with a new instance. don't + // remove the goal in this case, otherwise we will crash. + if (goal == it->second.goal) { + map.erase(it); + } + co_return result; + }(goal, map, it); + }; + cachedGoal.promise = kj::evalLater(std::move(removeWhenDone)).fork(); + children.add(cachedGoal.promise.addBranch().then([this](auto _result) { + if (_result.has_value()) { + auto & result = _result.value(); + permanentFailure |= result.permanentFailure; + timedOut |= result.timedOut; + hashMismatch |= result.hashMismatch; + checkMismatch |= result.checkMismatch; + } + })); + } else { + if (!modify(*goal)) { + cachedGoal = {}; + continue; + } + } + return {goal, cachedGoal.promise.addBranch()}; } - return goal; + assert(false && "could not make a goal. possible concurrent worker access"); } -std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath, - const OutputsSpec & wantedOutputs, BuildMode buildMode) +std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> Worker::makeDerivationGoal( + const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode +) { - return makeDerivationGoalCommon( + return makeGoalCommon( + derivationGoals, drvPath, - wantedOutputs, - [&]() -> std::shared_ptr<DerivationGoal> { + [&]() -> std::unique_ptr<DerivationGoal> { return !dynamic_cast<LocalStore *>(&store) - ? std::make_shared<DerivationGoal>( + ? std::make_unique<DerivationGoal>( drvPath, wantedOutputs, *this, running, buildMode ) : LocalDerivationGoal::makeLocalDerivationGoal( drvPath, wantedOutputs, *this, running, buildMode ); - } + }, + [&](DerivationGoal & g) { return g.addWantedOutputs(wantedOutputs); } ); } -std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) +std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> Worker::makeBasicDerivationGoal( + const StorePath & drvPath, + const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, + BuildMode buildMode +) { - return makeDerivationGoalCommon( + return makeGoalCommon( + derivationGoals, drvPath, - wantedOutputs, - [&]() -> std::shared_ptr<DerivationGoal> { + [&]() -> std::unique_ptr<DerivationGoal> { return !dynamic_cast<LocalStore *>(&store) - ? std::make_shared<DerivationGoal>( + ? std::make_unique<DerivationGoal>( drvPath, drv, wantedOutputs, *this, running, buildMode ) : LocalDerivationGoal::makeLocalDerivationGoal( drvPath, drv, wantedOutputs, *this, running, buildMode ); - } + }, + [&](DerivationGoal & g) { return g.addWantedOutputs(wantedOutputs); } ); } -std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca) +std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> +Worker::makePathSubstitutionGoal( + const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca +) { - std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path]; - auto goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared<PathSubstitutionGoal>(path, *this, running, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; + return makeGoalCommon( + substitutionGoals, + path, + [&] { return std::make_unique<PathSubstitutionGoal>(path, *this, running, repair, ca); }, + [&](auto &) { return true; } + ); } -std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca) +std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> +Worker::makeDrvOutputSubstitutionGoal( + const DrvOutput & id, RepairFlag repair, std::optional<ContentAddress> ca +) { - std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id]; - auto goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, running, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; + return makeGoalCommon( + drvOutputSubstitutionGoals, + id, + [&] { return std::make_unique<DrvOutputSubstitutionGoal>(id, *this, running, repair, ca); }, + [&](auto &) { return true; } + ); } -GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) +std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) -> GoalPtr { + [&](const DerivedPath::Built & bfd) -> std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> { if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath)) return makeDerivationGoal(bop->path, bfd.outputs, buildMode); else throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, - [&](const DerivedPath::Opaque & bo) -> GoalPtr { + [&](const DerivedPath::Opaque & bo) -> std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); }, }, req.raw()); } +kj::Promise<Result<Worker::Results>> Worker::updateStatistics() +try { + while (true) { + statisticsUpdateInhibitor = co_await statisticsUpdateSignal.acquire(); -template<typename K, typename G> -static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap) -{ - /* !!! inefficient */ - for (auto i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - auto j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) -{ - goal->trace("done"); - assert(!goal->exitCode.has_value()); - goal->exitCode = f.exitCode; - goal->ex = f.ex; - - permanentFailure |= f.permanentFailure; - timedOut |= f.timedOut; - hashMismatch |= f.hashMismatch; - checkMismatch |= f.checkMismatch; - - for (auto & i : goal->waiters) { - if (GoalPtr waiting = i.lock()) { - assert(waiting->waitees.count(goal)); - waiting->waitees.erase(goal); - - waiting->trace(fmt("waitee '%s' done; %d left", goal->name, waiting->waitees.size())); - - if (f.exitCode != Goal::ecSuccess) ++waiting->nrFailed; - if (f.exitCode == Goal::ecNoSubstituters) ++waiting->nrNoSubstituters; - if (f.exitCode == Goal::ecIncompleteClosure) ++waiting->nrIncompleteClosure; - - if (waiting->waitees.empty() || (f.exitCode == Goal::ecFailed && !settings.keepGoing)) { - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & i : waiting->waitees) { - i->waiters.extract(waiting); - } - waiting->waitees.clear(); - - wakeUp(waiting); - } - - waiting->waiteeDone(goal); - } - } - goal->waiters.clear(); - removeGoal(goal); - goal->cleanup(); -} - -void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) -{ - std::visit( - overloaded{ - [&](Goal::StillAlive) {}, - [&](Goal::WaitForSlot) { waitForBuildSlot(goal); }, - [&](Goal::WaitForAWhile) { waitForAWhile(goal); }, - [&](Goal::ContinueImmediately) { wakeUp(goal); }, - [&](Goal::WaitForGoals & w) { - for (auto & dep : w.goals) { - goal->waitees.insert(dep); - dep->waiters.insert(goal); - } - }, - [&](Goal::WaitForWorld & w) { childStarted(goal, w.fds, w.inBuildSlot); }, - [&](Goal::Finished & f) { goalFinished(goal, f); }, - }, - how - ); -} - -void Worker::removeGoal(GoalPtr goal) -{ - if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal)) - nix::removeGoal(drvGoal, derivationGoals); - else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal)) - nix::removeGoal(subGoal, substitutionGoals); - else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal)) - nix::removeGoal(subGoal, drvOutputSubstitutionGoals); - else - assert(false); - - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - awake.insert(goal); -} - - -void Worker::childStarted(GoalPtr goal, const std::set<int> & fds, - bool inBuildSlot) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - children.emplace_back(child); - if (inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: - nrSubstitutions++; - break; - case JobCategory::Build: - nrLocalBuilds++; - break; - default: - abort(); - } - } -} - - -void Worker::childTerminated(Goal * goal) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: - assert(nrSubstitutions > 0); - nrSubstitutions--; - break; - case JobCategory::Build: - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - break; - default: - abort(); - } - } - - children.erase(i); - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - goal->trace("wait for build slot"); - bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution; - if ((!isSubstitutionGoal && nrLocalBuilds < settings.maxBuildJobs) || - (isSubstitutionGoal && nrSubstitutions < settings.maxSubstitutionJobs)) - wakeUp(goal); /* we can do it right away */ - else - wantingToBuild.insert(goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - waitingForAWhile.insert(goal); -} - - -void Worker::updateStatistics() -{ - // only update progress info while running. this notably excludes updating - // progress info while destroying, which causes the progress bar to assert - if (running && statisticsOutdated) { + // only update progress info while running. this notably excludes updating + // progress info while destroying, which causes the progress bar to assert actDerivations.progress( doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds ); @@ -338,221 +222,82 @@ void Worker::updateStatistics() act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - statisticsOutdated = false; + // limit to 50fps. that should be more than good enough for anything we do + co_await aio.provider->getTimer().afterDelay(20 * kj::MILLISECONDS); } +} catch (...) { + co_return result::failure(std::current_exception()); } -Goals Worker::run(std::function<Goals (GoalFactory &)> req) +Worker::Results Worker::run(std::function<Targets (GoalFactory &)> req) { - auto _topGoals = req(goalFactory()); + auto topGoals = req(goalFactory()); assert(!running); running = true; Finally const _stop([&] { running = false; }); - updateStatistics(); + auto onInterrupt = kj::newPromiseAndCrossThreadFulfiller<Result<Results>>(); + auto interruptCallback = createInterruptCallback([&] { + onInterrupt.fulfiller->fulfill(result::failure(std::make_exception_ptr(makeInterrupted()))); + }); - topGoals = _topGoals; + auto promise = runImpl(std::move(topGoals)) + .exclusiveJoin(updateStatistics()) + .exclusiveJoin(std::move(onInterrupt.promise)); - debug("entered goal loop"); + // TODO GC interface? + if (auto localStore = dynamic_cast<LocalStore *>(&store); localStore && settings.minFree != 0u) { + // Periodically wake up to see if we need to run the garbage collector. + promise = promise.exclusiveJoin(boopGC(*localStore)); + } - while (1) { + return promise.wait(aio.waitScope).value(); +} - checkInterrupt(); +kj::Promise<Result<Worker::Results>> Worker::runImpl(Targets topGoals) +try { + debug("entered goal loop"); - // TODO GC interface? - if (auto localStore = dynamic_cast<LocalStore *>(&store)) - localStore->autoGC(false); + kj::Vector<Targets::value_type> promises(topGoals.size()); + for (auto & gp : topGoals) { + promises.add(std::move(gp)); + } - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - /* Make sure that we are always allowed to run at least one substitution. - This prevents infinite waiting. */ - const bool inSlot = goal->jobCategory() == JobCategory::Substitution - ? nrSubstitutions < std::max(1U, (unsigned int) settings.maxSubstitutionJobs) - : nrLocalBuilds < settings.maxBuildJobs; - handleWorkResult(goal, goal->work(inSlot).wait(aio.waitScope).value()); - updateStatistics(); - - if (topGoals.empty()) break; // stuff may have been cancelled - } - } + Results results; - if (topGoals.empty()) break; + auto collect = AsyncCollect(promises.releaseAsArray()); + while (auto done = co_await collect.next()) { + // propagate goal exceptions outward + BOOST_OUTCOME_CO_TRY(auto result, done->second); + results.emplace(done->first, result); - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - assert(!awake.empty()); + /* If a top-level goal failed, then kill all other goals + (unless keepGoing was set). */ + if (result.exitCode == Goal::ecFailed && !settings.keepGoing) { + children.clear(); + break; } } /* If --keep-going is not set, it's possible that the main goal exited while some of its subgoals were still active. But if --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); + assert(!settings.keepGoing || children.isEmpty()); - return _topGoals; + co_return std::move(results); +} catch (...) { + co_return result::failure(std::current_exception()); } -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - long timeout = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (auto goal = i.goal.lock()) { - if (!goal->respectsTimeouts()) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast<std::chrono::seconds>( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector<struct pollfd> pollStatus; - std::map<int, size_t> fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; - } - } - - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - if (!goal->exitCode.has_value() && - 0 != settings.maxSilentTime && - goal->respectsTimeouts() && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - handleWorkResult( - goal, - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), - settings.maxSilentTime - )) - ); - continue; - } - - else if (!goal->exitCode.has_value() && - 0 != settings.buildTimeout && - goal->respectsTimeouts() && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - handleWorkResult( - goal, - goal->timedOut( - Error("%1% timed out after %2% seconds", goal->getName(), settings.buildTimeout) - ) - ); - continue; - } - - std::set<int> fds2(j->fds); - std::vector<unsigned char> buffer(4096); - for (auto & k : fds2) { - const auto fdPollStatusId = get(fdToPollStatus, k); - assert(fdPollStatusId); - assert(*fdPollStatusId < pollStatus.size()); - if (pollStatus.at(*fdPollStatusId).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - handleWorkResult(goal, Goal::ContinueImmediately{}); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - std::string_view data(charptr_cast<char *>(buffer.data()), rd); - j->lastOutput = after; - handleWorkResult(goal, goal->handleChildOutput(k, data)); - } - } - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); +kj::Promise<Result<Worker::Results>> Worker::boopGC(LocalStore & localStore) +try { + while (true) { + co_await aio.provider->getTimer().afterDelay(10 * kj::SECONDS); + localStore.autoGC(false); } +} catch (...) { + co_return result::failure(std::current_exception()); } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6735ea0b9..1a913ca16 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include "async-semaphore.hh" +#include "concepts.hh" #include "notifying-counter.hh" #include "types.hh" #include "lock.hh" @@ -18,37 +20,22 @@ namespace nix { struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; +class LocalStore; typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; -/** - * A mapping used to remember for each child process to what goal it - * belongs, and file descriptors for receiving log data and output - * path creation commands. - */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - std::set<int> fds; - bool inBuildSlot; - /** - * Time we last got output on stdout/stderr - */ - steady_time_point lastOutput; - steady_time_point timeStarted; -}; - /* Forward definition. */ struct HookInstance; class GoalFactory { public: - virtual std::shared_ptr<DerivationGoal> makeDerivationGoal( + virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> + makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal ) = 0; - virtual std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( + virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> + makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -58,12 +45,14 @@ public: /** * @ref SubstitutionGoal "substitution goal" */ - virtual std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal( + virtual std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> + makePathSubstitutionGoal( const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt ) = 0; - virtual std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal( + virtual std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> + makeDrvOutputSubstitutionGoal( const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt @@ -75,7 +64,8 @@ public: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - virtual GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0; + virtual std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> + makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0; }; // elaborate hoax to let goals access factory methods while hiding them from the public @@ -94,61 +84,27 @@ protected: */ class Worker : public WorkerBase { +public: + using Targets = std::map<GoalPtr, kj::Promise<Result<Goal::WorkResult>>>; + using Results = std::map<GoalPtr, Goal::WorkResult>; + private: bool running = false; - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /** - * The top-level goals of the worker. - */ - Goals topGoals; - - /** - * Goals that are ready to do some work. - */ - WeakGoals awake; - - /** - * Goals waiting for a build slot. - */ - WeakGoals wantingToBuild; - - /** - * Child processes currently running. - */ - std::list<Child> children; - - /** - * Number of build slots occupied. This includes local builds but does not - * include substitutions or remote builds via the build hook. - */ - unsigned int nrLocalBuilds; - - /** - * Number of substitution slots occupied. - */ - unsigned int nrSubstitutions; - + template<typename G> + struct CachedGoal + { + std::shared_ptr<G> goal; + kj::ForkedPromise<Result<Goal::WorkResult>> promise{nullptr}; + }; /** * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ - std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals; - std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals; - std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals; - - /** - * Goals sleeping for a few seconds (polling a lock). - */ - WeakGoals waitingForAWhile; - - /** - * Last time the goals in `waitingForAWhile` where woken up. - */ - steady_time_point lastWokenUp; + std::map<StorePath, CachedGoal<DerivationGoal>> derivationGoals; + std::map<StorePath, CachedGoal<PathSubstitutionGoal>> substitutionGoals; + std::map<DrvOutput, CachedGoal<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals; /** * Cache for pathContentsGood(). @@ -176,60 +132,25 @@ private: */ bool checkMismatch = false; - void goalFinished(GoalPtr goal, Goal::Finished & f); - void handleWorkResult(GoalPtr goal, Goal::WorkResult how); - - /** - * Put `goal` to sleep until a build slot becomes available (which - * might be right away). - */ - void waitForBuildSlot(GoalPtr goal); - - /** - * Wait for a few seconds and then retry this goal. Used when - * waiting for a lock held by another process. This kind of - * polling is inefficient, but POSIX doesn't really provide a way - * to wait for multiple locks in the main select() loop. - */ - void waitForAWhile(GoalPtr goal); - - /** - * Wake up a goal (i.e., there is something for it to do). - */ - void wakeUp(GoalPtr goal); - - /** - * Wait for input to become available. - */ - void waitForInput(); - - /** - * Remove a dead goal. - */ - void removeGoal(GoalPtr goal); - - /** - * Registers a running child process. `inBuildSlot` means that - * the process counts towards the jobs limit. - */ - void childStarted(GoalPtr goal, const std::set<int> & fds, - bool inBuildSlot); - /** * Pass current stats counters to the logger for progress bar updates. */ - void updateStatistics(); + kj::Promise<Result<Results>> updateStatistics(); - bool statisticsOutdated = true; + AsyncSemaphore statisticsUpdateSignal{1}; + std::optional<AsyncSemaphore::Token> statisticsUpdateInhibitor; /** * Mark statistics as outdated, such that `updateStatistics` will be called. */ void updateStatisticsLater() { - statisticsOutdated = true; + statisticsUpdateInhibitor = {}; } + kj::Promise<Result<Results>> runImpl(Targets topGoals); + kj::Promise<Result<Results>> boopGC(LocalStore & localStore); + public: const Activity act; @@ -239,7 +160,12 @@ public: Store & store; Store & evalStore; kj::AsyncIoContext & aio; + AsyncSemaphore substitutions, localBuilds; +private: + kj::TaskSet children; + +public: struct HookState { std::unique_ptr<HookInstance> instance; @@ -277,21 +203,35 @@ public: * @ref DerivationGoal "derivation goal" */ private: - std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( - const StorePath & drvPath, const OutputsSpec & wantedOutputs, - std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal); - std::shared_ptr<DerivationGoal> makeDerivationGoal( + template<typename ID, std::derived_from<Goal> G> + std::pair<std::shared_ptr<G>, kj::Promise<Result<Goal::WorkResult>>> makeGoalCommon( + std::map<ID, CachedGoal<G>> & map, + const ID & key, + InvocableR<std::unique_ptr<G>> auto create, + InvocableR<bool, G &> auto modify + ); + std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; - std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( + std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; /** * @ref SubstitutionGoal "substitution goal" */ - std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override; - std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override; + std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> + makePathSubstitutionGoal( + const StorePath & storePath, + RepairFlag repair = NoRepair, + std::optional<ContentAddress> ca = std::nullopt + ) override; + std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> + makeDrvOutputSubstitutionGoal( + const DrvOutput & id, + RepairFlag repair = NoRepair, + std::optional<ContentAddress> ca = std::nullopt + ) override; /** * Make a goal corresponding to the `DerivedPath`. @@ -299,18 +239,14 @@ private: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; + std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> + makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; public: /** - * Unregisters a running child process. - */ - void childTerminated(Goal * goal); - - /** * Loop until the specified top-level goals have finished. */ - Goals run(std::function<Goals (GoalFactory &)> req); + Results run(std::function<Targets (GoalFactory &)> req); /*** * The exit status in case of failure. diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index d201fb3ac..e20d14b90 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -6,7 +6,7 @@ namespace nix { // TODO: make pluggable. -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); +void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData, const std::string & caFileData); void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 3fb769fe6..b28eb01d0 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -7,7 +7,7 @@ namespace nix { -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) +void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData, const std::string & caFileData) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just @@ -17,6 +17,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) writeFile(settings.netrcFile, netrcData, 0600); } + settings.caFile = "ca-certificates.crt"; + writeFile(settings.caFile, caFileData, 0600); + auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); if (i == drv.env.end()) throw Error("attribute '%s' missing", name); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 10c810e49..34b92148e 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -6,6 +6,7 @@ #include "signals.hh" #include "compression.hh" #include "strings.hh" +#include <cstddef> #if ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -115,7 +116,7 @@ struct curlFileTransfer : public FileTransfer if (!done) fail(FileTransferError(Interrupted, {}, "download of '%s' was interrupted", request.uri)); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -784,8 +785,10 @@ struct curlFileTransfer : public FileTransfer size_t read(char * data, size_t len) override { - auto readPartial = [this](char * data, size_t len) { + auto readPartial = [this](char * data, size_t len) -> size_t { const auto available = std::min(len, buffered.size()); + if (available == 0u) return 0u; + memcpy(data, buffered.data(), available); buffered.remove_prefix(available); return available; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d5903d01e..99bf80994 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -923,8 +923,8 @@ void LocalStore::autoGC(bool sync) } catch (...) { // FIXME: we could propagate the exception to the - // future, but we don't really care. - ignoreException(); + // future, but we don't really care. (what??) + ignoreExceptionInDestructor(); } }).detach(); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ffc2543ef..f43b759d2 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -443,7 +443,7 @@ static bool initLibStoreDone = false; void assertLibStoreInitialized() { if (!initLibStoreDone) { printError("The program must call nix::initNix() before calling any libstore library functions."); - abort(); + std::terminate(); }; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1af0f54de..c3248c2c3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -481,7 +481,7 @@ LocalStore::~LocalStore() unlink(fnTempRoots.c_str()); } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -1222,7 +1222,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, try { parseDump(sink, source); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } }; diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index d0897b81f..7314e3177 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -68,11 +68,11 @@ ref<Store> Machine::openStore() const { Store::Params storeParams; if (storeUri.starts_with("ssh://")) { + storeParams["log-fd"] = "4"; storeParams["max-connections"] = "1"; } if (storeUri.starts_with("ssh://") || storeUri.starts_with("ssh-ng://")) { - storeParams["log-fd"] = "4"; if (sshKey != "") storeParams["ssh-key"] = sshKey; if (sshPublicHostKey != "") diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 7600de6e7..f228004a9 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -20,10 +20,10 @@ struct NarMember file in the NAR. */ uint64_t start = 0, size = 0; - std::string target; + std::string target = {}; /* If this is a directory, all the children of the directory. */ - std::map<std::string, NarMember> children; + std::map<std::string, NarMember> children = {}; }; struct NarAccessor : public FSAccessor diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index c60e5a85d..14381b6e0 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -31,7 +31,7 @@ struct MakeReadOnly /* This will make the path read-only. */ if (path != "") canonicaliseTimestampAndPermissions(path); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } }; diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 57e03252d..8e2da1908 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -17,7 +17,7 @@ namespace nix { struct StorePathWithOutputs { StorePath path; - std::set<std::string> outputs; + std::set<std::string> outputs = {}; std::string to_string(const Store & store) const; diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index ced0f30bb..3225857ec 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -145,7 +145,7 @@ PathLocks::~PathLocks() try { unlock(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index d06d031b5..feb0f5548 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "error.hh" #include "file-descriptor.hh" namespace nix { @@ -53,7 +54,7 @@ struct FdLock if (acquired) lockFile(fd, ltNone, false); } catch (SysError &) { - ignoreException(); + ignoreExceptionInDestructor(); } } }; diff --git a/src/libstore/platform.cc b/src/libstore/platform.cc index f2c023c82..36f8e352a 100644 --- a/src/libstore/platform.cc +++ b/src/libstore/platform.cc @@ -25,7 +25,7 @@ std::shared_ptr<LocalStore> LocalStore::makeLocalStore(const Params & params) #endif } -std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal( +std::unique_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, @@ -34,17 +34,17 @@ std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoa ) { #if __linux__ - return std::make_shared<LinuxLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique<LinuxLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); #elif __APPLE__ - return std::make_shared<DarwinLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique<DarwinLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); #elif __FreeBSD__ - return std::make_shared<FreeBSDLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique<FreeBSDLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); #else - return std::make_shared<FallbackLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique<FallbackLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); #endif } -std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal( +std::unique_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -54,19 +54,19 @@ std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoa ) { #if __linux__ - return std::make_shared<LinuxLocalDerivationGoal>( + return std::make_unique<LinuxLocalDerivationGoal>( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #elif __APPLE__ - return std::make_shared<DarwinLocalDerivationGoal>( + return std::make_unique<DarwinLocalDerivationGoal>( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #elif __FreeBSD__ - return std::make_shared<FreeBSDLocalDerivationGoal>( + return std::make_unique<FreeBSDLocalDerivationGoal>( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #else - return std::make_shared<FallbackLocalDerivationGoal>( + return std::make_unique<FallbackLocalDerivationGoal>( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #endif diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index f2b228fa0..baeb7a2c9 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -50,7 +50,7 @@ struct Realisation { DrvOutput id; StorePath outPath; - StringSet signatures; + StringSet signatures = {}; /** * The realisations that are required for the current one to be valid. @@ -58,7 +58,7 @@ struct Realisation { * When importing this realisation, the store will first check that all its * dependencies exist, and map to the correct output path */ - std::map<DrvOutput, StorePath> dependentRealisations; + std::map<DrvOutput, StorePath> dependentRealisations = {}; nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 0689ce74d..59d267873 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -29,7 +29,7 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str /* FIXME: do this asynchronously. */ writeFile(makeCacheFile(hashPart, "nar"), nar); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } @@ -41,7 +41,7 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str nlohmann::json j = listNar(narAccessor, "", true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1f94ca03f..a9f9818be 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,3 +1,4 @@ +#include "error.hh" #include "serialise.hh" #include "signals.hh" #include "path-with-outputs.hh" @@ -855,7 +856,7 @@ RemoteStore::Connection::~Connection() try { to.flush(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -985,7 +986,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sin try { std::rethrow_exception(ex); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 8d0bfcb11..7aa0b6629 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -85,7 +85,7 @@ SQLite::~SQLite() if (db && sqlite3_close(db) != SQLITE_OK) SQLiteError::throw_(db, "closing database"); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -124,7 +124,7 @@ SQLiteStmt::~SQLiteStmt() if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) SQLiteError::throw_(db, "finalizing statement '%s'", sql); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -248,7 +248,7 @@ SQLiteTxn::~SQLiteTxn() if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) SQLiteError::throw_(db, "aborting transaction"); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index fb60326c1..5c1fc0c1f 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -30,11 +30,6 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { public: - // Hack for getting remote build log output. - // Intentionally not in `SSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; - SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) , RemoteStoreConfig(params) @@ -49,8 +44,7 @@ public: sshPublicHostKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - compress, - logFD) + compress) { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 50d392779..18f80eef8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -829,7 +829,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m { size_t left; StorePathSet valid; - std::exception_ptr exc; + std::exception_ptr exc = {}; }; Sync<State> state_(State{paths.size(), StorePathSet()}); @@ -1163,7 +1163,7 @@ std::map<StorePath, StorePath> copyPaths( // not be within our control to change that, and we might still want // to at least copy the output paths. if (e.missingFeature == Xp::CaDerivations) - ignoreException(); + ignoreExceptionExceptInterrupt(); else throw; } diff --git a/src/libutil/async-collect.hh b/src/libutil/async-collect.hh new file mode 100644 index 000000000..9e0b8bad9 --- /dev/null +++ b/src/libutil/async-collect.hh @@ -0,0 +1,101 @@ +#pragma once +/// @file + +#include <kj/async.h> +#include <kj/common.h> +#include <kj/vector.h> +#include <list> +#include <optional> +#include <type_traits> + +namespace nix { + +template<typename K, typename V> +class AsyncCollect +{ +public: + using Item = std::conditional_t<std::is_void_v<V>, K, std::pair<K, V>>; + +private: + kj::ForkedPromise<void> allPromises; + std::list<Item> results; + size_t remaining; + + kj::ForkedPromise<void> signal; + kj::Maybe<kj::Own<kj::PromiseFulfiller<void>>> notify; + + void oneDone(Item item) + { + results.emplace_back(std::move(item)); + remaining -= 1; + KJ_IF_MAYBE (n, notify) { + (*n)->fulfill(); + notify = nullptr; + } + } + + kj::Promise<void> collectorFor(K key, kj::Promise<V> promise) + { + if constexpr (std::is_void_v<V>) { + return promise.then([this, key{std::move(key)}] { oneDone(std::move(key)); }); + } else { + return promise.then([this, key{std::move(key)}](V v) { + oneDone(Item{std::move(key), std::move(v)}); + }); + } + } + + kj::ForkedPromise<void> waitForAll(kj::Array<std::pair<K, kj::Promise<V>>> & promises) + { + kj::Vector<kj::Promise<void>> wrappers; + for (auto & [key, promise] : promises) { + wrappers.add(collectorFor(std::move(key), std::move(promise))); + } + + return kj::joinPromisesFailFast(wrappers.releaseAsArray()).fork(); + } + +public: + AsyncCollect(kj::Array<std::pair<K, kj::Promise<V>>> && promises) + : allPromises(waitForAll(promises)) + , remaining(promises.size()) + , signal{nullptr} + { + } + + kj::Promise<std::optional<Item>> next() + { + if (remaining == 0 && results.empty()) { + return {std::nullopt}; + } + + if (!results.empty()) { + auto result = std::move(results.front()); + results.pop_front(); + return {{std::move(result)}}; + } + + if (notify == nullptr) { + auto pair = kj::newPromiseAndFulfiller<void>(); + notify = std::move(pair.fulfiller); + signal = pair.promise.fork(); + } + + return signal.addBranch().exclusiveJoin(allPromises.addBranch()).then([this] { + return next(); + }); + } +}; + +/** + * Collect the results of a list of promises, in order of completion. + * Once any input promise is rejected all promises that have not been + * resolved or rejected will be cancelled and the exception rethrown. + */ +template<typename K, typename V> +AsyncCollect<K, V> asyncCollect(kj::Array<std::pair<K, kj::Promise<V>>> promises) +{ + return AsyncCollect<K, V>(std::move(promises)); +} + +} diff --git a/src/libutil/async-semaphore.hh b/src/libutil/async-semaphore.hh new file mode 100644 index 000000000..f8db31a68 --- /dev/null +++ b/src/libutil/async-semaphore.hh @@ -0,0 +1,122 @@ +#pragma once +/// @file +/// @brief A semaphore implementation usable from within a KJ event loop. + +#include <cassert> +#include <kj/async.h> +#include <kj/common.h> +#include <kj/exception.h> +#include <kj/list.h> +#include <kj/source-location.h> +#include <memory> +#include <optional> + +namespace nix { + +class AsyncSemaphore +{ +public: + class [[nodiscard("destroying a semaphore guard releases the semaphore immediately")]] Token + { + struct Release + { + void operator()(AsyncSemaphore * sem) const + { + sem->unsafeRelease(); + } + }; + + std::unique_ptr<AsyncSemaphore, Release> parent; + + public: + Token() = default; + Token(AsyncSemaphore & parent, kj::Badge<AsyncSemaphore>) : parent(&parent) {} + + bool valid() const + { + return parent != nullptr; + } + }; + +private: + struct Waiter + { + kj::PromiseFulfiller<Token> & fulfiller; + kj::ListLink<Waiter> link; + kj::List<Waiter, &Waiter::link> & list; + + Waiter(kj::PromiseFulfiller<Token> & fulfiller, kj::List<Waiter, &Waiter::link> & list) + : fulfiller(fulfiller) + , list(list) + { + list.add(*this); + } + + ~Waiter() + { + if (link.isLinked()) { + list.remove(*this); + } + } + }; + + const unsigned capacity_; + unsigned used_ = 0; + kj::List<Waiter, &Waiter::link> waiters; + + void unsafeRelease() + { + used_ -= 1; + while (used_ < capacity_ && !waiters.empty()) { + used_ += 1; + auto & w = waiters.front(); + w.fulfiller.fulfill(Token{*this, {}}); + waiters.remove(w); + } + } + +public: + explicit AsyncSemaphore(unsigned capacity) : capacity_(capacity) {} + + KJ_DISALLOW_COPY_AND_MOVE(AsyncSemaphore); + + ~AsyncSemaphore() + { + assert(waiters.empty() && "destroyed a semaphore with active waiters"); + } + + std::optional<Token> tryAcquire() + { + if (used_ < capacity_) { + used_ += 1; + return Token{*this, {}}; + } else { + return {}; + } + } + + kj::Promise<Token> acquire() + { + if (auto t = tryAcquire()) { + return std::move(*t); + } else { + return kj::newAdaptedPromise<Token, Waiter>(waiters); + } + } + + unsigned capacity() const + { + return capacity_; + } + + unsigned used() const + { + return used_; + } + + unsigned available() const + { + return capacity_ - used_; + } +}; +} diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 33cda211b..3b3e46a9a 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -49,7 +49,7 @@ unsigned int getMaxCPU() auto period = cpuMaxParts[1]; if (quota != "max") return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } + } catch (Error &) { ignoreExceptionInDestructor(lvlDebug); } #endif return 0; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index a7cbfbfd0..f57e3ef7d 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -4,6 +4,7 @@ #include "position.hh" #include "terminal.hh" #include "strings.hh" +#include "signals.hh" #include <iostream> #include <optional> @@ -132,7 +133,7 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR /** * A development aid for finding missing positions, to improve error messages. Example use: * - * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 just test --suite installcheck -v functional-lang * git diff -U20 tests * */ @@ -416,7 +417,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } -void ignoreException(Verbosity lvl) +void ignoreExceptionInDestructor(Verbosity lvl) { /* Make sure no exceptions leave this function. printError() also throws when remote is closed. */ @@ -429,4 +430,15 @@ void ignoreException(Verbosity lvl) } catch (...) { } } +void ignoreExceptionExceptInterrupt(Verbosity lvl) +{ + try { + throw; + } catch (const Interrupted & e) { + throw; + } catch (std::exception & e) { + printMsg(lvl, "error (ignored): %1%", e.what()); + } +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 73c1ccadd..885a2b218 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -70,17 +70,17 @@ inline bool operator<=(const Trace& lhs, const Trace& rhs); inline bool operator>=(const Trace& lhs, const Trace& rhs); struct ErrorInfo { - Verbosity level; + Verbosity level = Verbosity::lvlError; HintFmt msg; std::shared_ptr<Pos> pos; - std::list<Trace> traces; + std::list<Trace> traces = {}; /** * Exit status. */ unsigned int status = 1; - Suggestions suggestions; + Suggestions suggestions = {}; static std::optional<std::string> programName; }; @@ -204,7 +204,22 @@ public: /** * Exception handling in destructors: print an error message, then * ignore the exception. + * + * If you're not in a destructor, you usually want to use `ignoreExceptionExceptInterrupt()`. + * + * This function might also be used in callbacks whose caller may not handle exceptions, + * but ideally we propagate the exception using an exception_ptr in such cases. + * See e.g. `PackBuilderContext` + */ +void ignoreExceptionInDestructor(Verbosity lvl = lvlError); + +/** + * Not destructor-safe. + * Print an error message, then ignore the exception. + * If the exception is an `Interrupted` exception, rethrow it. + * + * This may be used in a few places where Interrupt can't happen, but that's ok. */ -void ignoreException(Verbosity lvl = lvlError); +void ignoreExceptionExceptInterrupt(Verbosity lvl = lvlError); } diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 8385ea402..cbb2bb539 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -146,7 +146,7 @@ AutoCloseFD::~AutoCloseFD() try { close(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 1d3eba58f..c4ffb1d0c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -522,7 +522,7 @@ AutoDelete::~AutoDelete() } } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index ee3e1e2e7..5feefdf90 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -136,11 +136,17 @@ inline std::string fmt(const char * s) template<typename... Args> inline std::string fmt(const std::string & fs, const Args &... args) -{ +try { boost::format f(fs); fmt_internal::setExceptions(f); (f % ... % args); return f.str(); +} catch (boost::io::format_error & fe) { + // I don't care who catches this, we do not put up with boost format errors + // Give me a stack trace and a core dump + std::cerr << "nix::fmt threw format error. Original format string: '"; + std::cerr << fs << "'; number of arguments: " << sizeof...(args) << "\n"; + std::terminate(); } /** @@ -174,15 +180,13 @@ public: std::cerr << "HintFmt received incorrect number of format args. Original format string: '"; std::cerr << format << "'; number of arguments: " << sizeof...(args) << "\n"; // And regardless of the coredump give me a damn stacktrace. - printStackTrace(); - abort(); + std::terminate(); } } catch (boost::io::format_error & ex) { // Same thing, but for anything that happens in the member initializers. std::cerr << "HintFmt received incorrect format string. Original format string: '"; std::cerr << format << "'; number of arguments: " << sizeof...(args) << "\n"; - printStackTrace(); - abort(); + std::terminate(); } HintFmt(const HintFmt & hf) : fmt(hf.fmt) {} diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 7d9482814..7609e6e39 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -352,7 +352,7 @@ Activity::~Activity() try { logger.stopActivity(id); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index a3f21de59..afca4e021 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -53,6 +53,8 @@ libutil_headers = files( 'archive.hh', 'args/root.hh', 'args.hh', + 'async-collect.hh', + 'async-semaphore.hh', 'backed-string-view.hh', 'box_ptr.hh', 'canon-path.hh', diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh index dc09a9ba4..dd6e2978e 100644 --- a/src/libutil/processes.hh +++ b/src/libutil/processes.hh @@ -78,11 +78,11 @@ struct RunOptions { Path program; bool searchPath = true; - Strings args; - std::optional<uid_t> uid; - std::optional<uid_t> gid; - std::optional<Path> chdir; - std::optional<std::map<std::string, std::string>> environment; + Strings args = {}; + std::optional<uid_t> uid = {}; + std::optional<uid_t> gid = {}; + std::optional<Path> chdir = {}; + std::optional<std::map<std::string, std::string>> environment = {}; bool captureStdout = false; bool mergeStderrToStdout = false; bool isInteractive = false; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index f509fedff..2f5a11a28 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -83,7 +83,7 @@ void BufferedSink::flush() FdSink::~FdSink() { - try { flush(); } catch (...) { ignoreException(); } + try { flush(); } catch (...) { ignoreExceptionInDestructor(); } } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 3a9685e0e..08ea9a135 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -549,7 +549,7 @@ struct FramedSource : Source } } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -595,7 +595,7 @@ struct FramedSink : nix::BufferedSink to << 0; to.flush(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc index 04a697d01..dac2964ae 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/signals.cc @@ -12,13 +12,18 @@ std::atomic<bool> _isInterrupted = false; thread_local std::function<bool()> interruptCheck; +Interrupted makeInterrupted() +{ + return Interrupted("interrupted by the user"); +} + void _interrupted() { /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ if (!std::uncaught_exceptions()) { - throw Interrupted("interrupted by the user"); + throw makeInterrupted(); } } @@ -78,7 +83,7 @@ void triggerInterrupt() try { callback(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } } diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh index 02f8d2ca3..538ff94b4 100644 --- a/src/libutil/signals.hh +++ b/src/libutil/signals.hh @@ -16,10 +16,13 @@ namespace nix { /* User interruption. */ +class Interrupted; + extern std::atomic<bool> _isInterrupted; extern thread_local std::function<bool()> interruptCheck; +Interrupted makeInterrupted(); void _interrupted(); void inline checkInterrupt() diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 0ff83e997..1c4488373 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -109,9 +109,21 @@ void ThreadPool::doWork(bool mainThread) try { std::rethrow_exception(exc); } catch (std::exception & e) { - if (!dynamic_cast<Interrupted*>(&e) && - !dynamic_cast<ThreadPoolShutDown*>(&e)) - ignoreException(); + if (!dynamic_cast<ThreadPoolShutDown*>(&e)) { + // Yes, this is not a destructor, but we cannot + // safely propagate an exception out of here. + // + // What happens is that if we do, shutdown() + // will have join() throw an exception if we + // are on a worker thread, preventing us from + // joining the rest of the threads. Although we + // could make the joining eat exceptions too, + // we could just as well not let Interrupted + // fall out to begin with, since the thread + // will immediately cleanly quit because of + // quit == true anyway. + ignoreExceptionInDestructor(); + } } catch (...) { } } diff --git a/src/meson.build b/src/meson.build index 66fbb13ba..8b63ef995 100644 --- a/src/meson.build +++ b/src/meson.build @@ -26,54 +26,9 @@ libasanoptions = declare_dependency( link_whole: asanoptions ) -build_remote_sources = files( - 'build-remote/build-remote.cc', -) -nix_build_sources = files( - 'nix-build/nix-build.cc', -) -nix_channel_sources = files( - 'nix-channel/nix-channel.cc', -) -unpack_channel_gen = gen_header.process('nix-channel/unpack-channel.nix') -nix_collect_garbage_sources = files( - 'nix-collect-garbage/nix-collect-garbage.cc', -) -nix_copy_closure_sources = files( - 'nix-copy-closure/nix-copy-closure.cc', -) -nix_env_buildenv_gen = gen_header.process('nix-env/buildenv.nix') -nix_env_sources = files( - 'nix-env/nix-env.cc', - 'nix-env/user-env.cc', -) -nix_instantiate_sources = files( - 'nix-instantiate/nix-instantiate.cc', -) -nix_store_sources = files( - 'nix-store/dotgraph.cc', - 'nix-store/graphml.cc', - 'nix-store/nix-store.cc', -) - -# Hurray for Meson list flattening! -nix2_commands_sources = [ - build_remote_sources, - nix_build_sources, - nix_channel_sources, - unpack_channel_gen, - nix_collect_garbage_sources, - nix_copy_closure_sources, - nix_env_buildenv_gen, - nix_env_sources, - nix_instantiate_sources, - nix_store_sources, -] +# Legacy commands. +subdir('legacy') # Finally, the nix command itself, which all of the other commands are implmented in terms of # as a multicall binary. subdir('nix') - -# Just copies nix-channel/unpack-channel.nix to the build directory. -# Done as a subdir to get Meson to respect the path hierarchy. -subdir('nix-channel') diff --git a/src/nix-channel/meson.build b/src/nix-channel/meson.build deleted file mode 100644 index 952dfdb78..000000000 --- a/src/nix-channel/meson.build +++ /dev/null @@ -1,5 +0,0 @@ -configure_file( - input : 'unpack-channel.nix', - output : 'unpack-channel.nix', - copy : true, -) diff --git a/src/nix/daemon-command.hh b/src/nix/daemon-command.hh new file mode 100644 index 000000000..454af88e2 --- /dev/null +++ b/src/nix/daemon-command.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixDaemon(); + +} diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index ca65c38e6..e1d183d7b 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -14,6 +14,7 @@ #include "signals.hh" #include "daemon.hh" #include "unix-domain-socket.hh" +#include "daemon-command.hh" #include <algorithm> #include <climits> @@ -36,7 +37,8 @@ #include <sys/ucred.h> #endif -using namespace nix; +namespace nix { + using namespace nix::daemon; /** @@ -496,7 +498,9 @@ static int main_nix_daemon(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); +void registerNixDaemon() { + LegacyCommands::add("nix-daemon", main_nix_daemon); +} struct CmdDaemon : StoreCommand { @@ -560,3 +564,5 @@ struct CmdDaemon : StoreCommand }; static auto rCmdDaemon = registerCommand2<CmdDaemon>({"daemon"}); + +} diff --git a/src/nix/develop.cc b/src/nix/develop.cc index fb144c904..d1615ecdc 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -639,7 +639,7 @@ struct CmdDevelop : Common, MixEnvironment throw Error("package 'nixpkgs#bashInteractive' does not provide a 'bin/bash'"); } catch (Error &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } // Override SHELL with the one chosen for this environment. diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 15c393c90..0c704a995 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -16,6 +16,7 @@ #include "eval-cache.hh" #include "markdown.hh" #include "terminal.hh" +#include "signals.hh" #include <limits> #include <nlohmann/json.hpp> @@ -367,9 +368,11 @@ struct CmdFlakeCheck : FlakeCommand auto reportError = [&](const Error & e) { try { throw e; + } catch (Interrupted & e) { + throw; } catch (Error & e) { if (settings.keepGoing) { - ignoreException(); + ignoreExceptionExceptInterrupt(); hasErrors = true; } else diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 059904150..f47f2204a 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -39,14 +39,8 @@ struct CmdFmt : SourceExprCommand { Strings programArgs{app.program}; // Propagate arguments from the CLI - if (args.empty()) { - // Format the current flake out of the box - programArgs.push_back("."); - } else { - // User wants more power, let them decide which paths to include/exclude - for (auto &i : args) { - programArgs.push_back(i); - } + for (auto &i : args) { + programArgs.push_back(i); } runProgramInStore(store, UseSearchPath::DontUse, app.program, programArgs); diff --git a/src/nix/hash-command.hh b/src/nix/hash-command.hh new file mode 100644 index 000000000..5383171a5 --- /dev/null +++ b/src/nix/hash-command.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixHash(); + +} diff --git a/src/nix/hash.cc b/src/nix/hash.cc index f6add527a..40b00c978 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -5,8 +5,9 @@ #include "shared.hh" #include "references.hh" #include "archive.hh" +#include "hash-command.hh" -using namespace nix; +namespace nix { struct CmdHashBase : Command { @@ -221,4 +222,8 @@ static int compatNixHash(int argc, char * * argv) return 0; } -static RegisterLegacyCommand r_nix_hash("nix-hash", compatNixHash); +void registerNixHash() { + LegacyCommands::add("nix-hash", compatNixHash); +} + +} diff --git a/src/nix/main.cc b/src/nix/main.cc index 4a3a7b4e7..fdd3ac2ae 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -15,6 +15,17 @@ #include "markdown.hh" #include "experimental-features-json.hh" #include "deprecated-features-json.hh" +#include "build-remote.hh" +#include "daemon-command.hh" +#include "hash-command.hh" +#include "nix-build.hh" +#include "nix-channel.hh" +#include "nix-collect-garbage.hh" +#include "nix-copy-closure.hh" +#include "nix-env.hh" +#include "nix-instantiate.hh" +#include "nix-store.hh" +#include "prefetch-command.hh" #include <sys/types.h> #include <sys/socket.h> @@ -30,6 +41,21 @@ void chrootHelper(int argc, char * * argv); namespace nix { +void registerLegacyCommands() +{ + registerNixEnv(); + registerNixBuildAndNixShell(); + registerNixInstantiate(); + registerNixCopyClosure(); + registerNixCollectGarbage(); + registerNixChannel(); + registerNixStore(); + registerBuildRemote(); + registerNixDaemon(); + registerNixPrefetchUrl(); + registerNixHash(); +} + static bool haveProxyEnvironmentVariables() { static const std::vector<std::string> proxyVariables = { @@ -356,8 +382,10 @@ void mainWrapped(int argc, char * * argv) // Clean up the progress bar if shown using --log-format in a legacy command too. // Otherwise, this is a harmless no-op. Finally f([] { logger->pause(); }); + { - auto legacy = (*RegisterLegacyCommand::commands)[programName]; + registerLegacyCommands(); + auto legacy = (*LegacyCommands::commands)[programName]; if (legacy) return legacy(argc, argv); } diff --git a/src/nix/meson.build b/src/nix/meson.build index 80223a390..cabdf0d2c 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -71,11 +71,21 @@ nix_sources = files( 'why-depends.cc', ) +nix_headers = files( + 'daemon-command.hh', + 'hash-command.hh', + 'prefetch-command.hh', +) + nix = executable( 'nix', nix_sources, + legacy_sources, nix_generated_headers, - nix2_commands_sources, + nix_headers, + legacy_headers, + legacy_generated_headers, + include_directories : legacy_include_directories, dependencies : [ libasanoptions, liblixcmd, diff --git a/src/nix/prefetch-command.hh b/src/nix/prefetch-command.hh new file mode 100644 index 000000000..078e83485 --- /dev/null +++ b/src/nix/prefetch-command.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixPrefetchUrl(); + +} diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b99cd5dd0..0183b0008 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -9,10 +9,11 @@ #include "eval-inline.hh" // IWYU pragma: keep #include "legacy.hh" #include "terminal.hh" +#include "prefetch-command.hh" #include <nlohmann/json.hpp> -using namespace nix; +namespace nix { /* If ‘url’ starts with ‘mirror://’, then resolve it using the list of mirrors defined in Nixpkgs. */ @@ -248,7 +249,9 @@ static int main_nix_prefetch_url(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); +void registerNixPrefetchUrl() { + LegacyCommands::add("nix-prefetch-url", main_nix_prefetch_url); +} struct CmdStorePrefetchFile : StoreCommand, MixJSON { @@ -328,3 +331,5 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON }; static auto rCmdStorePrefetchFile = registerCommand2<CmdStorePrefetchFile>({"store", "prefetch-file"}); + +} diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 401d5bd77..6739cb5c6 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -8,7 +8,7 @@ #include "archive.hh" #include "builtins/buildenv.hh" #include "flake/flakeref.hh" -#include "../nix-env/user-env.hh" +#include "user-env.hh" #include "profiles.hh" #include "names.hh" diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 356985a64..58fba83aa 100644 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -144,13 +144,10 @@ test "$(<<<"$out" grep -E '^error:' | wc -l)" = 1 # --keep-going and FOD out="$(nix build -f fod-failing.nix -L 2>&1)" && status=0 || status=$? test "$status" = 1 -# one "hash mismatch" error, one "build of ... failed" -test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 -<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x1\\.drv'" -<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" -<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" -<<<"$out" grepQuiet -E "likely URL: https://meow.puppy.forge/puppy.tar.gz" -<<<"$out" grepQuiet -vE "likely URL: https://kitty.forge/cat.tar.gz" +# at least one "hash mismatch" error, one "build of ... failed" +test "$(<<<"$out" grep -E '^error:' | wc -l)" -ge 2 +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x.\\.drv'" +<<<"$out" grepQuiet -E "likely URL: " <<<"$out" grepQuiet -E "error: build of '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out' failed" out="$(nix build -f fod-failing.nix -L x1 x2 x3 --keep-going 2>&1)" && status=0 || status=$? @@ -167,9 +164,9 @@ test "$(<<<"$out" grep -E '^error:' | wc -l)" = 4 out="$(nix build -f fod-failing.nix -L x4 2>&1)" && status=0 || status=$? test "$status" = 1 -test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 -<<<"$out" grepQuiet -E "error: 1 dependencies of derivation '.*-x4\\.drv' failed to build" -<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x2\\.drv'" +test "$(<<<"$out" grep -E '^error:' | wc -l)" -ge 2 +<<<"$out" grepQuiet -E "error: [12] dependencies of derivation '.*-x4\\.drv' failed to build" +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x[23]\\.drv'" out="$(nix build -f fod-failing.nix -L x4 --keep-going 2>&1)" && status=0 || status=$? test "$status" = 1 diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in index 451cf5383..98892f660 100644 --- a/tests/functional/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -28,6 +28,7 @@ export NIX_REMOTE=${NIX_REMOTE_-} unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home export HOME=$TEST_HOME +export GIT_CONFIG_SYSTEM=/dev/null unset XDG_STATE_HOME unset XDG_DATA_HOME unset XDG_CONFIG_HOME diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index be1a7ae9a..0ede76b71 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -1,8 +1,7 @@ { hashInvalidator ? "" }: with import ./config.nix; -let { - +let input0 = mkDerivation { name = "dependencies-input-0"; buildCommand = "mkdir $out; echo foo > $out/bar"; @@ -32,17 +31,15 @@ let { outputHashAlgo = "sha256"; outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; }; - - body = mkDerivation { - name = "dependencies-top"; - builder = ./dependencies.builder0.sh + "/FOOBAR/../."; - input1 = input1 + "/."; - input2 = "${input2}/."; - input1_drv = input1; - input2_drv = input2; - input0_drv = input0; - fod_input_drv = fod_input; - meta.description = "Random test package"; - }; - +in +mkDerivation { + name = "dependencies-top"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + input2_drv = input2; + input0_drv = input0; + fod_input_drv = fod_input; + meta.description = "Random test package"; } diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index 2c00facc2..492c57602 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -53,8 +53,17 @@ out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\" [[ $status == 1 ]] [[ $out =~ 'Cannot find Git revision' ]] +# allow revs as refs (for 2.3 compat) [[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] +rm -rf "$TEST_ROOT/test-home" +[[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] + +rm -rf "$TEST_ROOT/test-home" +out=$(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; rev = \"$devrev\"; ref = \"lolkek\"; } + \"/differentbranch\")" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'Cannot find Git revision' ]] + # In pure eval mode, fetchGit without a revision should fail. [[ $(nix eval --impure --raw --expr "builtins.readFile (fetchGit \"file://$repo\" + \"/hello\")") = world ]] (! nix eval --raw --expr "builtins.readFile (fetchGit \"file://$repo\" + \"/hello\")") @@ -228,6 +237,12 @@ export _NIX_FORCE_HTTP=1 rev_tag1_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/tag1\"; }).rev") rev_tag1=$(git -C $repo rev-parse refs/tags/tag1) [[ $rev_tag1_nix = $rev_tag1 ]] + +# Allow fetching tags w/o specifying refs/tags +rm -rf "$TEST_ROOT/test-home" +rev_tag1_nix_alt=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"tag1\"; }).rev") +[[ $rev_tag1_nix_alt = $rev_tag1 ]] + rev_tag2_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/tag2\"; }).rev") rev_tag2=$(git -C $repo rev-parse refs/tags/tag2) [[ $rev_tag2_nix = $rev_tag2 ]] @@ -254,3 +269,33 @@ git -C "$repo" add hello .gitignore git -C "$repo" commit -m 'Bla1' cd "$repo" path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath") + +# test behavior if both branch and tag with same name exist +repo="$TEST_ROOT/git" +rm -rf "$repo"/.git +git init "$repo" +git -C "$repo" config user.email "foobar@example.com" +git -C "$repo" config user.name "Foobar" + +touch "$repo"/test +echo "hello world" > "$repo"/test +git -C "$repo" checkout -b branch +git -C "$repo" add test + +git -C "$repo" commit -m "Init" + +git -C "$repo" tag branch + +echo "goodbye world" > "$repo"/test +git -C "$repo" add test +git -C "$repo" commit -m "Update test" + +path12=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"branch\"; }).outPath") +[[ "$(cat "$path12"/test)" =~ 'hello world' ]] +[[ "$(cat "$repo"/test)" =~ 'goodbye world' ]] + +path13=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/heads/branch\"; }).outPath") +[[ "$(cat "$path13"/test)" =~ 'goodbye world' ]] + +path14=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/branch\"; }).outPath") +[[ "$path14" = "$path12" ]] diff --git a/tests/functional/fmt.sh b/tests/functional/fmt.sh index 3c1bd9989..7d6add9b6 100644 --- a/tests/functional/fmt.sh +++ b/tests/functional/fmt.sh @@ -26,7 +26,10 @@ cat << EOF > flake.nix }; } EOF -nix fmt ./file ./folder | grep 'Formatting: ./file ./folder' +# No arguments check +[[ "$(nix fmt)" = "Formatting(0):" ]] +# Argument forwarding check +nix fmt ./file ./folder | grep 'Formatting(2): ./file ./folder' nix flake check nix flake show | grep -P "package 'formatter'" diff --git a/tests/functional/fmt.simple.sh b/tests/functional/fmt.simple.sh index 03109a655..f655846ca 100755 --- a/tests/functional/fmt.simple.sh +++ b/tests/functional/fmt.simple.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -echo Formatting: "${@}" +echo "Formatting(${#}):" "${@}" diff --git a/tests/functional/hash-check.nix b/tests/functional/hash-check.nix index f029f0cc9..6095bc57f 100644 --- a/tests/functional/hash-check.nix +++ b/tests/functional/hash-check.nix @@ -1,5 +1,4 @@ -let { - +let input1 = derivation { name = "dependencies-input-1"; system = "i086-msdos"; @@ -16,14 +15,12 @@ let { outputHashAlgo = "md5"; outputHash = "ffffffffffffffffffffffffffffffff"; }; - - body = derivation { - name = "dependencies"; - system = "i086-msdos"; - builder = "/bar/sh"; - args = ["-e" "-x" (./dummy + "/FOOBAR/../.")]; - input1 = input1 + "/."; - inherit input2; - }; - +in +derivation { + name = "dependencies"; + system = "i086-msdos"; + builder = "/bar/sh"; + args = ["-e" "-x" (./dummy + "/FOOBAR/../.")]; + input1 = input1 + "/."; + inherit input2; } diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 59db10340..cec4f9352 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -41,7 +41,12 @@ badExitCode=0 for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; i=$(basename "$i" .nix) - if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" + + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" + fi + if expectStderr 1 nix-instantiate --parse "${flags[@]}" - < "lang/$i.nix" > "lang/$i.err" then diffAndAccept "$i" err err.exp else @@ -54,13 +59,12 @@ for i in lang/parse-okay-*.nix; do echo "parsing $i (should succeed)"; i=$(basename "$i" .nix) - if [ -e "lang/$i.flags" ]; then - extraArgs="$(cat "lang/$i.flags")" - else - extraArgs="" + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" fi if - expect 0 nix-instantiate --parse ${extraArgs-} - < "lang/$i.nix" \ + expect 0 nix-instantiate --parse "${flags[@]}" - < "lang/$i.nix" \ 1> "lang/$i.out" \ 2> "lang/$i.err" then @@ -77,13 +81,12 @@ for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; i=$(basename "$i" .nix) - if [ -e "lang/$i.flags" ]; then - extraArgs="$(cat "lang/$i.flags")" - else - extraArgs="" + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" fi if - expectStderr 1 nix-instantiate --eval --strict --show-trace ${extraArgs-} "lang/$i.nix" \ + expectStderr 1 nix-instantiate --eval --strict --show-trace "${flags[@]}" "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then diffAndAccept "$i" err err.exp @@ -97,14 +100,13 @@ for i in lang/eval-okay-*.nix; do echo "evaluating $i (should succeed)"; i=$(basename "$i" .nix) - if [ -e "lang/$i.flags" ]; then - extraArgs="$(cat "lang/$i.flags")" - else - extraArgs="" + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" fi if test -e "lang/$i.exp.xml"; then - if expect 0 nix-instantiate --eval --xml --no-location --strict ${extraArgs-} \ + if expect 0 nix-instantiate --eval --xml --no-location --strict "${flags[@]}" \ "lang/$i.nix" > "lang/$i.out.xml" then diffAndAccept "$i" out.xml exp.xml @@ -113,11 +115,6 @@ for i in lang/eval-okay-*.nix; do badExitCode=1 fi elif test ! -e "lang/$i.exp-disabled"; then - declare -a flags=() - if test -e "lang/$i.flags"; then - read -r -a flags < "lang/$i.flags" - fi - if expect 0 env \ NIX_PATH=lang/dir3:lang/dir4 \ @@ -155,7 +152,7 @@ else echo '' echo 'You can rerun this test with:' echo '' - echo ' _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test' + echo ' _NIX_TEST_ACCEPT=1 just test --suite installcheck -v functional-lang' echo '' echo 'to regenerate the files containing the expected output,' echo 'and then view the git diff to decide whether a change is' diff --git a/tests/functional/lang/eval-okay-ind-string.exp b/tests/functional/lang/eval-okay-ind-string.exp index 7862331fa..1531af0f4 100644 --- a/tests/functional/lang/eval-okay-ind-string.exp +++ b/tests/functional/lang/eval-okay-ind-string.exp @@ -1 +1 @@ -"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', \${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: \${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n" +"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', \${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: \${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n Lines without any indentation effectively disable the indentation\n stripping for the entire string:\n\n cat >$out/foo/data <<EOF\n lasjdöaxnasd\nasdom 12398\nä\"§Æẞ¢«»”alsd\nEOF\nEmpty lines with a bit of whitespace don't affect the indentation calculation:\n\nAnd empty lines with more whitespace will have whitespace in the string:\n \nUnless it's the last line:\n Indentation stripping\n must not be impressed by\nthe last line not being empty\t Nor by people\n weirdly mixing tabs\n\tand spaces\n\t" diff --git a/tests/functional/lang/eval-okay-ind-string.nix b/tests/functional/lang/eval-okay-ind-string.nix index 95d59b508..44df83458 100644 --- a/tests/functional/lang/eval-okay-ind-string.nix +++ b/tests/functional/lang/eval-okay-ind-string.nix @@ -64,6 +64,36 @@ let is significant (not whitespace). ''; + s18 = '' + Lines without any indentation effectively disable the indentation + stripping for the entire string: + + cat >$out/foo/data <<EOF + lasjdöaxnasd +asdom 12398 +ä"§Æẞ¢«»”alsd +EOF + ''; + + s19 = '' + Empty lines with a bit of whitespace don't affect the indentation calculation: + + And empty lines with more whitespace will have whitespace in the string: + + Unless it's the last line: + ''; + + s20 = '' + Indentation stripping + must not be impressed by + the last line not being empty''; + + s21 = '' + Nor by people + weirdly mixing tabs + and spaces + ''; + s10 = '' ''; @@ -125,4 +155,4 @@ let # Accept dollars at end of strings s17 = ''ending dollar $'' + ''$'' + "\n"; -in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + s16 + s17 +in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + s16 + s17 + s18 + s19 + s20 + s21 diff --git a/tests/functional/lang/parse-okay-ind-string.exp b/tests/functional/lang/parse-okay-ind-string.exp new file mode 100644 index 000000000..570785ee2 --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.exp @@ -0,0 +1 @@ +(let s1 = "This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\n"; s10 = ""; s11 = ""; s12 = ""; s13 = ("start on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n " + "${(if true then "ln -sf 123 /var/run/opengl-driver" else (if true then "ln -sf 456 /var/run/opengl-driver" else ""))}" + "\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=" + "abc" + "\nenv SLIM_THEMESDIR=" + "def" + "\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=" + "foo" + "/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=" + "libX11" + "/lib:" + "libXext" + "/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\n" + "${(if true then ("env XORG_DRI_DRIVER_PATH=" + "nvidiaDrivers" + "/X11R6/lib/modules/drivers/") else (if true then ("env XORG_DRI_DRIVER_PATH=" + "mesa" + "/lib/modules/dri") else ""))}" + " \n\nexec " + "slim" + "/bin/slim\n"); s14 = "Escaping of ' followed by ': ''\nEscaping of $ followed by {: \${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\n"; s15 = (let x = "bla"; in ("foo\n'" + "${x}" + "'\nbar\n")); s16 = "cut -d $'\\t' -f 1\n"; s17 = (("ending dollar $" + "$") + "\n"); s18 = " Lines without any indentation effectively disable the indentation\n stripping for the entire string:\n\n cat >$out/foo/data <<EOF\n lasjdöaxnasd\nasdom 12398\nä\"§Æẞ¢«»”alsd\nEOF\n"; s19 = "Empty lines with a bit of whitespace don't affect the indentation calculation:\n\nAnd empty lines with more whitespace will have whitespace in the string:\n \nUnless it's the last line:\n"; s2 = "If the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \n"; s20 = " Indentation stripping\n must not be impressed by\nthe last line not being empty"; s21 = "\t Nor by people\n weirdly mixing tabs\n\tand spaces\n\t"; s3 = "This line is indented\na bit further.\n"; s4 = ("Anti-quotations, like " + "${(if true then "so" else "not so")}" + ", are\nalso allowed.\n"); s5 = (" The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: " + "''" + ", " + "\${" + ".\n"); s6 = " Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\n"; s7 = "Also note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. "; s8 = ("" + "\nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n"); s9 = ("" + " Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\n"); in ((((((((((((((((((((s1 + s2) + s3) + s4) + s5) + s6) + s7) + s8) + s9) + s10) + s11) + s12) + s13) + s14) + s15) + s16) + s17) + s18) + s19) + s20) + s21)) diff --git a/tests/functional/lang/parse-okay-ind-string.nix b/tests/functional/lang/parse-okay-ind-string.nix new file mode 120000 index 000000000..43864cd8c --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.nix @@ -0,0 +1 @@ +eval-okay-ind-string.nix
\ No newline at end of file diff --git a/tests/functional/lang/parse-okay-regression-751.exp b/tests/functional/lang/parse-okay-regression-751.exp index e2ed886fe..0cbf55d49 100644 --- a/tests/functional/lang/parse-okay-regression-751.exp +++ b/tests/functional/lang/parse-okay-regression-751.exp @@ -1 +1 @@ -(let const = (a: "const"); in ((const { x = "q"; }))) +(let const = (a: "const"); in ("${(const { x = "q"; })}")) diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 450674bd6..dd6386278 100644 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -11,8 +11,8 @@ nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.n (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src +(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/legacy') +nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/legacy' -I src=../../src (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=. diff --git a/tests/functional2/README.md b/tests/functional2/README.md new file mode 100644 index 000000000..b0440fb3a --- /dev/null +++ b/tests/functional2/README.md @@ -0,0 +1,24 @@ +# functional2 tests + +This uncreatively named test suite is a Pytest based replacement for the shell framework used to write traditional Nix integration tests. +Its primary goal is to make tests more concise, more self-contained, easier to write, and to produce better errors. + +## Goals + +- Eliminate implicit dependencies on files in the test directory as well as the requirement to copy the test files to the build directory as is currently hacked in the other functional test suite. + - You should be able to write a DirectoryTree of files for your test declaratively. +- Reduce the amount of global environment state being thrown around in the test suite. +- Make tests very concise and easy to reuse code for, and hopefully turn more of what is currently code into data. + - Provide rich ways of calling `nix` with pleasant syntax. + +## TODO: Intended features + +- [ ] Expect tests ([pytest-expect-test]) or snapshot tests ([pytest-insta]) or, likely, both! + We::jade prefer to have short output written in-line as it makes it greatly easier to read the tests, but pytest-expect doesn't allow for putting larger stuff in external files, so something else is necessary for those. +- [ ] Web server fixture: we don't test our network functionality because background processes are hard and this is simply goofy. + We could just test it. +- [ ] Nix daemon fixture. +- [x] Parallelism via pytest-xdist. + +[pytest-expect-test]: https://pypi.org/project/pytest-expect-test/ +[pytest-insta]: https://pypi.org/project/pytest-insta/ diff --git a/tests/functional2/__init__.py b/tests/functional2/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/functional2/__init__.py diff --git a/tests/functional2/conftest.py b/tests/functional2/conftest.py new file mode 100644 index 000000000..406f97b75 --- /dev/null +++ b/tests/functional2/conftest.py @@ -0,0 +1,8 @@ +import pytest +from pathlib import Path +from .testlib import fixtures + + +@pytest.fixture +def nix(tmp_path: Path): + return fixtures.Nix(tmp_path) diff --git a/tests/functional2/meson.build b/tests/functional2/meson.build new file mode 100644 index 000000000..b18d7035d --- /dev/null +++ b/tests/functional2/meson.build @@ -0,0 +1,29 @@ +xdist_opts = [ + # auto number of workers, max 12 jobs + '-n', 'auto', '--maxprocesses=12', + # group tests by module or class; ensures that any setup work occurs as little as possible + '--dist=loadscope', +] + +# surprisingly, this actually works even if PATH is set to something before +# meson gets hold of it. neat! +functional2_env = environment() +functional2_env.prepend('PATH', bindir) + +test( + 'functional2', + python, + args : [ + '-m', 'pytest', + '-v', + xdist_opts, + meson.current_source_dir() + ], + env : functional2_env, + # FIXME: Although we can trivially use TAP here with pytest-tap, due to a meson bug, it is unusable. + # (failure output does not get displayed to the console. at all. someone should go fix it): + # https://github.com/mesonbuild/meson/issues/11185 + # protocol : 'tap', + suite : 'installcheck', + timeout : 300, +) diff --git a/tests/functional2/test_eval_trivial.py b/tests/functional2/test_eval_trivial.py new file mode 100644 index 000000000..8bde9d22c --- /dev/null +++ b/tests/functional2/test_eval_trivial.py @@ -0,0 +1,4 @@ +from .testlib.fixtures import Nix + +def test_trivial_addition(nix: Nix): + assert nix.eval('1 + 1').json() == 2 diff --git a/tests/functional2/testlib/__init__.py b/tests/functional2/testlib/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/functional2/testlib/__init__.py diff --git a/tests/functional2/testlib/fixtures.py b/tests/functional2/testlib/fixtures.py new file mode 100644 index 000000000..bbaaae51d --- /dev/null +++ b/tests/functional2/testlib/fixtures.py @@ -0,0 +1,121 @@ +import os +import json +import subprocess +from typing import Any +from pathlib import Path +import dataclasses + + +@dataclasses.dataclass +class CommandResult: + cmd: list[str] + rc: int + """Return code""" + stderr: bytes + """Outputted stderr""" + stdout: bytes + """Outputted stdout""" + + def ok(self): + if self.rc != 0: + raise subprocess.CalledProcessError(returncode=self.rc, + cmd=self.cmd, + stderr=self.stderr, + output=self.stdout) + return self + + def json(self) -> Any: + self.ok() + return json.loads(self.stdout) + + +@dataclasses.dataclass +class NixSettings: + """Settings for invoking Nix""" + experimental_features: set[str] | None = None + + def feature(self, *names: str): + self.experimental_features = (self.experimental_features + or set()) | set(names) + return self + + def to_config(self) -> str: + config = '' + + def serialise(value): + if type(value) in {str, int}: + return str(value) + elif type(value) in {list, set}: + return ' '.join(str(e) for e in value) + else: + raise ValueError( + f'Value is unsupported in nix config: {value!r}') + + def field_may(name, value, serialiser=serialise): + nonlocal config + if value is not None: + config += f'{name} = {serialiser(value)}\n' + + field_may('experimental-features', self.experimental_features) + return config + + +@dataclasses.dataclass +class Nix: + test_root: Path + + def hermetic_env(self): + # mirroring vars-and-functions.sh + home = self.test_root / 'test-home' + home.mkdir(parents=True, exist_ok=True) + return { + 'NIX_STORE_DIR': self.test_root / 'store', + 'NIX_LOCALSTATE_DIR': self.test_root / 'var', + 'NIX_LOG_DIR': self.test_root / 'var/log/nix', + 'NIX_STATE_DIR': self.test_root / 'var/nix', + 'NIX_CONF_DIR': self.test_root / 'etc', + 'NIX_DAEMON_SOCKET_PATH': self.test_root / 'daemon-socket', + 'NIX_USER_CONF_FILES': '', + 'HOME': home, + } + + def make_env(self): + # We conservatively assume that people might want to successfully get + # some env through to the subprocess, so we override whatever is in the + # global env. + d = os.environ.copy() + d.update(self.hermetic_env()) + return d + + def call(self, cmd: list[str], extra_env: dict[str, str] = {}): + """ + Calls a process in the test environment. + """ + env = self.make_env() + env.update(extra_env) + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.test_root, + env=env, + ) + (stdout, stderr) = proc.communicate() + rc = proc.returncode + return CommandResult(cmd=cmd, rc=rc, stdout=stdout, stderr=stderr) + + def nix(self, + cmd: list[str], + settings: NixSettings = NixSettings(), + extra_env: dict[str, str] = {}): + extra_env = extra_env.copy() + extra_env.update({'NIX_CONFIG': settings.to_config()}) + return self.call(['nix', *cmd], extra_env) + + def eval( + self, expr: str, + settings: NixSettings = NixSettings()) -> CommandResult: + # clone due to reference-shenanigans + settings = dataclasses.replace(settings).feature('nix-command') + + return self.nix(['eval', '--json', '--expr', expr], settings=settings) diff --git a/tests/nixos/fetchurl.nix b/tests/nixos/fetchurl.nix index 63c639c31..97365d053 100644 --- a/tests/nixos/fetchurl.nix +++ b/tests/nixos/fetchurl.nix @@ -67,6 +67,9 @@ in out = machine.succeed("curl https://good/index.html") assert out == "hello world\n" + out = machine.succeed("cat ${badCert}/cert.pem > /tmp/cafile.pem; curl --cacert /tmp/cafile.pem https://bad/index.html") + assert out == "foobar\n" + # Fetching from a server with a trusted cert should work. machine.succeed("nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://good/index.html\"; hash = \"sha256-qUiQTy8PR5uPgZdpSzAYSw0u0cHNKh7A+4XSmaGSpEc=\"; }'") @@ -74,5 +77,8 @@ in err = machine.fail("nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }' 2>&1") print(err) assert "SSL certificate problem: self-signed certificate" in err or "SSL peer certificate or SSH remote key was not OK" in err + + # Fetching from a server with a trusted cert should work via environment variable override. + machine.succeed("NIX_SSL_CERT_FILE=/tmp/cafile.pem nix build --no-substitute --expr 'import <nix/fetchurl.nix> { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }'") ''; } diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix index 8deb9a504..ec12f9066 100644 --- a/tests/nixos/remote-builds-ssh-ng.nix +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -97,7 +97,8 @@ in builder.wait_for_unit("sshd.service") out = client.fail("nix-build ${expr nodes.client 1} 2>&1") - assert "error: failed to start SSH connection to 'root@builder': Host key verification failed" in out, f"No host verification error in {out}" + assert "Host key verification failed." in out, f"No host verification error:\n{out}" + assert "warning: SSH to 'root@builder' failed, stdout first line: '''" in out, f"No details about which host:\n{out}" client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world' >&2") diff --git a/tests/unit/libmain/crash.cc b/tests/unit/libmain/crash.cc new file mode 100644 index 000000000..883dc39bd --- /dev/null +++ b/tests/unit/libmain/crash.cc @@ -0,0 +1,56 @@ +#include <gtest/gtest.h> +#include "crash-handler.hh" + +namespace nix { + +class OopsException : public std::exception +{ + const char * msg; + +public: + OopsException(const char * msg) : msg(msg) {} + const char * what() const noexcept override + { + return msg; + } +}; + +void causeCrashForTesting(std::function<void()> fixture) +{ + registerCrashHandler(); + std::cerr << "time to crash\n"; + try { + fixture(); + } catch (...) { + std::terminate(); + } +} + +TEST(CrashHandler, exceptionName) +{ + ASSERT_DEATH( + causeCrashForTesting([]() { throw OopsException{"lol oops"}; }), + "time to crash\nLix crashed.*OopsException: lol oops" + ); +} + +TEST(CrashHandler, unknownTerminate) +{ + ASSERT_DEATH( + causeCrashForTesting([]() { std::terminate(); }), + "time to crash\nLix crashed.*std::terminate\\(\\) called without exception" + ); +} + +TEST(CrashHandler, nonStdException) +{ + ASSERT_DEATH( + causeCrashForTesting([]() { + // NOLINTNEXTLINE(hicpp-exception-baseclass): intentional + throw 4; + }), + "time to crash\nLix crashed.*Unknown exception! Spooky\\." + ); +} + +} diff --git a/tests/unit/libstore/filetransfer.cc b/tests/unit/libstore/filetransfer.cc index 71e7392fc..fd4d326f0 100644 --- a/tests/unit/libstore/filetransfer.cc +++ b/tests/unit/libstore/filetransfer.cc @@ -150,6 +150,14 @@ TEST(FileTransfer, exceptionAbortsDownload) } } +TEST(FileTransfer, exceptionAbortsRead) +{ + auto [port, srv] = serveHTTP("200 ok", "content-length: 0\r\n", [] { return ""; }); + auto ft = makeFileTransfer(); + char buf[10] = ""; + ASSERT_THROW(ft->download(FileTransferRequest(fmt("http://[::1]:%d/index", port)))->read(buf, 10), EndOfFile); +} + TEST(FileTransfer, NOT_ON_DARWIN(reportsSetupErrors)) { auto [port, srv] = serveHTTP("404 not found", "", [] { return ""; }); diff --git a/tests/unit/libutil/async-collect.cc b/tests/unit/libutil/async-collect.cc new file mode 100644 index 000000000..770374d21 --- /dev/null +++ b/tests/unit/libutil/async-collect.cc @@ -0,0 +1,104 @@ +#include "async-collect.hh" + +#include <gtest/gtest.h> +#include <kj/array.h> +#include <kj/async.h> +#include <kj/exception.h> +#include <stdexcept> + +namespace nix { + +TEST(AsyncCollect, void) +{ + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + auto a = kj::newPromiseAndFulfiller<void>(); + auto b = kj::newPromiseAndFulfiller<void>(); + auto c = kj::newPromiseAndFulfiller<void>(); + auto d = kj::newPromiseAndFulfiller<void>(); + + auto collect = asyncCollect(kj::arr( + std::pair(1, std::move(a.promise)), + std::pair(2, std::move(b.promise)), + std::pair(3, std::move(c.promise)), + std::pair(4, std::move(d.promise)) + )); + + auto p = collect.next(); + ASSERT_FALSE(p.poll(waitScope)); + + // collection is ordered + c.fulfiller->fulfill(); + b.fulfiller->fulfill(); + + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_EQ(p.wait(waitScope), 3); + + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_EQ(p.wait(waitScope), 2); + + p = collect.next(); + ASSERT_FALSE(p.poll(waitScope)); + + // exceptions propagate + a.fulfiller->rejectIfThrows([] { throw std::runtime_error("test"); }); + + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_THROW(p.wait(waitScope), kj::Exception); + + // first exception aborts collection + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_THROW(p.wait(waitScope), kj::Exception); +} + +TEST(AsyncCollect, nonVoid) +{ + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + auto a = kj::newPromiseAndFulfiller<int>(); + auto b = kj::newPromiseAndFulfiller<int>(); + auto c = kj::newPromiseAndFulfiller<int>(); + auto d = kj::newPromiseAndFulfiller<int>(); + + auto collect = asyncCollect(kj::arr( + std::pair(1, std::move(a.promise)), + std::pair(2, std::move(b.promise)), + std::pair(3, std::move(c.promise)), + std::pair(4, std::move(d.promise)) + )); + + auto p = collect.next(); + ASSERT_FALSE(p.poll(waitScope)); + + // collection is ordered + c.fulfiller->fulfill(1); + b.fulfiller->fulfill(2); + + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_EQ(p.wait(waitScope), std::pair(3, 1)); + + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_EQ(p.wait(waitScope), std::pair(2, 2)); + + p = collect.next(); + ASSERT_FALSE(p.poll(waitScope)); + + // exceptions propagate + a.fulfiller->rejectIfThrows([] { throw std::runtime_error("test"); }); + + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_THROW(p.wait(waitScope), kj::Exception); + + // first exception aborts collection + p = collect.next(); + ASSERT_TRUE(p.poll(waitScope)); + ASSERT_THROW(p.wait(waitScope), kj::Exception); +} +} diff --git a/tests/unit/libutil/async-semaphore.cc b/tests/unit/libutil/async-semaphore.cc new file mode 100644 index 000000000..12b52885d --- /dev/null +++ b/tests/unit/libutil/async-semaphore.cc @@ -0,0 +1,74 @@ +#include "async-semaphore.hh" + +#include <gtest/gtest.h> +#include <kj/async.h> + +namespace nix { + +TEST(AsyncSemaphore, counting) +{ + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + AsyncSemaphore sem(2); + + ASSERT_EQ(sem.available(), 2); + ASSERT_EQ(sem.used(), 0); + + auto a = kj::evalNow([&] { return sem.acquire(); }); + ASSERT_EQ(sem.available(), 1); + ASSERT_EQ(sem.used(), 1); + auto b = kj::evalNow([&] { return sem.acquire(); }); + ASSERT_EQ(sem.available(), 0); + ASSERT_EQ(sem.used(), 2); + + auto c = kj::evalNow([&] { return sem.acquire(); }); + auto d = kj::evalNow([&] { return sem.acquire(); }); + + ASSERT_TRUE(a.poll(waitScope)); + ASSERT_TRUE(b.poll(waitScope)); + ASSERT_FALSE(c.poll(waitScope)); + ASSERT_FALSE(d.poll(waitScope)); + + a = nullptr; + ASSERT_TRUE(c.poll(waitScope)); + ASSERT_FALSE(d.poll(waitScope)); + + { + auto lock = b.wait(waitScope); + ASSERT_FALSE(d.poll(waitScope)); + } + + ASSERT_TRUE(d.poll(waitScope)); + + ASSERT_EQ(sem.available(), 0); + ASSERT_EQ(sem.used(), 2); + c = nullptr; + ASSERT_EQ(sem.available(), 1); + ASSERT_EQ(sem.used(), 1); + d = nullptr; + ASSERT_EQ(sem.available(), 2); + ASSERT_EQ(sem.used(), 0); +} + +TEST(AsyncSemaphore, cancelledWaiter) +{ + kj::EventLoop loop; + kj::WaitScope waitScope(loop); + + AsyncSemaphore sem(1); + + auto a = kj::evalNow([&] { return sem.acquire(); }); + auto b = kj::evalNow([&] { return sem.acquire(); }); + auto c = kj::evalNow([&] { return sem.acquire(); }); + + ASSERT_TRUE(a.poll(waitScope)); + ASSERT_FALSE(b.poll(waitScope)); + + b = nullptr; + a = nullptr; + + ASSERT_TRUE(c.poll(waitScope)); +} + +} diff --git a/tests/unit/libutil/compression.cc b/tests/unit/libutil/compression.cc index 8d181f53d..6dd8c96df 100644 --- a/tests/unit/libutil/compression.cc +++ b/tests/unit/libutil/compression.cc @@ -1,4 +1,5 @@ #include "compression.hh" +#include <cstddef> #include <gtest/gtest.h> namespace nix { @@ -147,7 +148,7 @@ TEST_P(PerTypeNonNullCompressionTest, truncatedValidInput) /* n.b. This also tests zero-length input, which is also invalid. * As of the writing of this comment, it returns empty output, but is * allowed to throw a compression error instead. */ - for (int i = 0; i < compressed.length(); ++i) { + for (size_t i = 0u; i < compressed.length(); ++i) { auto newCompressed = compressed.substr(compressed.length() - i); try { decompress(method, newCompressed); diff --git a/tests/unit/meson.build b/tests/unit/meson.build index 8ff0b5ec5..9db619c5d 100644 --- a/tests/unit/meson.build +++ b/tests/unit/meson.build @@ -12,7 +12,11 @@ # It's only ~200 lines; better to just refactor the tests themselves which we'll want to do anyway. default_test_env = { - 'ASAN_OPTIONS': 'detect_leaks=0:halt_on_error=1:abort_on_error=1:print_summary=1:dump_instruction_bytes=1' + 'ASAN_OPTIONS': 'detect_leaks=0:halt_on_error=1:abort_on_error=1:print_summary=1:dump_instruction_bytes=1', + # Prevents loading global configuration file in /etc/nix/nix.conf in tests 😱 + 'NIX_CONF_DIR': '/var/empty', + # Prevent loading user configuration files in tests + 'NIX_USER_CONF_FILES': '', } libutil_test_support_sources = files( @@ -39,6 +43,8 @@ liblixutil_test_support = declare_dependency( ) libutil_tests_sources = files( + 'libutil/async-collect.cc', + 'libutil/async-semaphore.cc', 'libutil/canon-path.cc', 'libutil/checked-arithmetic.cc', 'libutil/chunked-vector.cc', @@ -76,6 +82,7 @@ libutil_tester = executable( liblixexpr_mstatic, liblixutil_test_support, nlohmann_json, + kj, ], cpp_pch : cpp_pch, ) @@ -252,7 +259,7 @@ test( env : default_test_env + { # No special meaning here, it's just a file laying around that is unlikely to go anywhere # any time soon. - '_NIX_TEST_UNIT_DATA': meson.project_source_root() / 'src/nix-env/buildenv.nix', + '_NIX_TEST_UNIT_DATA': meson.project_source_root() / 'src/legacy/buildenv.nix', # Use a temporary home directory for the unit tests. # Otherwise, /homeless-shelter is created in the single-user sandbox, and functional tests will fail. # TODO(alois31): handle TMPDIR properly (meson can't, and setting HOME in the test is too late)… @@ -262,9 +269,14 @@ test( protocol : 'gtest', ) +libmain_tests_sources = files( + 'libmain/crash.cc', + 'libmain/progress-bar.cc', +) + libmain_tester = executable( 'liblixmain-tests', - files('libmain/progress-bar.cc'), + libmain_tests_sources, dependencies : [ liblixmain, liblixexpr, |