diff options
30 files changed, 1358 insertions, 1149 deletions
diff --git a/.gitignore b/.gitignore index 0ae2c44a2..15c871218 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ buildtime.bin # Rust build files when using Cargo (not actually supported for building but it spews the files anyway) /target/ + +# Python compiled files from the test suite +*.pyc diff --git a/bench/bench.sh b/bench/bench.sh index 70acd4640..15d8af05a 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -1,4 +1,5 @@ -#!/usr/bin/env bash +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p bash -p hyperfine set -euo pipefail shopt -s inherit_errexit @@ -21,16 +22,21 @@ fi _exit="" trap "$_exit" EXIT +flake_args=("--extra-experimental-features" "nix-command flakes") + # XXX: yes this is very silly. flakes~!! -nix build --impure --expr '(builtins.getFlake "git+file:.").inputs.nixpkgs.outPath' -o bench/nixpkgs +nix build "${flake_args[@]}" --impure --expr '(builtins.getFlake "git+file:.").inputs.nixpkgs.outPath' -o bench/nixpkgs +# We must ignore the global config, or else NIX_PATH won't work reliably. +# See https://github.com/NixOS/nix/issues/9574 +export NIX_CONF_DIR='/var/empty' export NIX_REMOTE="$(mktemp -d)" _exit='rm -rfv "$NIX_REMOTE"; $_exit' export NIX_PATH="nixpkgs=bench/nixpkgs:nixos-config=bench/configuration.nix" builds=("$@") -flake_args="--extra-experimental-features 'nix-command flakes'" +flake_args="${flake_args[*]@Q}" hyperfineArgs=( --parameter-list BUILD "$(IFS=,; echo "${builds[*]}")" diff --git a/doc/manual/rl-next/download-protocols.md b/doc/manual/rl-next/download-protocols.md new file mode 100644 index 000000000..bf1bf79a3 --- /dev/null +++ b/doc/manual/rl-next/download-protocols.md @@ -0,0 +1,10 @@ +--- +synopsis: "transfers no longer allow arbitrary url schemas" +category: Breaking Changes +cls: [2106] +credits: horrors +--- + +Lix no longer allows transfers using arbitrary url schemas. Only `http://`, `https://`, `ftp://`, `ftps://`, and `file://` urls are supported going forward. This affects `builtins.fetchurl`, `<nix/fetchurl.nix>`, transfers to and from binary caches, and all other uses of the internal file transfer code. Flake inputs using multi-protocol schemas (e.g. `git+ssh`) are not affected as those use external utilities to transfer data. + +The `s3://` scheme is not affected at all by this change and continues to work if S3 support is built into Lix. diff --git a/flake.lock b/flake.lock index 84e578bc3..ac33089f1 100644 --- a/flake.lock +++ b/flake.lock @@ -19,11 +19,11 @@ "nix2container": { "flake": false, "locked": { - "lastModified": 1720642556, - "narHash": "sha256-qsnqk13UmREKmRT7c8hEnz26X3GFFyIQrqx4EaRc1Is=", + "lastModified": 1724996935, + "narHash": "sha256-njRK9vvZ1JJsP8oV2OgkBrpJhgQezI03S7gzskCcHos=", "owner": "nlewo", "repo": "nix2container", - "rev": "3853e5caf9ad24103b13aa6e0e8bcebb47649fe4", + "rev": "fa6bb0a1159f55d071ba99331355955ae30b3401", "type": "github" }, "original": { @@ -34,11 +34,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1721931987, - "narHash": "sha256-1Zg8LY0T5EfXtv0Kf4M6SFnjH7Eto4VV+EKJ/YSnhiI=", + "lastModified": 1727184566, + "narHash": "sha256-mgdK8BcFsLSNhe780+cHbEUbZ3OruLa1T/xgQlL4Aj4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e21630230c77140bc6478a21cd71e8bb73706fce", + "rev": "48c3030083c46042584531bc9d931020f1975677", "type": "github" }, "original": { @@ -67,11 +67,11 @@ "pre-commit-hooks": { "flake": false, "locked": { - "lastModified": 1721042469, - "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "lastModified": 1726745158, + "narHash": "sha256-D5AegvGoEjt4rkKedmxlSEmC+nNLMBPWFxvmYnVLhjk=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "rev": "4e743a6920eab45e8ba0fbe49dc459f1423a4b74", "type": "github" }, "original": { @@ -269,6 +269,8 @@ nix = pkgs.callPackage ./package.nix { inherit versionSuffix officialRelease buildUnreleasedNotes; inherit (pkgs) build-release-notes; + # Required since we don't support gcc stdenv + stdenv = pkgs.clangStdenv; internalApiDocs = true; busybox-sandbox-shell = pkgs.busybox-sandbox-shell; }; @@ -326,6 +328,8 @@ inherit (nixpkgs) pkgs; in pkgs.callPackage ./package.nix { + # Required since we don't support gcc stdenv + stdenv = pkgs.clangStdenv; versionSuffix = ""; lintInsteadOfBuild = true; }; diff --git a/misc/capnproto.nix b/misc/capnproto.nix new file mode 100644 index 000000000..0160050a0 --- /dev/null +++ b/misc/capnproto.nix @@ -0,0 +1,60 @@ +# FIXME: upstream to nixpkgs (do NOT build with gcc due to gcc coroutine bugs) +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + openssl, + zlib, +}: +assert stdenv.cc.isClang; +stdenv.mkDerivation rec { + pname = "capnproto"; + version = "1.0.2"; + + # release tarballs are missing some ekam rules + src = fetchFromGitHub { + owner = "capnproto"; + repo = "capnproto"; + rev = "v${version}"; + sha256 = "sha256-LVdkqVBTeh8JZ1McdVNtRcnFVwEJRNjt0JV2l7RkuO8="; + }; + + nativeBuildInputs = [ cmake ]; + propagatedBuildInputs = [ + openssl + zlib + ]; + + # FIXME: separate the binaries from the stuff that user systems actually use + # This runs into a terrible UX issue in Lix and I just don't want to debug it + # right now for the couple MB of closure size: + # https://git.lix.systems/lix-project/lix/issues/551 + # outputs = [ "bin" "dev" "out" ]; + + cmakeFlags = [ + (lib.cmakeBool "BUILD_SHARED_LIBS" true) + # Take optimization flags from CXXFLAGS rather than cmake injecting them + (lib.cmakeFeature "CMAKE_BUILD_TYPE" "None") + ]; + + env = { + # Required to build the coroutine library + CXXFLAGS = "-std=c++20"; + }; + + separateDebugInfo = true; + + meta = with lib; { + homepage = "https://capnproto.org/"; + description = "Cap'n Proto cerealization protocol"; + longDescription = '' + Cap’n Proto is an insanely fast data interchange format and + capability-based RPC system. Think JSON, except binary. Or think Protocol + Buffers, except faster. + ''; + license = licenses.mit; + platforms = platforms.all; + maintainers = lib.teams.lix.members; + }; +} diff --git a/package.nix b/package.nix index 2d485be93..39ecea714 100644 --- a/package.nix +++ b/package.nix @@ -16,7 +16,6 @@ bzip2, callPackage, capnproto-lix ? __forDefaults.capnproto-lix, - capnproto, cmake, curl, doxygen, @@ -32,6 +31,8 @@ lix-clang-tidy ? null, llvmPackages, lsof, + # FIXME: remove default after dropping NixOS 24.05 + lowdown-unsandboxed ? lowdown, lowdown, mdbook, mdbook-linkcheck, @@ -104,13 +105,14 @@ build-release-notes = callPackage ./maintainers/build-release-notes.nix { }; - # needs explicit c++20 to enable coroutine support - capnproto-lix = capnproto.overrideAttrs { CXXFLAGS = "-std=c++20"; }; + # needs derivation patching to add debuginfo and coroutine library support + # !! must build this with clang as it is affected by the gcc coroutine bugs + capnproto-lix = callPackage ./misc/capnproto.nix { inherit stdenv; }; }, }: # gcc miscompiles coroutines at least until 13.2, possibly longer -assert stdenv.cc.isClang || lintInsteadOfBuild || internalApiDocs; +assert stdenv.cc.isClang; let inherit (__forDefaults) canRunInstalled; @@ -253,7 +255,7 @@ stdenv.mkDerivation (finalAttrs: { capnproto-lix ] ++ [ - (lib.getBin lowdown) + (lib.getBin lowdown-unsandboxed) mdbook mdbook-linkcheck ] @@ -272,6 +274,10 @@ stdenv.mkDerivation (finalAttrs: { ++ lib.optionals lintInsteadOfBuild [ # required for a wrapped clang-tidy llvmPackages.clang-tools + # load-bearing order (just as below); the actual stdenv wrapped clang + # needs to precede the unwrapped clang in PATH such that calling `clang` + # can compile things. + stdenv.cc # required for run-clang-tidy llvmPackages.clang-unwrapped ]; @@ -443,6 +449,7 @@ stdenv.mkDerivation (finalAttrs: { editline-lix build-release-notes pegtl + capnproto-lix ; # The collection of dependency logic for this derivation is complicated enough that diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 4b659b71a..0c1a1ec0e 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -247,9 +247,24 @@ void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) co { bool first = true; str << "("; - for (auto & i : es) { - if (first) first = false; else str << " + "; - i.second->show(symbols, str); + for (auto & [_pos, part] : es) { + if (first) + first = false; + else + str << " + "; + + if (forceString && !dynamic_cast<ExprString *>(part.get())) { + /* Print as a string with an interpolation, to preserve the + * semantics of the value having to be a string. + * Interpolations are weird and someone should eventually + * move them out into their own AST node please. + */ + str << "\"${"; + part->show(symbols, str); + str << "}\""; + } else { + part->show(symbols, str); + } } str << ")"; } diff --git a/src/libexpr/parser/grammar.hh b/src/libexpr/parser/grammar.hh index 2c5a3d1be..701b40505 100644 --- a/src/libexpr/parser/grammar.hh +++ b/src/libexpr/parser/grammar.hh @@ -12,7 +12,7 @@ // eolf rules in favor of reproducing the old flex lexer as faithfully as // possible, and deferring calculation of positions to downstream users. -namespace nix::parser::grammar { +namespace nix::parser::grammar::v1 { using namespace tao::pegtl; namespace p = tao::pegtl; @@ -225,7 +225,8 @@ struct string : _string, seq< > {}; struct _ind_string { - template<bool Indented, typename... Inner> + struct line_start : semantic, star<one<' '>> {}; + template<typename... Inner> struct literal : semantic, seq<Inner...> {}; struct interpolation : semantic, seq< p::string<'$', '{'>, seps, @@ -233,34 +234,53 @@ struct _ind_string { must<one<'}'>> > {}; struct escape : semantic, must<any> {}; + /* Marker for non-empty lines */ + struct has_content : semantic, seq<> {}; }; struct ind_string : _ind_string, seq< TAO_PEGTL_STRING("''"), + // Strip first line completely if empty opt<star<one<' '>>, one<'\n'>>, - star< - sor< - _ind_string::literal< - true, + list< + seq< + // Start a line with some indentation + // (we always match even the empty string if no indentation, as this creates the line) + _ind_string::line_start, + // The actual line + opt< plus< sor< - not_one<'$', '\''>, - seq<one<'$'>, not_one<'{', '\''>>, - seq<one<'\''>, not_one<'\'', '$'>> - > - > - >, - _ind_string::interpolation, - _ind_string::literal<false, one<'$'>>, - _ind_string::literal<false, one<'\''>, not_at<one<'\''>>>, - seq<one<'\''>, _ind_string::literal<false, p::string<'\'', '\''>>>, - seq< - p::string<'\'', '\''>, - sor< - _ind_string::literal<false, one<'$'>>, - seq<one<'\\'>, _ind_string::escape> + _ind_string::literal< + plus< + sor< + not_one<'$', '\'', '\n'>, + // TODO probably factor this out like the others for performance + seq<one<'$'>, not_one<'{', '\'', '\n'>>, + seq<one<'$'>, at<one<'\n'>>>, + seq<one<'\''>, not_one<'\'', '$', '\n'>>, + seq<one<'\''>, at<one<'\n'>>> + > + > + >, + _ind_string::interpolation, + _ind_string::literal<one<'$'>>, + _ind_string::literal<one<'\''>, not_at<one<'\''>>>, + seq<one<'\''>, _ind_string::literal<p::string<'\'', '\''>>>, + seq< + p::string<'\'', '\''>, + sor< + _ind_string::literal<one<'$'>>, + seq<one<'\\'>, _ind_string::escape> + > + > + >, + _ind_string::has_content > > - > + >, + // End of line, LF. CR is just ignored and not treated as ending a line + // (for the purpose of indentation stripping) + _ind_string::literal<one<'\n'>> >, must<TAO_PEGTL_STRING("''")> > {}; @@ -352,10 +372,10 @@ struct formals : semantic, _formals, seq< struct _attr { struct simple : semantic, sor<t::identifier, t::kw_or> {}; - struct string : semantic, seq<grammar::string> {}; + struct string : semantic, seq<grammar::v1::string> {}; struct expr : semantic, seq< TAO_PEGTL_STRING("${"), seps, - must<grammar::expr>, seps, + must<grammar::v1::expr>, seps, must<one<'}'>> > {}; }; @@ -452,9 +472,9 @@ struct _expr { struct id : semantic, t::identifier {}; struct int_ : semantic, t::integer {}; struct float_ : semantic, t::floating {}; - struct string : semantic, seq<grammar::string> {}; - struct ind_string : semantic, seq<grammar::ind_string> {}; - struct path : semantic, seq<grammar::path> {}; + struct string : semantic, seq<grammar::v1::string> {}; + struct ind_string : semantic, seq<grammar::v1::ind_string> {}; + struct path : semantic, seq<grammar::v1::path> {}; struct uri : semantic, t::uri {}; struct ancient_let : semantic, _attrset<must, t::kw_let, seps> {}; struct rec_set : semantic, _attrset<must, t::kw_rec, seps> {}; @@ -628,34 +648,34 @@ struct nothing : p::nothing<Rule> { template<typename Self, typename OpCtx, typename AttrPathT, typename ExprT> struct operator_semantics { - struct has_attr : grammar::op::has_attr { + struct has_attr : grammar::v1::op::has_attr { AttrPathT path; }; struct OpEntry { OpCtx ctx; uint8_t prec; - grammar::op::kind assoc; + grammar::v1::op::kind assoc; std::variant< - grammar::op::not_, - grammar::op::unary_minus, - grammar::op::implies, - grammar::op::or_, - grammar::op::and_, - grammar::op::equals, - grammar::op::not_equals, - grammar::op::less_eq, - grammar::op::greater_eq, - grammar::op::update, - grammar::op::concat, - grammar::op::less, - grammar::op::greater, - grammar::op::plus, - grammar::op::minus, - grammar::op::mul, - grammar::op::div, - grammar::op::pipe_right, - grammar::op::pipe_left, + grammar::v1::op::not_, + grammar::v1::op::unary_minus, + grammar::v1::op::implies, + grammar::v1::op::or_, + grammar::v1::op::and_, + grammar::v1::op::equals, + grammar::v1::op::not_equals, + grammar::v1::op::less_eq, + grammar::v1::op::greater_eq, + grammar::v1::op::update, + grammar::v1::op::concat, + grammar::v1::op::less, + grammar::v1::op::greater, + grammar::v1::op::plus, + grammar::v1::op::minus, + grammar::v1::op::mul, + grammar::v1::op::div, + grammar::v1::op::pipe_right, + grammar::v1::op::pipe_left, has_attr > op; }; @@ -676,7 +696,7 @@ struct operator_semantics { auto & [ctx, precedence, kind, op] = ops.back(); // NOTE this relies on associativity not being mixed within a precedence level. if ((precedence > toPrecedence) - || (kind != grammar::op::kind::leftAssoc && precedence == toPrecedence)) + || (kind != grammar::v1::op::kind::leftAssoc && precedence == toPrecedence)) break; std::visit([&, ctx=std::move(ctx)] (auto & op) { exprs.push_back(static_cast<Self &>(*this).applyOp(ctx, op, args...)); @@ -694,9 +714,9 @@ struct operator_semantics { void pushOp(OpCtx ctx, auto o, auto &... args) { - if (o.kind != grammar::op::kind::unary) + if (o.kind != grammar::v1::op::kind::unary) reduce(o.precedence, args...); - if (!ops.empty() && o.kind == grammar::op::kind::nonAssoc) { + if (!ops.empty() && o.kind == grammar::v1::op::kind::nonAssoc) { auto & [_pos, _prec, _kind, _o] = ops.back(); if (_kind == o.kind && _prec == o.precedence) Self::badOperator(ctx, args...); diff --git a/src/libexpr/parser/parser-impl1.inc.cc b/src/libexpr/parser/parser-impl1.inc.cc new file mode 100644 index 000000000..c937d17fd --- /dev/null +++ b/src/libexpr/parser/parser-impl1.inc.cc @@ -0,0 +1,863 @@ +// flip this define when doing parser development to enable some g checks. +#if 0 +#include <tao/pegtl/contrib/analyze.hpp> +#define ANALYZE_GRAMMAR \ + ([] { \ + const std::size_t issues = tao::pegtl::analyze<grammar::v1::root>(); \ + assert(issues == 0); \ + })() +#else +#define ANALYZE_GRAMMAR ((void) 0) +#endif + +namespace p = tao::pegtl; + +namespace nix::parser::v1 { +namespace { + +template<typename> +inline constexpr const char * error_message = nullptr; + +#define error_message_for(...) \ + template<> inline constexpr auto error_message<__VA_ARGS__> + +error_message_for(p::one<'{'>) = "expecting '{'"; +error_message_for(p::one<'}'>) = "expecting '}'"; +error_message_for(p::one<'"'>) = "expecting '\"'"; +error_message_for(p::one<';'>) = "expecting ';'"; +error_message_for(p::one<')'>) = "expecting ')'"; +error_message_for(p::one<']'>) = "expecting ']'"; +error_message_for(p::one<':'>) = "expecting ':'"; +error_message_for(p::string<'\'', '\''>) = "expecting \"''\""; +error_message_for(p::any) = "expecting any character"; +error_message_for(grammar::v1::eof) = "expecting end of file"; +error_message_for(grammar::v1::seps) = "expecting separators"; +error_message_for(grammar::v1::path::forbid_prefix_triple_slash) = "too many slashes in path"; +error_message_for(grammar::v1::path::forbid_prefix_double_slash_no_interp) = "path has a trailing slash"; +error_message_for(grammar::v1::expr) = "expecting expression"; +error_message_for(grammar::v1::expr::unary) = "expecting expression"; +error_message_for(grammar::v1::binding::equal) = "expecting '='"; +error_message_for(grammar::v1::expr::lambda::arg) = "expecting identifier"; +error_message_for(grammar::v1::formals) = "expecting formals"; +error_message_for(grammar::v1::attrpath) = "expecting attribute path"; +error_message_for(grammar::v1::expr::select) = "expecting selection expression"; +error_message_for(grammar::v1::t::kw_then) = "expecting 'then'"; +error_message_for(grammar::v1::t::kw_else) = "expecting 'else'"; +error_message_for(grammar::v1::t::kw_in) = "expecting 'in'"; + +struct SyntaxErrors +{ + template<typename Rule> + static constexpr auto message = error_message<Rule>; + + template<typename Rule> + static constexpr bool raise_on_failure = false; +}; + +template<typename Rule> +struct Control : p::must_if<SyntaxErrors>::control<Rule> +{ + template<typename ParseInput, typename... States> + [[noreturn]] static void raise(const ParseInput & in, States &&... st) + { + if (in.empty()) { + std::string expected; + if constexpr (constexpr auto msg = error_message<Rule>) + expected = fmt(", %s", msg); + throw p::parse_error("unexpected end of file" + expected, in); + } + p::must_if<SyntaxErrors>::control<Rule>::raise(in, st...); + } +}; + +struct ExprState + : grammar::v1:: + operator_semantics<ExprState, PosIdx, AttrPath, std::pair<PosIdx, std::unique_ptr<Expr>>> +{ + std::unique_ptr<Expr> popExprOnly() { + return std::move(popExpr().second); + } + + template<typename Op, typename... Args> + std::unique_ptr<Expr> applyUnary(Args &&... args) { + return std::make_unique<Op>(popExprOnly(), std::forward<Args>(args)...); + } + + template<typename Op> + std::unique_ptr<Expr> applyBinary(PosIdx pos) { + auto right = popExprOnly(), left = popExprOnly(); + return std::make_unique<Op>(pos, std::move(left), std::move(right)); + } + + std::unique_ptr<Expr> call(PosIdx pos, Symbol fn, bool flip = false) + { + std::vector<std::unique_ptr<Expr>> args(2); + args[flip ? 0 : 1] = popExprOnly(); + args[flip ? 1 : 0] = popExprOnly(); + return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args)); + } + + std::unique_ptr<Expr> pipe(PosIdx pos, State & state, bool flip = false) + { + if (!state.featureSettings.isEnabled(Xp::PipeOperator)) + throw ParseError({ + .msg = HintFmt("Pipe operator is disabled"), + .pos = state.positions[pos] + }); + + // Reverse the order compared to normal function application: arg |> fn + std::unique_ptr<Expr> fn, arg; + if (flip) { + fn = popExprOnly(); + arg = popExprOnly(); + } else { + arg = popExprOnly(); + fn = popExprOnly(); + } + std::vector<std::unique_ptr<Expr>> args{1}; + args[0] = std::move(arg); + + return std::make_unique<ExprCall>(pos, std::move(fn), std::move(args)); + } + + std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state) + { + return call(pos, state.s.lessThan, !less); + } + + std::unique_ptr<Expr> concatStrings(PosIdx pos) + { + std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> args(2); + args[1] = popExpr(); + args[0] = popExpr(); + return std::make_unique<ExprConcatStrings>(pos, false, std::move(args)); + } + + std::unique_ptr<Expr> negate(PosIdx pos, State & state) + { + std::vector<std::unique_ptr<Expr>> args(2); + args[0] = std::make_unique<ExprInt>(0); + args[1] = popExprOnly(); + return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(state.s.sub), std::move(args)); + } + + std::pair<PosIdx, std::unique_ptr<Expr>> applyOp(PosIdx pos, auto & op, State & state) { + using Op = grammar::v1::op; + + auto not_ = [] (auto e) { + return std::make_unique<ExprOpNot>(std::move(e)); + }; + + return { + pos, + (overloaded { + [&] (Op::implies) { return applyBinary<ExprOpImpl>(pos); }, + [&] (Op::or_) { return applyBinary<ExprOpOr>(pos); }, + [&] (Op::and_) { return applyBinary<ExprOpAnd>(pos); }, + [&] (Op::equals) { return applyBinary<ExprOpEq>(pos); }, + [&] (Op::not_equals) { return applyBinary<ExprOpNEq>(pos); }, + [&] (Op::less) { return order(pos, true, state); }, + [&] (Op::greater_eq) { return not_(order(pos, true, state)); }, + [&] (Op::greater) { return order(pos, false, state); }, + [&] (Op::less_eq) { return not_(order(pos, false, state)); }, + [&] (Op::update) { return applyBinary<ExprOpUpdate>(pos); }, + [&] (Op::not_) { return applyUnary<ExprOpNot>(); }, + [&] (Op::plus) { return concatStrings(pos); }, + [&] (Op::minus) { return call(pos, state.s.sub); }, + [&] (Op::mul) { return call(pos, state.s.mul); }, + [&] (Op::div) { return call(pos, state.s.div); }, + [&] (Op::concat) { return applyBinary<ExprOpConcatLists>(pos); }, + [&] (has_attr & a) { return applyUnary<ExprOpHasAttr>(std::move(a.path)); }, + [&] (Op::unary_minus) { return negate(pos, state); }, + [&] (Op::pipe_right) { return pipe(pos, state, true); }, + [&] (Op::pipe_left) { return pipe(pos, state); }, + })(op) + }; + } + + // always_inline is needed, otherwise pushOp slows down considerably + [[noreturn, gnu::always_inline]] + static void badOperator(PosIdx pos, State & state) + { + throw ParseError({ + .msg = HintFmt("syntax error, unexpected operator"), + .pos = state.positions[pos] + }); + } + + template<typename Expr, typename... Args> + Expr & pushExpr(PosIdx pos, Args && ... args) + { + auto p = std::make_unique<Expr>(std::forward<Args>(args)...); + auto & result = *p; + exprs.emplace_back(pos, std::move(p)); + return result; + } +}; + +struct SubexprState { +private: + ExprState * up; + +public: + explicit SubexprState(ExprState & up, auto &...) : up(&up) {} + operator ExprState &() { return *up; } + ExprState * operator->() { return up; } +}; + + + +template<typename Rule> +struct BuildAST : grammar::v1::nothing<Rule> {}; + +struct LambdaState : SubexprState { + using SubexprState::SubexprState; + + Symbol arg; + std::unique_ptr<Formals> formals; +}; + +struct FormalsState : SubexprState { + using SubexprState::SubexprState; + + Formals formals{}; + Formal formal{}; +}; + +template<> struct BuildAST<grammar::v1::formal::name> { + static void apply(const auto & in, FormalsState & s, State & ps) { + s.formal = { + .pos = ps.at(in), + .name = ps.symbols.create(in.string_view()), + }; + } +}; + +template<> struct BuildAST<grammar::v1::formal> { + static void apply0(FormalsState & s, State &) { + s.formals.formals.emplace_back(std::move(s.formal)); + } +}; + +template<> struct BuildAST<grammar::v1::formal::default_value> { + static void apply0(FormalsState & s, State & ps) { + s.formal.def = s->popExprOnly(); + } +}; + +template<> struct BuildAST<grammar::v1::formals::ellipsis> { + static void apply0(FormalsState & s, State &) { + s.formals.ellipsis = true; + } +}; + +template<> struct BuildAST<grammar::v1::formals> : change_head<FormalsState> { + static void success0(FormalsState & f, LambdaState & s, State &) { + s.formals = std::make_unique<Formals>(std::move(f.formals)); + } +}; + +struct AttrState : SubexprState { + using SubexprState::SubexprState; + + std::vector<AttrName> attrs; + + template <typename T> + void pushAttr(T && attr, PosIdx) { attrs.emplace_back(std::forward<T>(attr)); } +}; + +template<> struct BuildAST<grammar::v1::attr::simple> { + static void apply(const auto & in, auto & s, State & ps) { + s.pushAttr(ps.symbols.create(in.string_view()), ps.at(in)); + } +}; + +template<> struct BuildAST<grammar::v1::attr::string> { + static void apply(const auto & in, auto & s, State & ps) { + auto e = s->popExprOnly(); + if (auto str = dynamic_cast<ExprString *>(e.get())) + s.pushAttr(ps.symbols.create(str->s), ps.at(in)); + else + s.pushAttr(std::move(e), ps.at(in)); + } +}; + +template<> struct BuildAST<grammar::v1::attr::expr> : BuildAST<grammar::v1::attr::string> {}; + +struct BindingsState : SubexprState { + using SubexprState::SubexprState; + + ExprAttrs attrs; + AttrPath path; + std::unique_ptr<Expr> value; +}; + +struct InheritState : SubexprState { + using SubexprState::SubexprState; + + std::vector<std::pair<AttrName, PosIdx>> attrs; + std::unique_ptr<Expr> from; + PosIdx fromPos; + + template <typename T> + void pushAttr(T && attr, PosIdx pos) { attrs.emplace_back(std::forward<T>(attr), pos); } +}; + +template<> struct BuildAST<grammar::v1::inherit::from> { + static void apply(const auto & in, InheritState & s, State & ps) { + s.from = s->popExprOnly(); + s.fromPos = ps.at(in); + } +}; + +template<> struct BuildAST<grammar::v1::inherit> : change_head<InheritState> { + static void success0(InheritState & s, BindingsState & b, State & ps) { + auto & attrs = b.attrs.attrs; + // TODO this should not reuse generic attrpath rules. + for (auto & [i, iPos] : s.attrs) { + if (i.symbol) + continue; + if (auto str = dynamic_cast<ExprString *>(i.expr.get())) + i = AttrName(ps.symbols.create(str->s)); + else { + throw ParseError({ + .msg = HintFmt("dynamic attributes not allowed in inherit"), + .pos = ps.positions[iPos] + }); + } + } + if (s.from != nullptr) { + if (!b.attrs.inheritFromExprs) + b.attrs.inheritFromExprs = std::make_unique<std::vector<ref<Expr>>>(); + auto fromExpr = ref<Expr>(std::move(s.from)); + b.attrs.inheritFromExprs->push_back(fromExpr); + for (auto & [i, iPos] : s.attrs) { + if (attrs.find(i.symbol) != attrs.end()) + ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); + auto inheritFrom = std::make_unique<ExprInheritFrom>( + s.fromPos, + b.attrs.inheritFromExprs->size() - 1, + fromExpr + ); + attrs.emplace( + i.symbol, + ExprAttrs::AttrDef( + std::make_unique<ExprSelect>(iPos, std::move(inheritFrom), i.symbol), + iPos, + ExprAttrs::AttrDef::Kind::InheritedFrom)); + } + } else { + for (auto & [i, iPos] : s.attrs) { + if (attrs.find(i.symbol) != attrs.end()) + ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); + attrs.emplace( + i.symbol, + ExprAttrs::AttrDef( + std::make_unique<ExprVar>(iPos, i.symbol), + iPos, + ExprAttrs::AttrDef::Kind::Inherited)); + } + } + } +}; + +template<> struct BuildAST<grammar::v1::binding::path> : change_head<AttrState> { + static void success0(AttrState & a, BindingsState & s, State & ps) { + s.path = std::move(a.attrs); + } +}; + +template<> struct BuildAST<grammar::v1::binding::value> { + static void apply0(BindingsState & s, State & ps) { + s.value = s->popExprOnly(); + } +}; + +template<> struct BuildAST<grammar::v1::binding> { + static void apply(const auto & in, BindingsState & s, State & ps) { + ps.addAttr(&s.attrs, std::move(s.path), std::move(s.value), ps.at(in)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::id> { + static void apply(const auto & in, ExprState & s, State & ps) { + if (in.string_view() == "__curPos") + s.pushExpr<ExprPos>(ps.at(in), ps.at(in)); + else + s.pushExpr<ExprVar>(ps.at(in), ps.at(in), ps.symbols.create(in.string_view())); + } +}; + +template<> struct BuildAST<grammar::v1::expr::int_> { + static void apply(const auto & in, ExprState & s, State & ps) { + int64_t v; + if (std::from_chars(in.begin(), in.end(), v).ec != std::errc{}) { + throw ParseError({ + .msg = HintFmt("invalid integer '%1%'", in.string_view()), + .pos = ps.positions[ps.at(in)], + }); + } + s.pushExpr<ExprInt>(noPos, v); + } +}; + +template<> struct BuildAST<grammar::v1::expr::float_> { + static void apply(const auto & in, ExprState & s, State & ps) { + // copy the input into a temporary string so we can call stod. + // can't use from_chars because libc++ (thus darwin) does not have it, + // and floats are not performance-sensitive anyway. if they were you'd + // be in much bigger trouble than this. + // + // we also get to do a locale-save dance because stod is locale-aware and + // something (a plugin?) may have called setlocale or uselocale. + static struct locale_hack { + locale_t posix; + locale_hack(): posix(newlocale(LC_ALL_MASK, "POSIX", 0)) + { + if (posix == 0) + throw SysError("could not get POSIX locale"); + } + } locale; + + auto tmp = in.string(); + double v = [&] { + auto oldLocale = uselocale(locale.posix); + Finally resetLocale([=] { uselocale(oldLocale); }); + try { + return std::stod(tmp); + } catch (...) { + throw ParseError({ + .msg = HintFmt("invalid float '%1%'", in.string_view()), + .pos = ps.positions[ps.at(in)], + }); + } + }(); + s.pushExpr<ExprFloat>(noPos, v); + } +}; + +struct StringState : SubexprState { + using SubexprState::SubexprState; + + std::string currentLiteral; + PosIdx currentPos; + std::vector<std::pair<nix::PosIdx, std::unique_ptr<Expr>>> parts; + + void append(PosIdx pos, std::string_view s) + { + if (currentLiteral.empty()) + currentPos = pos; + currentLiteral += s; + } + + // FIXME this truncates strings on NUL for compat with the old parser. ideally + // we should use the decomposition the g gives us instead of iterating over + // the entire string again. + static void unescapeStr(std::string & str) + { + char * s = str.data(); + char * t = s; + char c; + while ((c = *s++)) { + if (c == '\\') { + c = *s++; + if (c == 'n') *t = '\n'; + else if (c == 'r') *t = '\r'; + else if (c == 't') *t = '\t'; + else *t = c; + } + else if (c == '\r') { + /* Normalise CR and CR/LF into LF. */ + *t = '\n'; + if (*s == '\n') s++; /* cr/lf */ + } + else *t = c; + t++; + } + str.resize(t - str.data()); + } + + void endLiteral() + { + if (!currentLiteral.empty()) { + unescapeStr(currentLiteral); + parts.emplace_back(currentPos, std::make_unique<ExprString>(std::move(currentLiteral))); + } + } + + std::unique_ptr<Expr> finish() + { + if (parts.empty()) { + unescapeStr(currentLiteral); + return std::make_unique<ExprString>(std::move(currentLiteral)); + } else { + endLiteral(); + auto pos = parts[0].first; + return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts)); + } + } +}; + +template<typename... Content> struct BuildAST<grammar::v1::string::literal<Content...>> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::string::cr_lf> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), in.string_view()); // FIXME compat with old parser + } +}; + +template<> struct BuildAST<grammar::v1::string::interpolation> { + static void apply(const auto & in, StringState & s, State & ps) { + s.endLiteral(); + s.parts.emplace_back(ps.at(in), s->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::string::escape> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), "\\"); // FIXME compat with old parser + s.append(ps.at(in), in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::string> : change_head<StringState> { + static void success0(StringState & s, ExprState & e, State &) { + e.exprs.emplace_back(noPos, s.finish()); + } +}; + +struct IndStringState : SubexprState { + using SubexprState::SubexprState; + + std::vector<IndStringLine> lines; +}; + +template<> struct BuildAST<grammar::v1::ind_string::line_start> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.push_back(IndStringLine { in.string_view(), ps.at(in) }); + } +}; + +template<typename... Content> +struct BuildAST<grammar::v1::ind_string::literal<Content...>> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.back().parts.emplace_back(ps.at(in), in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::ind_string::interpolation> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.back().parts.emplace_back(ps.at(in), s->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::ind_string::escape> { + static void apply(const auto & in, IndStringState & s, State & ps) { + switch (*in.begin()) { + case 'n': s.lines.back().parts.emplace_back(ps.at(in), "\n"); break; + case 'r': s.lines.back().parts.emplace_back(ps.at(in), "\r"); break; + case 't': s.lines.back().parts.emplace_back(ps.at(in), "\t"); break; + default: s.lines.back().parts.emplace_back(ps.at(in), in.string_view()); break; + } + } +}; + +template<> struct BuildAST<grammar::v1::ind_string::has_content> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.back().hasContent = true; + } +}; + +template<> struct BuildAST<grammar::v1::ind_string> : change_head<IndStringState> { + static void success(const auto & in, IndStringState & s, ExprState & e, State & ps) { + e.exprs.emplace_back(noPos, ps.stripIndentation(ps.at(in), std::move(s.lines))); + } +}; + +template<typename... Content> struct BuildAST<grammar::v1::path::literal<Content...>> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), in.string_view()); + s.endLiteral(); + } +}; + +template<> struct BuildAST<grammar::v1::path::interpolation> : BuildAST<grammar::v1::string::interpolation> {}; + +template<> struct BuildAST<grammar::v1::path::anchor> { + static void apply(const auto & in, StringState & s, State & ps) { + Path path(absPath(in.string(), ps.basePath.path.abs())); + /* add back in the trailing '/' to the first segment */ + if (in.string_view().ends_with('/') && in.size() > 1) + path += "/"; + s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); + } +}; + +template<> struct BuildAST<grammar::v1::path::home_anchor> { + static void apply(const auto & in, StringState & s, State & ps) { + if (evalSettings.pureEval) + throw Error("the path '%s' can not be resolved in pure mode", in.string_view()); + Path path(getHome() + in.string_view().substr(1)); + s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); + } +}; + +template<> struct BuildAST<grammar::v1::path::searched_path> { + static void apply(const auto & in, StringState & s, State & ps) { + std::vector<std::unique_ptr<Expr>> args{2}; + args[0] = std::make_unique<ExprVar>(ps.s.nixPath); + args[1] = std::make_unique<ExprString>(in.string()); + s.parts.emplace_back( + ps.at(in), + std::make_unique<ExprCall>( + ps.at(in), + std::make_unique<ExprVar>(ps.s.findFile), + std::move(args))); + } +}; + +template<> struct BuildAST<grammar::v1::path> : change_head<StringState> { + template<typename E> + static void check_slash(PosIdx end, StringState & s, State & ps) { + auto e = dynamic_cast<E *>(s.parts.back().second.get()); + if (!e || !e->s.ends_with('/')) + return; + if (s.parts.size() > 1 || e->s != "/") + throw ParseError({ + .msg = HintFmt("path has a trailing slash"), + .pos = ps.positions[end], + }); + } + + static void success(const auto & in, StringState & s, ExprState & e, State & ps) { + s.endLiteral(); + check_slash<ExprPath>(ps.atEnd(in), s, ps); + check_slash<ExprString>(ps.atEnd(in), s, ps); + if (s.parts.size() == 1) { + e.exprs.emplace_back(noPos, std::move(s.parts.back().second)); + } else { + e.pushExpr<ExprConcatStrings>(ps.at(in), ps.at(in), false, std::move(s.parts)); + } + } +}; + +// strings and paths sare handled fully by the grammar-level rule for now +template<> struct BuildAST<grammar::v1::expr::string> : p::maybe_nothing {}; +template<> struct BuildAST<grammar::v1::expr::ind_string> : p::maybe_nothing {}; +template<> struct BuildAST<grammar::v1::expr::path> : p::maybe_nothing {}; + +template<> struct BuildAST<grammar::v1::expr::uri> { + static void apply(const auto & in, ExprState & s, State & ps) { + bool URLLiterals = ps.featureSettings.isEnabled(Dep::UrlLiterals); + if (!URLLiterals) + throw ParseError({ + .msg = HintFmt("URL literals are deprecated, allow using them with %s", "--extra-deprecated-features url-literals"), + .pos = ps.positions[ps.at(in)] + }); + s.pushExpr<ExprString>(ps.at(in), in.string()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::ancient_let> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + // Added 2024-09-18. Turn into an error at some point in the future. + // See the documentation on deprecated features for more details. + if (!ps.featureSettings.isEnabled(Dep::AncientLet)) + warn( + "%s found at %s. This feature is deprecated and will be removed in the future. Use %s to silence this warning.", + "let {", + ps.positions[ps.at(in)], + "--extra-deprecated-features ancient-let" + ); + + b.attrs.pos = ps.at(in); + b.attrs.recursive = true; + s.pushExpr<ExprSelect>(b.attrs.pos, b.attrs.pos, std::make_unique<ExprAttrs>(std::move(b.attrs)), ps.s.body); + } +}; + +template<> struct BuildAST<grammar::v1::expr::rec_set> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + // Before inserting new attrs, check for __override and throw an error + // (the error will initially be a warning to ease migration) + if (!featureSettings.isEnabled(Dep::RecSetOverrides) && b.attrs.attrs.contains(ps.s.overrides)) { + ps.overridesFound(ps.at(in)); + } + + b.attrs.pos = ps.at(in); + b.attrs.recursive = true; + s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::set> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + b.attrs.pos = ps.at(in); + s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); + } +}; + +using ListState = std::vector<std::unique_ptr<Expr>>; + +template<> struct BuildAST<grammar::v1::expr::list> : change_head<ListState> { + static void success(const auto & in, ListState & ls, ExprState & s, State & ps) { + auto e = std::make_unique<ExprList>(); + e->elems = std::move(ls); + s.exprs.emplace_back(ps.at(in), std::move(e)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::list::entry> : change_head<ExprState> { + static void success0(ExprState & e, ListState & s, State & ps) { + s.emplace_back(e.finish(ps).second); + } +}; + +struct SelectState : SubexprState { + using SubexprState::SubexprState; + + PosIdx pos; + ExprSelect * e = nullptr; +}; + +template<> struct BuildAST<grammar::v1::expr::select::head> { + static void apply(const auto & in, SelectState & s, State & ps) { + s.pos = ps.at(in); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select::attr> : change_head<AttrState> { + static void success0(AttrState & a, SelectState & s, State &) { + s.e = &s->pushExpr<ExprSelect>(s.pos, s.pos, s->popExprOnly(), std::move(a.attrs), nullptr); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select::attr_or> { + static void apply0(SelectState & s, State &) { + s.e->def = s->popExprOnly(); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select::as_app_or> { + static void apply(const auto & in, SelectState & s, State & ps) { + std::vector<std::unique_ptr<Expr>> args(1); + args[0] = std::make_unique<ExprVar>(ps.at(in), ps.s.or_); + s->pushExpr<ExprCall>(s.pos, s.pos, s->popExprOnly(), std::move(args)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select> : change_head<SelectState> { + static void success0(const auto &...) {} +}; + +struct AppState : SubexprState { + using SubexprState::SubexprState; + + PosIdx pos; + ExprCall * e = nullptr; +}; + +template<> struct BuildAST<grammar::v1::expr::app::select_or_fn> { + static void apply(const auto & in, AppState & s, State & ps) { + s.pos = ps.at(in); + } +}; + +template<> struct BuildAST<grammar::v1::expr::app::first_arg> { + static void apply(auto & in, AppState & s, State & ps) { + auto arg = s->popExprOnly(), fn = s->popExprOnly(); + if ((s.e = dynamic_cast<ExprCall *>(fn.get()))) { + // TODO remove. + // AST compat with old parser, semantics are the same. + // this can happen on occasions such as `<p> <p>` or `a or b or`, + // neither of which are super worth optimizing. + s.e->args.push_back(std::move(arg)); + s->exprs.emplace_back(noPos, std::move(fn)); + } else { + std::vector<std::unique_ptr<Expr>> args{1}; + args[0] = std::move(arg); + s.e = &s->pushExpr<ExprCall>(s.pos, s.pos, std::move(fn), std::move(args)); + } + } +}; + +template<> struct BuildAST<grammar::v1::expr::app::another_arg> { + static void apply0(AppState & s, State & ps) { + s.e->args.push_back(s->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::app> : change_head<AppState> { + static void success0(const auto &...) {} +}; + +template<typename Op> struct BuildAST<grammar::v1::expr::operator_<Op>> { + static void apply(const auto & in, ExprState & s, State & ps) { + s.pushOp(ps.at(in), Op{}, ps); + } +}; +template<> struct BuildAST<grammar::v1::expr::operator_<grammar::v1::op::has_attr>> : change_head<AttrState> { + static void success(const auto & in, AttrState & a, ExprState & s, State & ps) { + s.pushOp(ps.at(in), ExprState::has_attr{{}, std::move(a.attrs)}, ps); + } +}; + +template<> struct BuildAST<grammar::v1::expr::lambda::arg> { + static void apply(const auto & in, LambdaState & s, State & ps) { + s.arg = ps.symbols.create(in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::lambda> : change_head<LambdaState> { + static void success(const auto & in, LambdaState & l, ExprState & s, State & ps) { + if (l.formals) + l.formals = ps.validateFormals(std::move(l.formals), ps.at(in), l.arg); + s.pushExpr<ExprLambda>(ps.at(in), ps.at(in), l.arg, std::move(l.formals), l->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::assert_> { + static void apply(const auto & in, ExprState & s, State & ps) { + auto body = s.popExprOnly(), cond = s.popExprOnly(); + s.pushExpr<ExprAssert>(ps.at(in), ps.at(in), std::move(cond), std::move(body)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::with> { + static void apply(const auto & in, ExprState & s, State & ps) { + auto body = s.popExprOnly(), scope = s.popExprOnly(); + s.pushExpr<ExprWith>(ps.at(in), ps.at(in), std::move(scope), std::move(body)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::let> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + if (!b.attrs.dynamicAttrs.empty()) + throw ParseError({ + .msg = HintFmt("dynamic attributes not allowed in let"), + .pos = ps.positions[ps.at(in)] + }); + + s.pushExpr<ExprLet>(ps.at(in), std::make_unique<ExprAttrs>(std::move(b.attrs)), b->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::if_> { + static void apply(const auto & in, ExprState & s, State & ps) { + auto else_ = s.popExprOnly(), then = s.popExprOnly(), cond = s.popExprOnly(); + s.pushExpr<ExprIf>(ps.at(in), ps.at(in), std::move(cond), std::move(then), std::move(else_)); + } +}; + +template<> struct BuildAST<grammar::v1::expr> : change_head<ExprState> { + static void success0(ExprState & inner, ExprState & outer, State & ps) { + outer.exprs.push_back(inner.finish(ps)); + } +}; + +} +} diff --git a/src/libexpr/parser/parser.cc b/src/libexpr/parser/parser.cc index 17463056c..896a54981 100644 --- a/src/libexpr/parser/parser.cc +++ b/src/libexpr/parser/parser.cc @@ -14,857 +14,11 @@ #include <charconv> #include <memory> -// flip this define when doing parser development to enable some g checks. -#if 0 -#include <tao/pegtl/contrib/analyze.hpp> -#define ANALYZE_GRAMMAR \ - ([] { \ - const std::size_t issues = tao::pegtl::analyze<grammar::root>(); \ - assert(issues == 0); \ - })() -#else -#define ANALYZE_GRAMMAR ((void) 0) -#endif - -namespace p = tao::pegtl; - -namespace nix::parser { -namespace { - -template<typename> -inline constexpr const char * error_message = nullptr; - -#define error_message_for(...) \ - template<> inline constexpr auto error_message<__VA_ARGS__> - -error_message_for(p::one<'{'>) = "expecting '{'"; -error_message_for(p::one<'}'>) = "expecting '}'"; -error_message_for(p::one<'"'>) = "expecting '\"'"; -error_message_for(p::one<';'>) = "expecting ';'"; -error_message_for(p::one<')'>) = "expecting ')'"; -error_message_for(p::one<']'>) = "expecting ']'"; -error_message_for(p::one<':'>) = "expecting ':'"; -error_message_for(p::string<'\'', '\''>) = "expecting \"''\""; -error_message_for(p::any) = "expecting any character"; -error_message_for(grammar::eof) = "expecting end of file"; -error_message_for(grammar::seps) = "expecting separators"; -error_message_for(grammar::path::forbid_prefix_triple_slash) = "too many slashes in path"; -error_message_for(grammar::path::forbid_prefix_double_slash_no_interp) = "path has a trailing slash"; -error_message_for(grammar::expr) = "expecting expression"; -error_message_for(grammar::expr::unary) = "expecting expression"; -error_message_for(grammar::binding::equal) = "expecting '='"; -error_message_for(grammar::expr::lambda::arg) = "expecting identifier"; -error_message_for(grammar::formals) = "expecting formals"; -error_message_for(grammar::attrpath) = "expecting attribute path"; -error_message_for(grammar::expr::select) = "expecting selection expression"; -error_message_for(grammar::t::kw_then) = "expecting 'then'"; -error_message_for(grammar::t::kw_else) = "expecting 'else'"; -error_message_for(grammar::t::kw_in) = "expecting 'in'"; - -struct SyntaxErrors -{ - template<typename Rule> - static constexpr auto message = error_message<Rule>; - - template<typename Rule> - static constexpr bool raise_on_failure = false; -}; - -template<typename Rule> -struct Control : p::must_if<SyntaxErrors>::control<Rule> -{ - template<typename ParseInput, typename... States> - [[noreturn]] static void raise(const ParseInput & in, States &&... st) - { - if (in.empty()) { - std::string expected; - if constexpr (constexpr auto msg = error_message<Rule>) - expected = fmt(", %s", msg); - throw p::parse_error("unexpected end of file" + expected, in); - } - p::must_if<SyntaxErrors>::control<Rule>::raise(in, st...); - } -}; - -struct ExprState - : grammar:: - operator_semantics<ExprState, PosIdx, AttrPath, std::pair<PosIdx, std::unique_ptr<Expr>>> -{ - std::unique_ptr<Expr> popExprOnly() { - return std::move(popExpr().second); - } - - template<typename Op, typename... Args> - std::unique_ptr<Expr> applyUnary(Args &&... args) { - return std::make_unique<Op>(popExprOnly(), std::forward<Args>(args)...); - } - - template<typename Op> - std::unique_ptr<Expr> applyBinary(PosIdx pos) { - auto right = popExprOnly(), left = popExprOnly(); - return std::make_unique<Op>(pos, std::move(left), std::move(right)); - } - - std::unique_ptr<Expr> call(PosIdx pos, Symbol fn, bool flip = false) - { - std::vector<std::unique_ptr<Expr>> args(2); - args[flip ? 0 : 1] = popExprOnly(); - args[flip ? 1 : 0] = popExprOnly(); - return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args)); - } - - std::unique_ptr<Expr> pipe(PosIdx pos, State & state, bool flip = false) - { - if (!state.featureSettings.isEnabled(Xp::PipeOperator)) - throw ParseError({ - .msg = HintFmt("Pipe operator is disabled"), - .pos = state.positions[pos] - }); - - // Reverse the order compared to normal function application: arg |> fn - std::unique_ptr<Expr> fn, arg; - if (flip) { - fn = popExprOnly(); - arg = popExprOnly(); - } else { - arg = popExprOnly(); - fn = popExprOnly(); - } - std::vector<std::unique_ptr<Expr>> args{1}; - args[0] = std::move(arg); - - return std::make_unique<ExprCall>(pos, std::move(fn), std::move(args)); - } - - std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state) - { - return call(pos, state.s.lessThan, !less); - } - - std::unique_ptr<Expr> concatStrings(PosIdx pos) - { - std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> args(2); - args[1] = popExpr(); - args[0] = popExpr(); - return std::make_unique<ExprConcatStrings>(pos, false, std::move(args)); - } - - std::unique_ptr<Expr> negate(PosIdx pos, State & state) - { - std::vector<std::unique_ptr<Expr>> args(2); - args[0] = std::make_unique<ExprInt>(0); - args[1] = popExprOnly(); - return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(state.s.sub), std::move(args)); - } - - std::pair<PosIdx, std::unique_ptr<Expr>> applyOp(PosIdx pos, auto & op, State & state) { - using Op = grammar::op; - - auto not_ = [] (auto e) { - return std::make_unique<ExprOpNot>(std::move(e)); - }; - - return { - pos, - (overloaded { - [&] (Op::implies) { return applyBinary<ExprOpImpl>(pos); }, - [&] (Op::or_) { return applyBinary<ExprOpOr>(pos); }, - [&] (Op::and_) { return applyBinary<ExprOpAnd>(pos); }, - [&] (Op::equals) { return applyBinary<ExprOpEq>(pos); }, - [&] (Op::not_equals) { return applyBinary<ExprOpNEq>(pos); }, - [&] (Op::less) { return order(pos, true, state); }, - [&] (Op::greater_eq) { return not_(order(pos, true, state)); }, - [&] (Op::greater) { return order(pos, false, state); }, - [&] (Op::less_eq) { return not_(order(pos, false, state)); }, - [&] (Op::update) { return applyBinary<ExprOpUpdate>(pos); }, - [&] (Op::not_) { return applyUnary<ExprOpNot>(); }, - [&] (Op::plus) { return concatStrings(pos); }, - [&] (Op::minus) { return call(pos, state.s.sub); }, - [&] (Op::mul) { return call(pos, state.s.mul); }, - [&] (Op::div) { return call(pos, state.s.div); }, - [&] (Op::concat) { return applyBinary<ExprOpConcatLists>(pos); }, - [&] (has_attr & a) { return applyUnary<ExprOpHasAttr>(std::move(a.path)); }, - [&] (Op::unary_minus) { return negate(pos, state); }, - [&] (Op::pipe_right) { return pipe(pos, state, true); }, - [&] (Op::pipe_left) { return pipe(pos, state); }, - })(op) - }; - } - - // always_inline is needed, otherwise pushOp slows down considerably - [[noreturn, gnu::always_inline]] - static void badOperator(PosIdx pos, State & state) - { - throw ParseError({ - .msg = HintFmt("syntax error, unexpected operator"), - .pos = state.positions[pos] - }); - } - - template<typename Expr, typename... Args> - Expr & pushExpr(PosIdx pos, Args && ... args) - { - auto p = std::make_unique<Expr>(std::forward<Args>(args)...); - auto & result = *p; - exprs.emplace_back(pos, std::move(p)); - return result; - } -}; - -struct SubexprState { -private: - ExprState * up; - -public: - explicit SubexprState(ExprState & up, auto &...) : up(&up) {} - operator ExprState &() { return *up; } - ExprState * operator->() { return up; } -}; - - - -template<typename Rule> -struct BuildAST : grammar::nothing<Rule> {}; - -struct LambdaState : SubexprState { - using SubexprState::SubexprState; - - Symbol arg; - std::unique_ptr<Formals> formals; -}; - -struct FormalsState : SubexprState { - using SubexprState::SubexprState; - - Formals formals{}; - Formal formal{}; -}; - -template<> struct BuildAST<grammar::formal::name> { - static void apply(const auto & in, FormalsState & s, State & ps) { - s.formal = { - .pos = ps.at(in), - .name = ps.symbols.create(in.string_view()), - }; - } -}; - -template<> struct BuildAST<grammar::formal> { - static void apply0(FormalsState & s, State &) { - s.formals.formals.emplace_back(std::move(s.formal)); - } -}; - -template<> struct BuildAST<grammar::formal::default_value> { - static void apply0(FormalsState & s, State & ps) { - s.formal.def = s->popExprOnly(); - } -}; - -template<> struct BuildAST<grammar::formals::ellipsis> { - static void apply0(FormalsState & s, State &) { - s.formals.ellipsis = true; - } -}; - -template<> struct BuildAST<grammar::formals> : change_head<FormalsState> { - static void success0(FormalsState & f, LambdaState & s, State &) { - s.formals = std::make_unique<Formals>(std::move(f.formals)); - } -}; - -struct AttrState : SubexprState { - using SubexprState::SubexprState; - - std::vector<AttrName> attrs; - - template <typename T> - void pushAttr(T && attr, PosIdx) { attrs.emplace_back(std::forward<T>(attr)); } -}; - -template<> struct BuildAST<grammar::attr::simple> { - static void apply(const auto & in, auto & s, State & ps) { - s.pushAttr(ps.symbols.create(in.string_view()), ps.at(in)); - } -}; - -template<> struct BuildAST<grammar::attr::string> { - static void apply(const auto & in, auto & s, State & ps) { - auto e = s->popExprOnly(); - if (auto str = dynamic_cast<ExprString *>(e.get())) - s.pushAttr(ps.symbols.create(str->s), ps.at(in)); - else - s.pushAttr(std::move(e), ps.at(in)); - } -}; - -template<> struct BuildAST<grammar::attr::expr> : BuildAST<grammar::attr::string> {}; - -struct BindingsState : SubexprState { - using SubexprState::SubexprState; - - ExprAttrs attrs; - AttrPath path; - std::unique_ptr<Expr> value; -}; - -struct InheritState : SubexprState { - using SubexprState::SubexprState; - - std::vector<std::pair<AttrName, PosIdx>> attrs; - std::unique_ptr<Expr> from; - PosIdx fromPos; - - template <typename T> - void pushAttr(T && attr, PosIdx pos) { attrs.emplace_back(std::forward<T>(attr), pos); } -}; - -template<> struct BuildAST<grammar::inherit::from> { - static void apply(const auto & in, InheritState & s, State & ps) { - s.from = s->popExprOnly(); - s.fromPos = ps.at(in); - } -}; - -template<> struct BuildAST<grammar::inherit> : change_head<InheritState> { - static void success0(InheritState & s, BindingsState & b, State & ps) { - auto & attrs = b.attrs.attrs; - // TODO this should not reuse generic attrpath rules. - for (auto & [i, iPos] : s.attrs) { - if (i.symbol) - continue; - if (auto str = dynamic_cast<ExprString *>(i.expr.get())) - i = AttrName(ps.symbols.create(str->s)); - else { - throw ParseError({ - .msg = HintFmt("dynamic attributes not allowed in inherit"), - .pos = ps.positions[iPos] - }); - } - } - if (s.from != nullptr) { - if (!b.attrs.inheritFromExprs) - b.attrs.inheritFromExprs = std::make_unique<std::vector<ref<Expr>>>(); - auto fromExpr = ref<Expr>(std::move(s.from)); - b.attrs.inheritFromExprs->push_back(fromExpr); - for (auto & [i, iPos] : s.attrs) { - if (attrs.find(i.symbol) != attrs.end()) - ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); - auto inheritFrom = std::make_unique<ExprInheritFrom>( - s.fromPos, - b.attrs.inheritFromExprs->size() - 1, - fromExpr - ); - attrs.emplace( - i.symbol, - ExprAttrs::AttrDef( - std::make_unique<ExprSelect>(iPos, std::move(inheritFrom), i.symbol), - iPos, - ExprAttrs::AttrDef::Kind::InheritedFrom)); - } - } else { - for (auto & [i, iPos] : s.attrs) { - if (attrs.find(i.symbol) != attrs.end()) - ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); - attrs.emplace( - i.symbol, - ExprAttrs::AttrDef( - std::make_unique<ExprVar>(iPos, i.symbol), - iPos, - ExprAttrs::AttrDef::Kind::Inherited)); - } - } - } -}; - -template<> struct BuildAST<grammar::binding::path> : change_head<AttrState> { - static void success0(AttrState & a, BindingsState & s, State & ps) { - s.path = std::move(a.attrs); - } -}; - -template<> struct BuildAST<grammar::binding::value> { - static void apply0(BindingsState & s, State & ps) { - s.value = s->popExprOnly(); - } -}; - -template<> struct BuildAST<grammar::binding> { - static void apply(const auto & in, BindingsState & s, State & ps) { - ps.addAttr(&s.attrs, std::move(s.path), std::move(s.value), ps.at(in)); - } -}; - -template<> struct BuildAST<grammar::expr::id> { - static void apply(const auto & in, ExprState & s, State & ps) { - if (in.string_view() == "__curPos") - s.pushExpr<ExprPos>(ps.at(in), ps.at(in)); - else - s.pushExpr<ExprVar>(ps.at(in), ps.at(in), ps.symbols.create(in.string_view())); - } -}; - -template<> struct BuildAST<grammar::expr::int_> { - static void apply(const auto & in, ExprState & s, State & ps) { - int64_t v; - if (std::from_chars(in.begin(), in.end(), v).ec != std::errc{}) { - throw ParseError({ - .msg = HintFmt("invalid integer '%1%'", in.string_view()), - .pos = ps.positions[ps.at(in)], - }); - } - s.pushExpr<ExprInt>(noPos, v); - } -}; - -template<> struct BuildAST<grammar::expr::float_> { - static void apply(const auto & in, ExprState & s, State & ps) { - // copy the input into a temporary string so we can call stod. - // can't use from_chars because libc++ (thus darwin) does not have it, - // and floats are not performance-sensitive anyway. if they were you'd - // be in much bigger trouble than this. - // - // we also get to do a locale-save dance because stod is locale-aware and - // something (a plugin?) may have called setlocale or uselocale. - static struct locale_hack { - locale_t posix; - locale_hack(): posix(newlocale(LC_ALL_MASK, "POSIX", 0)) - { - if (posix == 0) - throw SysError("could not get POSIX locale"); - } - } locale; - - auto tmp = in.string(); - double v = [&] { - auto oldLocale = uselocale(locale.posix); - Finally resetLocale([=] { uselocale(oldLocale); }); - try { - return std::stod(tmp); - } catch (...) { - throw ParseError({ - .msg = HintFmt("invalid float '%1%'", in.string_view()), - .pos = ps.positions[ps.at(in)], - }); - } - }(); - s.pushExpr<ExprFloat>(noPos, v); - } -}; - -struct StringState : SubexprState { - using SubexprState::SubexprState; - - std::string currentLiteral; - PosIdx currentPos; - std::vector<std::pair<nix::PosIdx, std::unique_ptr<Expr>>> parts; - - void append(PosIdx pos, std::string_view s) - { - if (currentLiteral.empty()) - currentPos = pos; - currentLiteral += s; - } - - // FIXME this truncates strings on NUL for compat with the old parser. ideally - // we should use the decomposition the g gives us instead of iterating over - // the entire string again. - static void unescapeStr(std::string & str) - { - char * s = str.data(); - char * t = s; - char c; - while ((c = *s++)) { - if (c == '\\') { - c = *s++; - if (c == 'n') *t = '\n'; - else if (c == 'r') *t = '\r'; - else if (c == 't') *t = '\t'; - else *t = c; - } - else if (c == '\r') { - /* Normalise CR and CR/LF into LF. */ - *t = '\n'; - if (*s == '\n') s++; /* cr/lf */ - } - else *t = c; - t++; - } - str.resize(t - str.data()); - } - - void endLiteral() - { - if (!currentLiteral.empty()) { - unescapeStr(currentLiteral); - parts.emplace_back(currentPos, std::make_unique<ExprString>(std::move(currentLiteral))); - } - } - - std::unique_ptr<Expr> finish() - { - if (parts.empty()) { - unescapeStr(currentLiteral); - return std::make_unique<ExprString>(std::move(currentLiteral)); - } else { - endLiteral(); - auto pos = parts[0].first; - return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts)); - } - } -}; - -template<typename... Content> struct BuildAST<grammar::string::literal<Content...>> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), in.string_view()); - } -}; - -template<> struct BuildAST<grammar::string::cr_lf> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), in.string_view()); // FIXME compat with old parser - } -}; - -template<> struct BuildAST<grammar::string::interpolation> { - static void apply(const auto & in, StringState & s, State & ps) { - s.endLiteral(); - s.parts.emplace_back(ps.at(in), s->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::string::escape> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), "\\"); // FIXME compat with old parser - s.append(ps.at(in), in.string_view()); - } -}; - -template<> struct BuildAST<grammar::string> : change_head<StringState> { - static void success0(StringState & s, ExprState & e, State &) { - e.exprs.emplace_back(noPos, s.finish()); - } -}; - -struct IndStringState : SubexprState { - using SubexprState::SubexprState; - - std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> parts; -}; - -template<bool Indented, typename... Content> -struct BuildAST<grammar::ind_string::literal<Indented, Content...>> { - static void apply(const auto & in, IndStringState & s, State & ps) { - s.parts.emplace_back(ps.at(in), StringToken{in.string_view(), Indented}); - } -}; - -template<> struct BuildAST<grammar::ind_string::interpolation> { - static void apply(const auto & in, IndStringState & s, State & ps) { - s.parts.emplace_back(ps.at(in), s->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::ind_string::escape> { - static void apply(const auto & in, IndStringState & s, State & ps) { - switch (*in.begin()) { - case 'n': s.parts.emplace_back(ps.at(in), StringToken{"\n"}); break; - case 'r': s.parts.emplace_back(ps.at(in), StringToken{"\r"}); break; - case 't': s.parts.emplace_back(ps.at(in), StringToken{"\t"}); break; - default: s.parts.emplace_back(ps.at(in), StringToken{in.string_view()}); break; - } - } -}; - -template<> struct BuildAST<grammar::ind_string> : change_head<IndStringState> { - static void success(const auto & in, IndStringState & s, ExprState & e, State & ps) { - e.exprs.emplace_back(noPos, ps.stripIndentation(ps.at(in), std::move(s.parts))); - } -}; - -template<typename... Content> struct BuildAST<grammar::path::literal<Content...>> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), in.string_view()); - s.endLiteral(); - } -}; - -template<> struct BuildAST<grammar::path::interpolation> : BuildAST<grammar::string::interpolation> {}; - -template<> struct BuildAST<grammar::path::anchor> { - static void apply(const auto & in, StringState & s, State & ps) { - Path path(absPath(in.string(), ps.basePath.path.abs())); - /* add back in the trailing '/' to the first segment */ - if (in.string_view().ends_with('/') && in.size() > 1) - path += "/"; - s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); - } -}; - -template<> struct BuildAST<grammar::path::home_anchor> { - static void apply(const auto & in, StringState & s, State & ps) { - if (evalSettings.pureEval) - throw Error("the path '%s' can not be resolved in pure mode", in.string_view()); - Path path(getHome() + in.string_view().substr(1)); - s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); - } -}; - -template<> struct BuildAST<grammar::path::searched_path> { - static void apply(const auto & in, StringState & s, State & ps) { - std::vector<std::unique_ptr<Expr>> args{2}; - args[0] = std::make_unique<ExprVar>(ps.s.nixPath); - args[1] = std::make_unique<ExprString>(in.string()); - s.parts.emplace_back( - ps.at(in), - std::make_unique<ExprCall>( - ps.at(in), - std::make_unique<ExprVar>(ps.s.findFile), - std::move(args))); - } -}; - -template<> struct BuildAST<grammar::path> : change_head<StringState> { - template<typename E> - static void check_slash(PosIdx end, StringState & s, State & ps) { - auto e = dynamic_cast<E *>(s.parts.back().second.get()); - if (!e || !e->s.ends_with('/')) - return; - if (s.parts.size() > 1 || e->s != "/") - throw ParseError({ - .msg = HintFmt("path has a trailing slash"), - .pos = ps.positions[end], - }); - } - - static void success(const auto & in, StringState & s, ExprState & e, State & ps) { - s.endLiteral(); - check_slash<ExprPath>(ps.atEnd(in), s, ps); - check_slash<ExprString>(ps.atEnd(in), s, ps); - if (s.parts.size() == 1) { - e.exprs.emplace_back(noPos, std::move(s.parts.back().second)); - } else { - e.pushExpr<ExprConcatStrings>(ps.at(in), ps.at(in), false, std::move(s.parts)); - } - } -}; - -// strings and paths sare handled fully by the grammar-level rule for now -template<> struct BuildAST<grammar::expr::string> : p::maybe_nothing {}; -template<> struct BuildAST<grammar::expr::ind_string> : p::maybe_nothing {}; -template<> struct BuildAST<grammar::expr::path> : p::maybe_nothing {}; - -template<> struct BuildAST<grammar::expr::uri> { - static void apply(const auto & in, ExprState & s, State & ps) { - bool URLLiterals = ps.featureSettings.isEnabled(Dep::UrlLiterals); - if (!URLLiterals) - throw ParseError({ - .msg = HintFmt("URL literals are deprecated, allow using them with --extra-deprecated-features=url-literals"), - .pos = ps.positions[ps.at(in)] - }); - s.pushExpr<ExprString>(ps.at(in), in.string()); - } -}; - -template<> struct BuildAST<grammar::expr::ancient_let> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - // Added 2024-09-18. Turn into an error at some point in the future. - // See the documentation on deprecated features for more details. - if (!ps.featureSettings.isEnabled(Dep::AncientLet)) - warn( - "%s found at %s. This feature is deprecated and will be removed in the future. Use %s to silence this warning.", - "let {", - ps.positions[ps.at(in)], - "--extra-deprecated-features ancient-let" - ); - - b.attrs.pos = ps.at(in); - b.attrs.recursive = true; - s.pushExpr<ExprSelect>(b.attrs.pos, b.attrs.pos, std::make_unique<ExprAttrs>(std::move(b.attrs)), ps.s.body); - } -}; - -template<> struct BuildAST<grammar::expr::rec_set> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - // Before inserting new attrs, check for __override and throw an error - // (the error will initially be a warning to ease migration) - if (!featureSettings.isEnabled(Dep::RecSetOverrides) && b.attrs.attrs.contains(ps.s.overrides)) { - ps.overridesFound(ps.at(in)); - } - - b.attrs.pos = ps.at(in); - b.attrs.recursive = true; - s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); - } -}; - -template<> struct BuildAST<grammar::expr::set> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - b.attrs.pos = ps.at(in); - s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); - } -}; - -using ListState = std::vector<std::unique_ptr<Expr>>; - -template<> struct BuildAST<grammar::expr::list> : change_head<ListState> { - static void success(const auto & in, ListState & ls, ExprState & s, State & ps) { - auto e = std::make_unique<ExprList>(); - e->elems = std::move(ls); - s.exprs.emplace_back(ps.at(in), std::move(e)); - } -}; - -template<> struct BuildAST<grammar::expr::list::entry> : change_head<ExprState> { - static void success0(ExprState & e, ListState & s, State & ps) { - s.emplace_back(e.finish(ps).second); - } -}; - -struct SelectState : SubexprState { - using SubexprState::SubexprState; - - PosIdx pos; - ExprSelect * e = nullptr; -}; - -template<> struct BuildAST<grammar::expr::select::head> { - static void apply(const auto & in, SelectState & s, State & ps) { - s.pos = ps.at(in); - } -}; - -template<> struct BuildAST<grammar::expr::select::attr> : change_head<AttrState> { - static void success0(AttrState & a, SelectState & s, State &) { - s.e = &s->pushExpr<ExprSelect>(s.pos, s.pos, s->popExprOnly(), std::move(a.attrs), nullptr); - } -}; - -template<> struct BuildAST<grammar::expr::select::attr_or> { - static void apply0(SelectState & s, State &) { - s.e->def = s->popExprOnly(); - } -}; - -template<> struct BuildAST<grammar::expr::select::as_app_or> { - static void apply(const auto & in, SelectState & s, State & ps) { - std::vector<std::unique_ptr<Expr>> args(1); - args[0] = std::make_unique<ExprVar>(ps.at(in), ps.s.or_); - s->pushExpr<ExprCall>(s.pos, s.pos, s->popExprOnly(), std::move(args)); - } -}; - -template<> struct BuildAST<grammar::expr::select> : change_head<SelectState> { - static void success0(const auto &...) {} -}; - -struct AppState : SubexprState { - using SubexprState::SubexprState; - - PosIdx pos; - ExprCall * e = nullptr; -}; - -template<> struct BuildAST<grammar::expr::app::select_or_fn> { - static void apply(const auto & in, AppState & s, State & ps) { - s.pos = ps.at(in); - } -}; - -template<> struct BuildAST<grammar::expr::app::first_arg> { - static void apply(auto & in, AppState & s, State & ps) { - auto arg = s->popExprOnly(), fn = s->popExprOnly(); - if ((s.e = dynamic_cast<ExprCall *>(fn.get()))) { - // TODO remove. - // AST compat with old parser, semantics are the same. - // this can happen on occasions such as `<p> <p>` or `a or b or`, - // neither of which are super worth optimizing. - s.e->args.push_back(std::move(arg)); - s->exprs.emplace_back(noPos, std::move(fn)); - } else { - std::vector<std::unique_ptr<Expr>> args{1}; - args[0] = std::move(arg); - s.e = &s->pushExpr<ExprCall>(s.pos, s.pos, std::move(fn), std::move(args)); - } - } -}; - -template<> struct BuildAST<grammar::expr::app::another_arg> { - static void apply0(AppState & s, State & ps) { - s.e->args.push_back(s->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::expr::app> : change_head<AppState> { - static void success0(const auto &...) {} -}; - -template<typename Op> struct BuildAST<grammar::expr::operator_<Op>> { - static void apply(const auto & in, ExprState & s, State & ps) { - s.pushOp(ps.at(in), Op{}, ps); - } -}; -template<> struct BuildAST<grammar::expr::operator_<grammar::op::has_attr>> : change_head<AttrState> { - static void success(const auto & in, AttrState & a, ExprState & s, State & ps) { - s.pushOp(ps.at(in), ExprState::has_attr{{}, std::move(a.attrs)}, ps); - } -}; - -template<> struct BuildAST<grammar::expr::lambda::arg> { - static void apply(const auto & in, LambdaState & s, State & ps) { - s.arg = ps.symbols.create(in.string_view()); - } -}; - -template<> struct BuildAST<grammar::expr::lambda> : change_head<LambdaState> { - static void success(const auto & in, LambdaState & l, ExprState & s, State & ps) { - if (l.formals) - l.formals = ps.validateFormals(std::move(l.formals), ps.at(in), l.arg); - s.pushExpr<ExprLambda>(ps.at(in), ps.at(in), l.arg, std::move(l.formals), l->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::expr::assert_> { - static void apply(const auto & in, ExprState & s, State & ps) { - auto body = s.popExprOnly(), cond = s.popExprOnly(); - s.pushExpr<ExprAssert>(ps.at(in), ps.at(in), std::move(cond), std::move(body)); - } -}; - -template<> struct BuildAST<grammar::expr::with> { - static void apply(const auto & in, ExprState & s, State & ps) { - auto body = s.popExprOnly(), scope = s.popExprOnly(); - s.pushExpr<ExprWith>(ps.at(in), ps.at(in), std::move(scope), std::move(body)); - } -}; - -template<> struct BuildAST<grammar::expr::let> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - if (!b.attrs.dynamicAttrs.empty()) - throw ParseError({ - .msg = HintFmt("dynamic attributes not allowed in let"), - .pos = ps.positions[ps.at(in)] - }); - - s.pushExpr<ExprLet>(ps.at(in), std::make_unique<ExprAttrs>(std::move(b.attrs)), b->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::expr::if_> { - static void apply(const auto & in, ExprState & s, State & ps) { - auto else_ = s.popExprOnly(), then = s.popExprOnly(), cond = s.popExprOnly(); - s.pushExpr<ExprIf>(ps.at(in), ps.at(in), std::move(cond), std::move(then), std::move(else_)); - } -}; - -template<> struct BuildAST<grammar::expr> : change_head<ExprState> { - static void success0(ExprState & inner, ExprState & outer, State & ps) { - outer.exprs.push_back(inner.finish(ps)); - } -}; - -} -} +// Linter complains that this is a "suspicious include of file with '.cc' extension". +// While that is correct and generally not great, it is one of the less bad options to pick +// in terms of diff noise. +// NOLINTNEXTLINE(bugprone-suspicious-include) +#include "parser-impl1.inc.cc" namespace nix { @@ -884,7 +38,6 @@ Expr * EvalState::parse( exprSymbols, featureSettings, }; - parser::ExprState x; assert(length >= 2); assert(text[length - 1] == 0); @@ -893,7 +46,12 @@ Expr * EvalState::parse( p::string_input<p::tracking_mode::lazy> inp{std::string_view{text, length}, "input"}; try { - p::parse<parser::grammar::root, parser::BuildAST, parser::Control>(inp, x, s); + parser::v1::ExprState x; + p::parse<parser::grammar::v1::root, parser::v1::BuildAST, parser::v1::Control>(inp, x, s); + + auto [_pos, result] = x.finish(s); + result->bindVars(*this, staticEnv); + return result.release(); } catch (p::parse_error & e) { auto pos = e.positions().back(); throw ParseError({ @@ -901,10 +59,6 @@ Expr * EvalState::parse( .pos = positions[s.positions.add(s.origin, pos.byte)] }); } - - auto [_pos, result] = x.finish(s); - result->bindVars(*this, staticEnv); - return result.release(); } } diff --git a/src/libexpr/parser/state.hh b/src/libexpr/parser/state.hh index 3b9b90b94..b969f73e4 100644 --- a/src/libexpr/parser/state.hh +++ b/src/libexpr/parser/state.hh @@ -6,11 +6,21 @@ namespace nix::parser { -struct StringToken -{ - std::string_view s; - bool hasIndentation = false; - operator std::string_view() const { return s; } +struct IndStringLine { + // String containing only the leading whitespace of the line. May be empty. + std::string_view indentation; + // Position of the line start (before the indentation) + PosIdx pos; + + // Whether the line contains anything besides indentation and line break + bool hasContent = false; + + std::vector< + std::pair< + PosIdx, + std::variant<std::unique_ptr<Expr>, std::string_view> + > + > parts = {}; }; struct State @@ -27,8 +37,7 @@ struct State void overridesFound(const PosIdx pos); void addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos); std::unique_ptr<Formals> validateFormals(std::unique_ptr<Formals> formals, PosIdx pos = noPos, Symbol arg = {}); - std::unique_ptr<Expr> stripIndentation(const PosIdx pos, - std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es); + std::unique_ptr<Expr> stripIndentation(const PosIdx pos, std::vector<IndStringLine> && line); // lazy positioning means we don't get byte offsets directly, in.position() would work // but also requires line and column (which is expensive) @@ -182,98 +191,87 @@ inline std::unique_ptr<Formals> State::validateFormals(std::unique_ptr<Formals> return formals; } -inline std::unique_ptr<Expr> State::stripIndentation(const PosIdx pos, - std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es) +inline std::unique_ptr<Expr> State::stripIndentation( + const PosIdx pos, + std::vector<IndStringLine> && lines) { - if (es.empty()) return std::make_unique<ExprString>(""); + /* If the only line is whitespace-only, directly return empty string. + * The rest of the code relies on the final string not being empty. + */ + if (lines.size() == 1 && lines.front().parts.empty()) { + return std::make_unique<ExprString>(""); + } - /* Figure out the minimum indentation. Note that by design - whitespace-only final lines are not taken into account. (So - the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ - bool atStartOfLine = true; /* = seen only whitespace in the current line */ + /* If the last line only contains whitespace, trim it to not cause excessive whitespace. + * (Other whitespace-only lines get stripped only of the common indentation, and excess + * whitespace becomes part of the string.) + */ + if (lines.back().parts.empty()) { + lines.back().indentation = {}; + } + + /* Figure out the minimum indentation. Note that by design + whitespace-only lines are not taken into account. */ size_t minIndent = 1000000; - size_t curIndent = 0; - for (auto & [i_pos, i] : es) { - auto * str = std::get_if<StringToken>(&i); - if (!str || !str->hasIndentation) { - /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ - if (atStartOfLine) { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - continue; - } - for (size_t j = 0; j < str->s.size(); ++j) { - if (atStartOfLine) { - if (str->s[j] == ' ') - curIndent++; - else if (str->s[j] == '\n') { - /* Empty line, doesn't influence minimum - indentation. */ - curIndent = 0; - } else { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - } else if (str->s[j] == '\n') { - atStartOfLine = true; - curIndent = 0; - } + for (auto & line : lines) { + if (line.hasContent) { + minIndent = std::min(minIndent, line.indentation.size()); } } /* Strip spaces from each line. */ - std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es2; - atStartOfLine = true; - size_t curDropped = 0; - size_t n = es.size(); - auto i = es.begin(); - const auto trimExpr = [&] (std::unique_ptr<Expr> e) { - atStartOfLine = false; - curDropped = 0; - es2.emplace_back(i->first, std::move(e)); - }; - const auto trimString = [&] (const StringToken t) { - std::string s2; - for (size_t j = 0; j < t.s.size(); ++j) { - if (atStartOfLine) { - if (t.s[j] == ' ') { - if (curDropped++ >= minIndent) - s2 += t.s[j]; - } - else if (t.s[j] == '\n') { - curDropped = 0; - s2 += t.s[j]; - } else { - atStartOfLine = false; - curDropped = 0; - s2 += t.s[j]; - } - } else { - s2 += t.s[j]; - if (t.s[j] == '\n') atStartOfLine = true; - } - } + for (auto & line : lines) { + line.indentation.remove_prefix(std::min(minIndent, line.indentation.size())); + } + + /* Concat the parts together again */ + + std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> parts; + /* Accumulator for merging intermediates */ + PosIdx merged_pos; + std::string merged = ""; - /* Remove the last line if it is empty and consists only of - spaces. */ - if (n == 1) { - std::string::size_type p = s2.find_last_of('\n'); - if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) - s2 = std::string(s2, 0, p + 1); + auto push_merged = [&] (PosIdx i_pos, std::string_view str) { + if (merged.empty()) { + merged_pos = i_pos; } + merged += str; + }; - es2.emplace_back(i->first, std::make_unique<ExprString>(std::move(s2))); + auto flush_merged = [&] () { + if (!merged.empty()) { + parts.emplace_back(merged_pos, std::make_unique<ExprString>(std::string(merged))); + merged.clear(); + } }; - for (; i != es.end(); ++i, --n) { - std::visit(overloaded { trimExpr, trimString }, std::move(i->second)); + + for (auto && [li, line] : enumerate(lines)) { + push_merged(line.pos, line.indentation); + + for (auto & val : line.parts) { + auto &[i_pos, item] = val; + + std::visit(overloaded{ + [&](std::string_view str) { + push_merged(i_pos, str); + }, + [&](std::unique_ptr<Expr> expr) { + flush_merged(); + parts.emplace_back(i_pos, std::move(expr)); + }, + }, std::move(item)); + } } - /* If this is a single string, then don't do a concatenation. */ - if (es2.size() == 1 && dynamic_cast<ExprString *>(es2[0].second.get())) { - return std::move(es2[0].second); + flush_merged(); + + /* If this is a single string, then don't do a concatenation. + * (If it's a single expression, still do the ConcatStrings to properly force it being a string.) + */ + if (parts.size() == 1 && dynamic_cast<ExprString *>(parts[0].second.get())) { + return std::move(parts[0].second); } - return std::make_unique<ExprConcatStrings>(pos, true, std::move(es2)); + return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts)); } } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 96140e10b..765df5f5a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -127,7 +127,15 @@ Goal::WorkResult DerivationGoal::timedOut(Error && ex) kj::Promise<Result<Goal::WorkResult>> DerivationGoal::workImpl() noexcept { - return useDerivation ? getDerivation() : haveDerivation(); + KJ_DEFER({ + act.reset(); + actLock.reset(); + builderActivities.clear(); + }); + + BOOST_OUTCOME_CO_TRY(auto result, co_await (useDerivation ? getDerivation() : haveDerivation())); + result.storePath = drvPath; + co_return result; } bool DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 808179a4d..36ad35be0 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -6,26 +6,20 @@ namespace nix { -static auto runWorker(Worker & worker, auto mkGoals) -{ - return worker.run(mkGoals); -} - void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode, std::shared_ptr<Store> evalStore) { auto aio = kj::setupAsyncIo(); - Worker worker(*this, evalStore ? *evalStore : *this, aio); - auto goals = runWorker(worker, [&](GoalFactory & gf) { + auto results = processGoals(*this, evalStore ? *evalStore : *this, aio, [&](GoalFactory & gf) { Worker::Targets goals; for (auto & br : reqs) - goals.emplace(gf.makeGoal(br, buildMode)); + goals.emplace_back(gf.makeGoal(br, buildMode)); return goals; - }); + }).wait(aio.waitScope).value(); StringSet failed; std::shared_ptr<Error> ex; - for (auto & [i, result] : goals) { + for (auto & [i, result] : results.goals) { if (result.ex) { if (ex) logError(result.ex->info()); @@ -33,19 +27,17 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod ex = result.ex; } if (result.exitCode != Goal::ecSuccess) { - if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) - failed.insert(printStorePath(i2->drvPath)); - else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) - failed.insert(printStorePath(i2->storePath)); + if (result.storePath) + failed.insert(printStorePath(*result.storePath)); } } if (failed.size() == 1 && ex) { - ex->withExitStatus(worker.failingExitStatus()); + ex->withExitStatus(results.failingExitStatus); throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); + throw Error(results.failingExitStatus, "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } @@ -55,24 +47,19 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults( std::shared_ptr<Store> evalStore) { auto aio = kj::setupAsyncIo(); - Worker worker(*this, evalStore ? *evalStore : *this, aio); - std::vector<std::pair<const DerivedPath &, GoalPtr>> state; - - auto goals = runWorker(worker, [&](GoalFactory & gf) { + auto goals = processGoals(*this, evalStore ? *evalStore : *this, aio, [&](GoalFactory & gf) { Worker::Targets goals; for (const auto & req : reqs) { - auto goal = gf.makeGoal(req, buildMode); - state.push_back({req, goal.first}); - goals.emplace(std::move(goal)); + goals.emplace_back(gf.makeGoal(req, buildMode)); } return goals; - }); + }).wait(aio.waitScope).value().goals; std::vector<KeyedBuildResult> results; - for (auto & [req, goalPtr] : state) - results.emplace_back(goals[goalPtr].result.restrictTo(req)); + for (auto && [goalIdx, req] : enumerate(reqs)) + results.emplace_back(goals[goalIdx].result.restrictTo(req)); return results; } @@ -81,15 +68,14 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat BuildMode buildMode) { auto aio = kj::setupAsyncIo(); - Worker worker(*this, *this, aio); try { - auto goals = runWorker(worker, [&](GoalFactory & gf) { + auto results = processGoals(*this, *this, aio, [&](GoalFactory & gf) { Worker::Targets goals; - goals.emplace(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)); + goals.emplace_back(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)); return goals; - }); - auto [goal, result] = *goals.begin(); + }).wait(aio.waitScope).value(); + auto & result = results.goals.begin()->second; return result.result.restrictTo(DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, @@ -109,21 +95,20 @@ void Store::ensurePath(const StorePath & path) if (isValidPath(path)) return; auto aio = kj::setupAsyncIo(); - Worker worker(*this, *this, aio); - auto goals = runWorker(worker, [&](GoalFactory & gf) { + auto results = processGoals(*this, *this, aio, [&](GoalFactory & gf) { Worker::Targets goals; - goals.emplace(gf.makePathSubstitutionGoal(path)); + goals.emplace_back(gf.makePathSubstitutionGoal(path)); return goals; - }); - auto [goal, result] = *goals.begin(); + }).wait(aio.waitScope).value(); + auto & result = results.goals.begin()->second; if (result.exitCode != Goal::ecSuccess) { if (result.ex) { - result.ex->withExitStatus(worker.failingExitStatus()); + result.ex->withExitStatus(results.failingExitStatus); throw std::move(*result.ex); } else - throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); + throw Error(results.failingExitStatus, "path '%s' does not exist and cannot be created", printStorePath(path)); } } @@ -131,23 +116,22 @@ void Store::ensurePath(const StorePath & path) void Store::repairPath(const StorePath & path) { auto aio = kj::setupAsyncIo(); - Worker worker(*this, *this, aio); - auto goals = runWorker(worker, [&](GoalFactory & gf) { + auto results = processGoals(*this, *this, aio, [&](GoalFactory & gf) { Worker::Targets goals; - goals.emplace(gf.makePathSubstitutionGoal(path, Repair)); + goals.emplace_back(gf.makePathSubstitutionGoal(path, Repair)); return goals; - }); - auto [goal, result] = *goals.begin(); + }).wait(aio.waitScope).value(); + auto & result = results.goals.begin()->second; if (result.exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { - worker.run([&](GoalFactory & gf) { + processGoals(*this, *this, aio, [&](GoalFactory & gf) { Worker::Targets goals; - goals.emplace(gf.makeGoal( + goals.emplace_back(gf.makeGoal( DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*info->deriver), // FIXME: Should just build the specific output we need. @@ -156,9 +140,9 @@ void Store::repairPath(const StorePath & path) bmRepair )); return goals; - }); + }).wait(aio.waitScope).value(); } else - throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); + throw Error(results.failingExitStatus, "cannot repair path '%s'", printStorePath(path)); } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 02b22b8ad..db16e2cf8 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -22,6 +22,12 @@ kj::Promise<void> Goal::waitForAWhile() kj::Promise<Result<Goal::WorkResult>> Goal::work() noexcept try { + // always clear the slot token, no matter what happens. not doing this + // can cause builds to get stuck on exceptions (or other early exist). + // ideally we'd use scoped slot tokens instead of keeping them in some + // goal member variable, but we cannot do this yet for legacy reasons. + KJ_DEFER({ slotToken = {}; }); + BOOST_OUTCOME_CO_TRY(auto result, co_await workImpl()); trace("done"); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 29540dcd3..5b66a2c08 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -94,6 +94,9 @@ public: bool timedOut = false; bool hashMismatch = false; bool checkMismatch = false; + /// Store path this goal relates to. Will be set to drvPath for + /// derivations, or the substituted store path for substitions. + std::optional<StorePath> storePath = {}; }; protected: diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index e0ca23a86..2d0c594ac 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -3,6 +3,7 @@ #include "nar-info.hh" #include "signals.hh" #include "finally.hh" +#include <boost/outcome/try.hpp> #include <kj/array.h> #include <kj/vector.h> @@ -54,7 +55,7 @@ try { /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - return {done(ecSuccess, BuildResult::AlreadyValid)}; + co_return done(ecSuccess, BuildResult::AlreadyValid); } if (settings.readOnlyMode) @@ -62,9 +63,11 @@ try { subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>(); - return tryNext(); + BOOST_OUTCOME_CO_TRY(auto result, co_await tryNext()); + result.storePath = storePath; + co_return result; } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 10f58f5d3..2a764b193 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -229,8 +229,8 @@ try { co_return result::failure(std::current_exception()); } -Worker::Results Worker::run(std::function<Targets (GoalFactory &)> req) -{ +kj::Promise<Result<Worker::Results>> Worker::run(std::function<Targets (GoalFactory &)> req) +try { auto topGoals = req(goalFactory()); assert(!running); @@ -252,16 +252,18 @@ Worker::Results Worker::run(std::function<Targets (GoalFactory &)> req) promise = promise.exclusiveJoin(boopGC(*localStore)); } - return promise.wait(aio.waitScope).value(); + co_return co_await promise; +} catch (...) { + co_return result::failure(std::current_exception()); } kj::Promise<Result<Worker::Results>> Worker::runImpl(Targets topGoals) try { debug("entered goal loop"); - kj::Vector<Targets::value_type> promises(topGoals.size()); - for (auto & gp : topGoals) { - promises.add(std::move(gp)); + kj::Vector<std::pair<size_t, kj::Promise<Result<Goal::WorkResult>>>> promises(topGoals.size()); + for (auto && [idx, gp] : enumerate(topGoals)) { + promises.add(idx, std::move(gp.second)); } Results results; @@ -270,7 +272,7 @@ try { while (auto done = co_await collect.next()) { // propagate goal exceptions outward BOOST_OUTCOME_CO_TRY(auto result, done->second); - results.emplace(done->first, result); + results.goals.emplace(done->first, result); /* If a top-level goal failed, then kill all other goals (unless keepGoing was set). */ @@ -285,6 +287,25 @@ try { --keep-going *is* set, then they must all be finished now. */ assert(!settings.keepGoing || children.isEmpty()); + results.failingExitStatus = [&] { + // See API docs in header for explanation + unsigned int mask = 0; + bool buildFailure = permanentFailure || timedOut || hashMismatch; + if (buildFailure) + mask |= 0x04; // 100 + if (timedOut) + mask |= 0x01; // 101 + if (hashMismatch) + mask |= 0x02; // 102 + if (checkMismatch) { + mask |= 0x08; // 104 + } + + if (mask) + mask |= 0x60; + return mask ? mask : 1; + }(); + co_return std::move(results); } catch (...) { co_return result::failure(std::current_exception()); @@ -301,27 +322,6 @@ try { } -unsigned int Worker::failingExitStatus() -{ - // See API docs in header for explanation - unsigned int mask = 0; - bool buildFailure = permanentFailure || timedOut || hashMismatch; - if (buildFailure) - mask |= 0x04; // 100 - if (timedOut) - mask |= 0x01; // 101 - if (hashMismatch) - mask |= 0x02; // 102 - if (checkMismatch) { - mask |= 0x08; // 104 - } - - if (mask) - mask |= 0x60; - return mask ? mask : 1; -} - - bool Worker::pathContentsGood(const StorePath & path) { auto i = pathContentsGoodCache.find(path); diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 1a913ca16..369e58b41 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -85,8 +85,40 @@ protected: class Worker : public WorkerBase { public: - using Targets = std::map<GoalPtr, kj::Promise<Result<Goal::WorkResult>>>; - using Results = std::map<GoalPtr, Goal::WorkResult>; + using Targets = std::vector<std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>>>; + struct Results { + /** Results of individual goals, if available. Goal results will be + * added to this map with the index they had in the `Targets` list + * returned by the goal factory function passed to `work`. If some + * goals did not complete processing, e.g. due to an early exit on + * goal failures, not all indices will be set. This may be used to + * detect which of the goals were cancelled before they completed. + */ + std::map<size_t, Goal::WorkResult> goals; + + /** + * The exit status in case of failure. + * + * In the case of a build failure, returned value follows this + * bitmask: + * + * ``` + * 0b1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + * ``` + * + * In other words, the failure code is at least 100 (0b1100100), but + * might also be greater. + * + * Otherwise (no build failure, but some other sort of failure by + * assumption), this returned value is 1. + */ + unsigned int failingExitStatus; + }; private: @@ -192,6 +224,7 @@ public: NotifyingCounter<uint64_t> expectedNarSize{[this] { updateStatisticsLater(); }}; NotifyingCounter<uint64_t> doneNarSize{[this] { updateStatisticsLater(); }}; +private: Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio); ~Worker(); @@ -202,7 +235,6 @@ public: /** * @ref DerivationGoal "derivation goal" */ -private: template<typename ID, std::derived_from<Goal> G> std::pair<std::shared_ptr<G>, kj::Promise<Result<Goal::WorkResult>>> makeGoalCommon( std::map<ID, CachedGoal<G>> & map, @@ -246,30 +278,7 @@ public: /** * Loop until the specified top-level goals have finished. */ - Results run(std::function<Targets (GoalFactory &)> req); - - /*** - * The exit status in case of failure. - * - * In the case of a build failure, returned value follows this - * bitmask: - * - * ``` - * 0b1100100 - * ^^^^ - * |||`- timeout - * ||`-- output hash mismatch - * |`--- build failure - * `---- not deterministic - * ``` - * - * In other words, the failure code is at least 100 (0b1100100), but - * might also be greater. - * - * Otherwise (no build failure, but some other sort of failure by - * assumption), this returned value is 1. - */ - unsigned int failingExitStatus(); + kj::Promise<Result<Results>> run(std::function<Targets (GoalFactory &)> req); /** * Check whether the given valid path exists and has the right @@ -278,6 +287,20 @@ public: bool pathContentsGood(const StorePath & path); void markContentsGood(const StorePath & path); + + template<typename MkGoals> + friend kj::Promise<Result<Results>> processGoals( + Store & store, Store & evalStore, kj::AsyncIoContext & aio, MkGoals && mkGoals + ) noexcept; }; +template<typename MkGoals> +kj::Promise<Result<Worker::Results>> processGoals( + Store & store, Store & evalStore, kj::AsyncIoContext & aio, MkGoals && mkGoals +) noexcept +try { + co_return co_await Worker(store, evalStore, aio).run(std::forward<MkGoals>(mkGoals)); +} catch (...) { + co_return result::failure(std::current_exception()); +} } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 34b92148e..acbb042b7 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -282,6 +282,8 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_PROGRESSDATA, this); curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0); + curl_easy_setopt(req, CURLOPT_PROTOCOLS_STR, "http,https,ftp,ftps,file"); + curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); if (settings.downloadSpeed.get() > 0) diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 14381b6e0..31cb3d303 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -9,6 +9,9 @@ #include <unistd.h> #include <errno.h> +#if __APPLE__ +#include <regex> +#endif namespace nix { diff --git a/src/libstore/platform/darwin.cc b/src/libstore/platform/darwin.cc index 956fb1e9b..c039bd142 100644 --- a/src/libstore/platform/darwin.cc +++ b/src/libstore/platform/darwin.cc @@ -9,6 +9,7 @@ #include <libproc.h> #include <spawn.h> +#include <cstddef> #include <regex> namespace nix { @@ -18,16 +19,17 @@ void DarwinLocalStore::findPlatformRoots(UncheckedRoots & unchecked) auto storePathRegex = regex::storePathRegex(storeDir); std::vector<int> pids; - int pidBufSize = 1; + std::size_t pidBufSize = 1; while (pidBufSize > pids.size() * sizeof(int)) { // Reserve some extra size so we don't fail too much pids.resize((pidBufSize + pidBufSize / 8) / sizeof(int)); - pidBufSize = proc_listpids(PROC_ALL_PIDS, 0, pids.data(), pids.size() * sizeof(int)); + auto size = proc_listpids(PROC_ALL_PIDS, 0, pids.data(), pids.size() * sizeof(int)); - if (pidBufSize <= 0) { + if (size <= 0) { throw SysError("Listing PIDs"); } + pidBufSize = size; } pids.resize(pidBufSize / sizeof(int)); @@ -53,12 +55,12 @@ void DarwinLocalStore::findPlatformRoots(UncheckedRoots & unchecked) // File descriptors std::vector<struct proc_fdinfo> fds; - int fdBufSize = 1; + std::size_t fdBufSize = 1; while (fdBufSize > fds.size() * sizeof(struct proc_fdinfo)) { // Reserve some extra size so we don't fail too much fds.resize((fdBufSize + fdBufSize / 8) / sizeof(struct proc_fdinfo)); errno = 0; - fdBufSize = proc_pidinfo( + auto size = proc_pidinfo( pid, PROC_PIDLISTFDS, 0, fds.data(), fds.size() * sizeof(struct proc_fdinfo) ); @@ -72,13 +74,15 @@ void DarwinLocalStore::findPlatformRoots(UncheckedRoots & unchecked) // https://github.com/apple-opensource/xnu/blob/4f43d4276fc6a87f2461a3ab18287e4a2e5a1cc0/libsyscall/wrappers/libproc/libproc.c#L100-L110 // https://git.lix.systems/lix-project/lix/issues/446#issuecomment-5483 // FB14695751 - if (fdBufSize <= 0) { + if (size <= 0) { if (errno == 0) { + fdBufSize = 0; break; } else { throw SysError("Listing pid %1% file descriptors", pid); } } + fdBufSize = size; } fds.resize(fdBufSize / sizeof(struct proc_fdinfo)); diff --git a/src/libutil/async-collect.hh b/src/libutil/async-collect.hh index 9e0b8bad9..1c65ac8d6 100644 --- a/src/libutil/async-collect.hh +++ b/src/libutil/async-collect.hh @@ -63,6 +63,9 @@ public: { } + // oneDone promises capture `this` + KJ_DISALLOW_COPY_AND_MOVE(AsyncCollect); + kj::Promise<std::optional<Item>> next() { if (remaining == 0 && results.empty()) { diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in index 451cf5383..98892f660 100644 --- a/tests/functional/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -28,6 +28,7 @@ export NIX_REMOTE=${NIX_REMOTE_-} unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home export HOME=$TEST_HOME +export GIT_CONFIG_SYSTEM=/dev/null unset XDG_STATE_HOME unset XDG_DATA_HOME unset XDG_CONFIG_HOME diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index ca54fb88d..cec4f9352 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -41,7 +41,12 @@ badExitCode=0 for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; i=$(basename "$i" .nix) - if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" + + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" + fi + if expectStderr 1 nix-instantiate --parse "${flags[@]}" - < "lang/$i.nix" > "lang/$i.err" then diffAndAccept "$i" err err.exp else @@ -54,13 +59,12 @@ for i in lang/parse-okay-*.nix; do echo "parsing $i (should succeed)"; i=$(basename "$i" .nix) - if [ -e "lang/$i.flags" ]; then - extraArgs="$(cat "lang/$i.flags")" - else - extraArgs="" + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" fi if - expect 0 nix-instantiate --parse ${extraArgs-} - < "lang/$i.nix" \ + expect 0 nix-instantiate --parse "${flags[@]}" - < "lang/$i.nix" \ 1> "lang/$i.out" \ 2> "lang/$i.err" then @@ -77,13 +81,12 @@ for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; i=$(basename "$i" .nix) - if [ -e "lang/$i.flags" ]; then - extraArgs="$(cat "lang/$i.flags")" - else - extraArgs="" + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" fi if - expectStderr 1 nix-instantiate --eval --strict --show-trace ${extraArgs-} "lang/$i.nix" \ + expectStderr 1 nix-instantiate --eval --strict --show-trace "${flags[@]}" "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then diffAndAccept "$i" err err.exp @@ -97,14 +100,13 @@ for i in lang/eval-okay-*.nix; do echo "evaluating $i (should succeed)"; i=$(basename "$i" .nix) - if [ -e "lang/$i.flags" ]; then - extraArgs="$(cat "lang/$i.flags")" - else - extraArgs="" + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" fi if test -e "lang/$i.exp.xml"; then - if expect 0 nix-instantiate --eval --xml --no-location --strict ${extraArgs-} \ + if expect 0 nix-instantiate --eval --xml --no-location --strict "${flags[@]}" \ "lang/$i.nix" > "lang/$i.out.xml" then diffAndAccept "$i" out.xml exp.xml @@ -113,11 +115,6 @@ for i in lang/eval-okay-*.nix; do badExitCode=1 fi elif test ! -e "lang/$i.exp-disabled"; then - declare -a flags=() - if test -e "lang/$i.flags"; then - read -r -a flags < "lang/$i.flags" - fi - if expect 0 env \ NIX_PATH=lang/dir3:lang/dir4 \ diff --git a/tests/functional/lang/eval-okay-ind-string.exp b/tests/functional/lang/eval-okay-ind-string.exp index 7862331fa..1531af0f4 100644 --- a/tests/functional/lang/eval-okay-ind-string.exp +++ b/tests/functional/lang/eval-okay-ind-string.exp @@ -1 +1 @@ -"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', \${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: \${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n" +"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', \${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: \${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n Lines without any indentation effectively disable the indentation\n stripping for the entire string:\n\n cat >$out/foo/data <<EOF\n lasjdöaxnasd\nasdom 12398\nä\"§Æẞ¢«»”alsd\nEOF\nEmpty lines with a bit of whitespace don't affect the indentation calculation:\n\nAnd empty lines with more whitespace will have whitespace in the string:\n \nUnless it's the last line:\n Indentation stripping\n must not be impressed by\nthe last line not being empty\t Nor by people\n weirdly mixing tabs\n\tand spaces\n\t" diff --git a/tests/functional/lang/eval-okay-ind-string.nix b/tests/functional/lang/eval-okay-ind-string.nix index 95d59b508..44df83458 100644 --- a/tests/functional/lang/eval-okay-ind-string.nix +++ b/tests/functional/lang/eval-okay-ind-string.nix @@ -64,6 +64,36 @@ let is significant (not whitespace). ''; + s18 = '' + Lines without any indentation effectively disable the indentation + stripping for the entire string: + + cat >$out/foo/data <<EOF + lasjdöaxnasd +asdom 12398 +ä"§Æẞ¢«»”alsd +EOF + ''; + + s19 = '' + Empty lines with a bit of whitespace don't affect the indentation calculation: + + And empty lines with more whitespace will have whitespace in the string: + + Unless it's the last line: + ''; + + s20 = '' + Indentation stripping + must not be impressed by + the last line not being empty''; + + s21 = '' + Nor by people + weirdly mixing tabs + and spaces + ''; + s10 = '' ''; @@ -125,4 +155,4 @@ let # Accept dollars at end of strings s17 = ''ending dollar $'' + ''$'' + "\n"; -in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + s16 + s17 +in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + s16 + s17 + s18 + s19 + s20 + s21 diff --git a/tests/functional/lang/parse-okay-ind-string.exp b/tests/functional/lang/parse-okay-ind-string.exp new file mode 100644 index 000000000..570785ee2 --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.exp @@ -0,0 +1 @@ +(let s1 = "This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\n"; s10 = ""; s11 = ""; s12 = ""; s13 = ("start on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n " + "${(if true then "ln -sf 123 /var/run/opengl-driver" else (if true then "ln -sf 456 /var/run/opengl-driver" else ""))}" + "\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=" + "abc" + "\nenv SLIM_THEMESDIR=" + "def" + "\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=" + "foo" + "/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=" + "libX11" + "/lib:" + "libXext" + "/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\n" + "${(if true then ("env XORG_DRI_DRIVER_PATH=" + "nvidiaDrivers" + "/X11R6/lib/modules/drivers/") else (if true then ("env XORG_DRI_DRIVER_PATH=" + "mesa" + "/lib/modules/dri") else ""))}" + " \n\nexec " + "slim" + "/bin/slim\n"); s14 = "Escaping of ' followed by ': ''\nEscaping of $ followed by {: \${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\n"; s15 = (let x = "bla"; in ("foo\n'" + "${x}" + "'\nbar\n")); s16 = "cut -d $'\\t' -f 1\n"; s17 = (("ending dollar $" + "$") + "\n"); s18 = " Lines without any indentation effectively disable the indentation\n stripping for the entire string:\n\n cat >$out/foo/data <<EOF\n lasjdöaxnasd\nasdom 12398\nä\"§Æẞ¢«»”alsd\nEOF\n"; s19 = "Empty lines with a bit of whitespace don't affect the indentation calculation:\n\nAnd empty lines with more whitespace will have whitespace in the string:\n \nUnless it's the last line:\n"; s2 = "If the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \n"; s20 = " Indentation stripping\n must not be impressed by\nthe last line not being empty"; s21 = "\t Nor by people\n weirdly mixing tabs\n\tand spaces\n\t"; s3 = "This line is indented\na bit further.\n"; s4 = ("Anti-quotations, like " + "${(if true then "so" else "not so")}" + ", are\nalso allowed.\n"); s5 = (" The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: " + "''" + ", " + "\${" + ".\n"); s6 = " Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\n"; s7 = "Also note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. "; s8 = ("" + "\nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n"); s9 = ("" + " Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\n"); in ((((((((((((((((((((s1 + s2) + s3) + s4) + s5) + s6) + s7) + s8) + s9) + s10) + s11) + s12) + s13) + s14) + s15) + s16) + s17) + s18) + s19) + s20) + s21)) diff --git a/tests/functional/lang/parse-okay-ind-string.nix b/tests/functional/lang/parse-okay-ind-string.nix new file mode 120000 index 000000000..43864cd8c --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.nix @@ -0,0 +1 @@ +eval-okay-ind-string.nix
\ No newline at end of file diff --git a/tests/functional/lang/parse-okay-regression-751.exp b/tests/functional/lang/parse-okay-regression-751.exp index e2ed886fe..0cbf55d49 100644 --- a/tests/functional/lang/parse-okay-regression-751.exp +++ b/tests/functional/lang/parse-okay-regression-751.exp @@ -1 +1 @@ -(let const = (a: "const"); in ((const { x = "q"; }))) +(let const = (a: "const"); in ("${(const { x = "q"; })}")) |