diff options
Diffstat (limited to 'src')
105 files changed, 2670 insertions, 2243 deletions
diff --git a/src/build-remote/build-remote.cc b/src/legacy/build-remote.cc index 3c7af067b..62ceef283 100644 --- a/src/build-remote/build-remote.cc +++ b/src/legacy/build-remote.cc @@ -19,8 +19,9 @@ #include "legacy.hh" #include "experimental-features.hh" #include "hash.hh" +#include "build-remote.hh" -using namespace nix; +namespace nix { static void handleAlarm(int sig) { } @@ -388,4 +389,8 @@ connected: } } -static RegisterLegacyCommand r_build_remote("build-remote", main_build_remote); +void registerBuildRemote() { + LegacyCommands::add("build-remote", main_build_remote); +} + +} diff --git a/src/legacy/build-remote.hh b/src/legacy/build-remote.hh new file mode 100644 index 000000000..c4a35f706 --- /dev/null +++ b/src/legacy/build-remote.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerBuildRemote(); + +} diff --git a/src/nix-env/buildenv.nix b/src/legacy/buildenv.nix index c8955a94e..c8955a94e 100644 --- a/src/nix-env/buildenv.nix +++ b/src/legacy/buildenv.nix diff --git a/src/nix-store/dotgraph.cc b/src/legacy/dotgraph.cc index 2c530999b..2c530999b 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/legacy/dotgraph.cc diff --git a/src/nix-store/dotgraph.hh b/src/legacy/dotgraph.hh index 4fd944080..4fd944080 100644 --- a/src/nix-store/dotgraph.hh +++ b/src/legacy/dotgraph.hh diff --git a/src/nix-store/graphml.cc b/src/legacy/graphml.cc index 3e789a2d8..3e789a2d8 100644 --- a/src/nix-store/graphml.cc +++ b/src/legacy/graphml.cc diff --git a/src/nix-store/graphml.hh b/src/legacy/graphml.hh index bd3a4a37c..bd3a4a37c 100644 --- a/src/nix-store/graphml.hh +++ b/src/legacy/graphml.hh diff --git a/src/legacy/meson.build b/src/legacy/meson.build new file mode 100644 index 000000000..13b90314c --- /dev/null +++ b/src/legacy/meson.build @@ -0,0 +1,35 @@ +legacy_include_directories = include_directories('.') + +legacy_sources = files( + # `build-remote` is not really legacy (it powers all remote builds), but it's + # not a `nix3` command. + 'build-remote.cc', + 'dotgraph.cc', + 'graphml.cc', + 'nix-build.cc', + 'nix-channel.cc', + 'nix-collect-garbage.cc', + 'nix-copy-closure.cc', + 'nix-env.cc', + 'nix-env.hh', + 'nix-instantiate.cc', + 'nix-store.cc', + 'user-env.cc', +) + +legacy_headers = files( + 'build-remote.hh', + 'nix-build.hh', + 'nix-channel.hh', + 'nix-collect-garbage.hh', + 'nix-copy-closure.hh', + 'nix-instantiate.hh', + 'nix-store.hh', +) + +legacy_generated_headers = [ + gen_header.process('buildenv.nix', preserve_path_from: meson.current_source_dir()), + gen_header.process('unpack-channel.nix', preserve_path_from: meson.current_source_dir()), +] + +fs.copyfile('unpack-channel.nix') diff --git a/src/nix-build/nix-build.cc b/src/legacy/nix-build.cc index 4b8d7a2fa..eb9b6cd8d 100644 --- a/src/nix-build/nix-build.cc +++ b/src/legacy/nix-build.cc @@ -24,11 +24,13 @@ #include "attr-path.hh" #include "legacy.hh" #include "shlex.hh" +#include "nix-build.hh" -using namespace nix; -using namespace std::string_literals; +extern char * * environ __attribute__((weak)); // Man what even is this + +namespace nix { -extern char * * environ __attribute__((weak)); +using namespace std::string_literals; static void main_nix_build(int argc, char * * argv) { @@ -613,5 +615,9 @@ static void main_nix_build(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_build("nix-build", main_nix_build); -static RegisterLegacyCommand r_nix_shell("nix-shell", main_nix_build); +void registerNixBuildAndNixShell() { + LegacyCommands::add("nix-build", main_nix_build); + LegacyCommands::add("nix-shell", main_nix_build); +} + +} diff --git a/src/legacy/nix-build.hh b/src/legacy/nix-build.hh new file mode 100644 index 000000000..945ac06e2 --- /dev/null +++ b/src/legacy/nix-build.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixBuildAndNixShell(); + +} diff --git a/src/nix-channel/nix-channel.cc b/src/legacy/nix-channel.cc index 971337b63..2f79919dd 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/legacy/nix-channel.cc @@ -7,12 +7,13 @@ #include "fetchers.hh" #include "eval-settings.hh" // for defexpr #include "users.hh" +#include "nix-channel.hh" #include <fcntl.h> #include <regex> #include <pwd.h> -using namespace nix; +namespace nix { typedef std::map<std::string, std::string> Channels; @@ -264,4 +265,8 @@ static int main_nix_channel(int argc, char ** argv) } } -static RegisterLegacyCommand r_nix_channel("nix-channel", main_nix_channel); +void registerNixChannel() { + LegacyCommands::add("nix-channel", main_nix_channel); +} + +} diff --git a/src/legacy/nix-channel.hh b/src/legacy/nix-channel.hh new file mode 100644 index 000000000..f1583767f --- /dev/null +++ b/src/legacy/nix-channel.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixChannel(); + +} diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/legacy/nix-collect-garbage.cc index c831f132f..7640100a0 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/legacy/nix-collect-garbage.cc @@ -7,11 +7,12 @@ #include "globals.hh" #include "legacy.hh" #include "signals.hh" +#include "nix-collect-garbage.hh" #include <iostream> #include <cerrno> -using namespace nix; +namespace nix { std::string deleteOlderThan; bool dryRun = false; @@ -110,4 +111,8 @@ static int main_nix_collect_garbage(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_collect_garbage("nix-collect-garbage", main_nix_collect_garbage); +void registerNixCollectGarbage() { + LegacyCommands::add("nix-collect-garbage", main_nix_collect_garbage); +} + +} diff --git a/src/legacy/nix-collect-garbage.hh b/src/legacy/nix-collect-garbage.hh new file mode 100644 index 000000000..68515b537 --- /dev/null +++ b/src/legacy/nix-collect-garbage.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixCollectGarbage(); + +} diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/legacy/nix-copy-closure.cc index 7f2bb93b6..1e61e3c48 100644 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/legacy/nix-copy-closure.cc @@ -1,8 +1,9 @@ #include "shared.hh" #include "store-api.hh" #include "legacy.hh" +#include "nix-copy-closure.hh" -using namespace nix; +namespace nix { static int main_nix_copy_closure(int argc, char ** argv) { @@ -60,4 +61,8 @@ static int main_nix_copy_closure(int argc, char ** argv) } } -static RegisterLegacyCommand r_nix_copy_closure("nix-copy-closure", main_nix_copy_closure); +void registerNixCopyClosure() { + LegacyCommands::add("nix-copy-closure", main_nix_copy_closure); +} + +} diff --git a/src/legacy/nix-copy-closure.hh b/src/legacy/nix-copy-closure.hh new file mode 100644 index 000000000..fb5d0fc6e --- /dev/null +++ b/src/legacy/nix-copy-closure.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixCopyClosure(); + +} diff --git a/src/nix-env/nix-env.cc b/src/legacy/nix-env.cc index 13fadb1d8..4674fd783 100644 --- a/src/nix-env/nix-env.cc +++ b/src/legacy/nix-env.cc @@ -17,6 +17,7 @@ #include "xml-writer.hh" #include "legacy.hh" #include "eval-settings.hh" // for defexpr +#include "nix-env.hh" #include <ctime> #include <algorithm> @@ -28,9 +29,10 @@ #include <unistd.h> #include <nlohmann/json.hpp> -using namespace nix; using std::cout; +namespace nix { + typedef enum { srcNixExprDrvs, @@ -1544,4 +1546,8 @@ static int main_nix_env(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_env("nix-env", main_nix_env); +void registerNixEnv() { + LegacyCommands::add("nix-env", main_nix_env); +} + +} diff --git a/src/legacy/nix-env.hh b/src/legacy/nix-env.hh new file mode 100644 index 000000000..47d62b8e6 --- /dev/null +++ b/src/legacy/nix-env.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixEnv(); + +} diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/legacy/nix-instantiate.cc index 7487af1c1..5d1da0d70 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/legacy/nix-instantiate.cc @@ -11,12 +11,13 @@ #include "local-fs-store.hh" #include "common-eval-args.hh" #include "legacy.hh" +#include "nix-instantiate.hh" #include <map> #include <iostream> -using namespace nix; +namespace nix { static Path gcRoot; @@ -195,4 +196,8 @@ static int main_nix_instantiate(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_instantiate("nix-instantiate", main_nix_instantiate); +void registerNixInstantiate() { + LegacyCommands::add("nix-instantiate", main_nix_instantiate); +} + +} diff --git a/src/legacy/nix-instantiate.hh b/src/legacy/nix-instantiate.hh new file mode 100644 index 000000000..f4c35a6b5 --- /dev/null +++ b/src/legacy/nix-instantiate.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixInstantiate(); + +} diff --git a/src/nix-store/nix-store.cc b/src/legacy/nix-store.cc index bc43aa7b1..e42aa4065 100644 --- a/src/nix-store/nix-store.cc +++ b/src/legacy/nix-store.cc @@ -15,6 +15,7 @@ #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" +#include "nix-store.hh" #include <iostream> #include <algorithm> @@ -24,10 +25,9 @@ #include <fcntl.h> -namespace nix_store { +namespace nix { -using namespace nix; using std::cin; using std::cout; @@ -1176,6 +1176,8 @@ static int main_nix_store(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_store("nix-store", main_nix_store); +void registerNixStore() { + LegacyCommands::add("nix-store", main_nix_store); +} } diff --git a/src/legacy/nix-store.hh b/src/legacy/nix-store.hh new file mode 100644 index 000000000..b010e7b19 --- /dev/null +++ b/src/legacy/nix-store.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixStore(); + +} diff --git a/src/nix-channel/unpack-channel.nix b/src/legacy/unpack-channel.nix index 84e324a4d..84e324a4d 100644 --- a/src/nix-channel/unpack-channel.nix +++ b/src/legacy/unpack-channel.nix diff --git a/src/nix-env/user-env.cc b/src/legacy/user-env.cc index f5dbd06ca..f5dbd06ca 100644 --- a/src/nix-env/user-env.cc +++ b/src/legacy/user-env.cc diff --git a/src/nix-env/user-env.hh b/src/legacy/user-env.hh index af45d2d85..af45d2d85 100644 --- a/src/nix-env/user-env.hh +++ b/src/legacy/user-env.hh diff --git a/src/libcmd/legacy.cc b/src/libcmd/legacy.cc index 6df09ee37..8bbe9b031 100644 --- a/src/libcmd/legacy.cc +++ b/src/libcmd/legacy.cc @@ -2,6 +2,6 @@ namespace nix { -RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0; +LegacyCommands::Commands * LegacyCommands::commands = 0; } diff --git a/src/libcmd/legacy.hh b/src/libcmd/legacy.hh index 357500a4d..45a231983 100644 --- a/src/libcmd/legacy.hh +++ b/src/libcmd/legacy.hh @@ -9,12 +9,12 @@ namespace nix { typedef std::function<void(int, char * *)> MainFunction; -struct RegisterLegacyCommand +struct LegacyCommands { typedef std::map<std::string, MainFunction> Commands; static Commands * commands; - RegisterLegacyCommand(const std::string & name, MainFunction fun) + static void add(const std::string & name, MainFunction fun) { if (!commands) commands = new Commands; (*commands)[name] = fun; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 83bfd4fb0..e7336c7e8 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -79,7 +79,7 @@ struct AttrDb state->txn->commit(); state->txn.reset(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -90,7 +90,7 @@ struct AttrDb try { return fun(); } catch (SQLiteError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); failed = true; return 0; } @@ -329,7 +329,7 @@ static std::shared_ptr<AttrDb> makeAttrDb( try { return std::make_shared<AttrDb>(cfg, fingerprint, symbols); } catch (SQLiteError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); return nullptr; } } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 68da254e2..0c1a1ec0e 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -21,6 +21,14 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) return printIdentifier(str, s); } +AttrName::AttrName(Symbol s) : symbol(s) +{ +} + +AttrName::AttrName(std::unique_ptr<Expr> e) : expr(std::move(e)) +{ +} + void Expr::show(const SymbolTable & symbols, std::ostream & str) const { abort(); @@ -239,9 +247,24 @@ void ExprConcatStrings::show(const SymbolTable & symbols, std::ostream & str) co { bool first = true; str << "("; - for (auto & i : es) { - if (first) first = false; else str << " + "; - i.second->show(symbols, str); + for (auto & [_pos, part] : es) { + if (first) + first = false; + else + str << " + "; + + if (forceString && !dynamic_cast<ExprString *>(part.get())) { + /* Print as a string with an interpolation, to preserve the + * semantics of the value having to be a string. + * Interpolations are weird and someone should eventually + * move them out into their own AST node please. + */ + str << "\"${"; + part->show(symbols, str); + str << "}\""; + } else { + part->show(symbols, str); + } } str << ")"; } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index d16281c39..4e857b321 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -30,8 +30,8 @@ struct AttrName { Symbol symbol; std::unique_ptr<Expr> expr; - AttrName(Symbol s) : symbol(s) {}; - AttrName(std::unique_ptr<Expr> e) : expr(std::move(e)) {}; + AttrName(Symbol s); + AttrName(std::unique_ptr<Expr> e); }; typedef std::vector<AttrName> AttrPath; diff --git a/src/libexpr/parser/grammar.hh b/src/libexpr/parser/grammar.hh index 2c5a3d1be..701b40505 100644 --- a/src/libexpr/parser/grammar.hh +++ b/src/libexpr/parser/grammar.hh @@ -12,7 +12,7 @@ // eolf rules in favor of reproducing the old flex lexer as faithfully as // possible, and deferring calculation of positions to downstream users. -namespace nix::parser::grammar { +namespace nix::parser::grammar::v1 { using namespace tao::pegtl; namespace p = tao::pegtl; @@ -225,7 +225,8 @@ struct string : _string, seq< > {}; struct _ind_string { - template<bool Indented, typename... Inner> + struct line_start : semantic, star<one<' '>> {}; + template<typename... Inner> struct literal : semantic, seq<Inner...> {}; struct interpolation : semantic, seq< p::string<'$', '{'>, seps, @@ -233,34 +234,53 @@ struct _ind_string { must<one<'}'>> > {}; struct escape : semantic, must<any> {}; + /* Marker for non-empty lines */ + struct has_content : semantic, seq<> {}; }; struct ind_string : _ind_string, seq< TAO_PEGTL_STRING("''"), + // Strip first line completely if empty opt<star<one<' '>>, one<'\n'>>, - star< - sor< - _ind_string::literal< - true, + list< + seq< + // Start a line with some indentation + // (we always match even the empty string if no indentation, as this creates the line) + _ind_string::line_start, + // The actual line + opt< plus< sor< - not_one<'$', '\''>, - seq<one<'$'>, not_one<'{', '\''>>, - seq<one<'\''>, not_one<'\'', '$'>> - > - > - >, - _ind_string::interpolation, - _ind_string::literal<false, one<'$'>>, - _ind_string::literal<false, one<'\''>, not_at<one<'\''>>>, - seq<one<'\''>, _ind_string::literal<false, p::string<'\'', '\''>>>, - seq< - p::string<'\'', '\''>, - sor< - _ind_string::literal<false, one<'$'>>, - seq<one<'\\'>, _ind_string::escape> + _ind_string::literal< + plus< + sor< + not_one<'$', '\'', '\n'>, + // TODO probably factor this out like the others for performance + seq<one<'$'>, not_one<'{', '\'', '\n'>>, + seq<one<'$'>, at<one<'\n'>>>, + seq<one<'\''>, not_one<'\'', '$', '\n'>>, + seq<one<'\''>, at<one<'\n'>>> + > + > + >, + _ind_string::interpolation, + _ind_string::literal<one<'$'>>, + _ind_string::literal<one<'\''>, not_at<one<'\''>>>, + seq<one<'\''>, _ind_string::literal<p::string<'\'', '\''>>>, + seq< + p::string<'\'', '\''>, + sor< + _ind_string::literal<one<'$'>>, + seq<one<'\\'>, _ind_string::escape> + > + > + >, + _ind_string::has_content > > - > + >, + // End of line, LF. CR is just ignored and not treated as ending a line + // (for the purpose of indentation stripping) + _ind_string::literal<one<'\n'>> >, must<TAO_PEGTL_STRING("''")> > {}; @@ -352,10 +372,10 @@ struct formals : semantic, _formals, seq< struct _attr { struct simple : semantic, sor<t::identifier, t::kw_or> {}; - struct string : semantic, seq<grammar::string> {}; + struct string : semantic, seq<grammar::v1::string> {}; struct expr : semantic, seq< TAO_PEGTL_STRING("${"), seps, - must<grammar::expr>, seps, + must<grammar::v1::expr>, seps, must<one<'}'>> > {}; }; @@ -452,9 +472,9 @@ struct _expr { struct id : semantic, t::identifier {}; struct int_ : semantic, t::integer {}; struct float_ : semantic, t::floating {}; - struct string : semantic, seq<grammar::string> {}; - struct ind_string : semantic, seq<grammar::ind_string> {}; - struct path : semantic, seq<grammar::path> {}; + struct string : semantic, seq<grammar::v1::string> {}; + struct ind_string : semantic, seq<grammar::v1::ind_string> {}; + struct path : semantic, seq<grammar::v1::path> {}; struct uri : semantic, t::uri {}; struct ancient_let : semantic, _attrset<must, t::kw_let, seps> {}; struct rec_set : semantic, _attrset<must, t::kw_rec, seps> {}; @@ -628,34 +648,34 @@ struct nothing : p::nothing<Rule> { template<typename Self, typename OpCtx, typename AttrPathT, typename ExprT> struct operator_semantics { - struct has_attr : grammar::op::has_attr { + struct has_attr : grammar::v1::op::has_attr { AttrPathT path; }; struct OpEntry { OpCtx ctx; uint8_t prec; - grammar::op::kind assoc; + grammar::v1::op::kind assoc; std::variant< - grammar::op::not_, - grammar::op::unary_minus, - grammar::op::implies, - grammar::op::or_, - grammar::op::and_, - grammar::op::equals, - grammar::op::not_equals, - grammar::op::less_eq, - grammar::op::greater_eq, - grammar::op::update, - grammar::op::concat, - grammar::op::less, - grammar::op::greater, - grammar::op::plus, - grammar::op::minus, - grammar::op::mul, - grammar::op::div, - grammar::op::pipe_right, - grammar::op::pipe_left, + grammar::v1::op::not_, + grammar::v1::op::unary_minus, + grammar::v1::op::implies, + grammar::v1::op::or_, + grammar::v1::op::and_, + grammar::v1::op::equals, + grammar::v1::op::not_equals, + grammar::v1::op::less_eq, + grammar::v1::op::greater_eq, + grammar::v1::op::update, + grammar::v1::op::concat, + grammar::v1::op::less, + grammar::v1::op::greater, + grammar::v1::op::plus, + grammar::v1::op::minus, + grammar::v1::op::mul, + grammar::v1::op::div, + grammar::v1::op::pipe_right, + grammar::v1::op::pipe_left, has_attr > op; }; @@ -676,7 +696,7 @@ struct operator_semantics { auto & [ctx, precedence, kind, op] = ops.back(); // NOTE this relies on associativity not being mixed within a precedence level. if ((precedence > toPrecedence) - || (kind != grammar::op::kind::leftAssoc && precedence == toPrecedence)) + || (kind != grammar::v1::op::kind::leftAssoc && precedence == toPrecedence)) break; std::visit([&, ctx=std::move(ctx)] (auto & op) { exprs.push_back(static_cast<Self &>(*this).applyOp(ctx, op, args...)); @@ -694,9 +714,9 @@ struct operator_semantics { void pushOp(OpCtx ctx, auto o, auto &... args) { - if (o.kind != grammar::op::kind::unary) + if (o.kind != grammar::v1::op::kind::unary) reduce(o.precedence, args...); - if (!ops.empty() && o.kind == grammar::op::kind::nonAssoc) { + if (!ops.empty() && o.kind == grammar::v1::op::kind::nonAssoc) { auto & [_pos, _prec, _kind, _o] = ops.back(); if (_kind == o.kind && _prec == o.precedence) Self::badOperator(ctx, args...); diff --git a/src/libexpr/parser/parser-impl1.inc.cc b/src/libexpr/parser/parser-impl1.inc.cc new file mode 100644 index 000000000..c937d17fd --- /dev/null +++ b/src/libexpr/parser/parser-impl1.inc.cc @@ -0,0 +1,863 @@ +// flip this define when doing parser development to enable some g checks. +#if 0 +#include <tao/pegtl/contrib/analyze.hpp> +#define ANALYZE_GRAMMAR \ + ([] { \ + const std::size_t issues = tao::pegtl::analyze<grammar::v1::root>(); \ + assert(issues == 0); \ + })() +#else +#define ANALYZE_GRAMMAR ((void) 0) +#endif + +namespace p = tao::pegtl; + +namespace nix::parser::v1 { +namespace { + +template<typename> +inline constexpr const char * error_message = nullptr; + +#define error_message_for(...) \ + template<> inline constexpr auto error_message<__VA_ARGS__> + +error_message_for(p::one<'{'>) = "expecting '{'"; +error_message_for(p::one<'}'>) = "expecting '}'"; +error_message_for(p::one<'"'>) = "expecting '\"'"; +error_message_for(p::one<';'>) = "expecting ';'"; +error_message_for(p::one<')'>) = "expecting ')'"; +error_message_for(p::one<']'>) = "expecting ']'"; +error_message_for(p::one<':'>) = "expecting ':'"; +error_message_for(p::string<'\'', '\''>) = "expecting \"''\""; +error_message_for(p::any) = "expecting any character"; +error_message_for(grammar::v1::eof) = "expecting end of file"; +error_message_for(grammar::v1::seps) = "expecting separators"; +error_message_for(grammar::v1::path::forbid_prefix_triple_slash) = "too many slashes in path"; +error_message_for(grammar::v1::path::forbid_prefix_double_slash_no_interp) = "path has a trailing slash"; +error_message_for(grammar::v1::expr) = "expecting expression"; +error_message_for(grammar::v1::expr::unary) = "expecting expression"; +error_message_for(grammar::v1::binding::equal) = "expecting '='"; +error_message_for(grammar::v1::expr::lambda::arg) = "expecting identifier"; +error_message_for(grammar::v1::formals) = "expecting formals"; +error_message_for(grammar::v1::attrpath) = "expecting attribute path"; +error_message_for(grammar::v1::expr::select) = "expecting selection expression"; +error_message_for(grammar::v1::t::kw_then) = "expecting 'then'"; +error_message_for(grammar::v1::t::kw_else) = "expecting 'else'"; +error_message_for(grammar::v1::t::kw_in) = "expecting 'in'"; + +struct SyntaxErrors +{ + template<typename Rule> + static constexpr auto message = error_message<Rule>; + + template<typename Rule> + static constexpr bool raise_on_failure = false; +}; + +template<typename Rule> +struct Control : p::must_if<SyntaxErrors>::control<Rule> +{ + template<typename ParseInput, typename... States> + [[noreturn]] static void raise(const ParseInput & in, States &&... st) + { + if (in.empty()) { + std::string expected; + if constexpr (constexpr auto msg = error_message<Rule>) + expected = fmt(", %s", msg); + throw p::parse_error("unexpected end of file" + expected, in); + } + p::must_if<SyntaxErrors>::control<Rule>::raise(in, st...); + } +}; + +struct ExprState + : grammar::v1:: + operator_semantics<ExprState, PosIdx, AttrPath, std::pair<PosIdx, std::unique_ptr<Expr>>> +{ + std::unique_ptr<Expr> popExprOnly() { + return std::move(popExpr().second); + } + + template<typename Op, typename... Args> + std::unique_ptr<Expr> applyUnary(Args &&... args) { + return std::make_unique<Op>(popExprOnly(), std::forward<Args>(args)...); + } + + template<typename Op> + std::unique_ptr<Expr> applyBinary(PosIdx pos) { + auto right = popExprOnly(), left = popExprOnly(); + return std::make_unique<Op>(pos, std::move(left), std::move(right)); + } + + std::unique_ptr<Expr> call(PosIdx pos, Symbol fn, bool flip = false) + { + std::vector<std::unique_ptr<Expr>> args(2); + args[flip ? 0 : 1] = popExprOnly(); + args[flip ? 1 : 0] = popExprOnly(); + return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args)); + } + + std::unique_ptr<Expr> pipe(PosIdx pos, State & state, bool flip = false) + { + if (!state.featureSettings.isEnabled(Xp::PipeOperator)) + throw ParseError({ + .msg = HintFmt("Pipe operator is disabled"), + .pos = state.positions[pos] + }); + + // Reverse the order compared to normal function application: arg |> fn + std::unique_ptr<Expr> fn, arg; + if (flip) { + fn = popExprOnly(); + arg = popExprOnly(); + } else { + arg = popExprOnly(); + fn = popExprOnly(); + } + std::vector<std::unique_ptr<Expr>> args{1}; + args[0] = std::move(arg); + + return std::make_unique<ExprCall>(pos, std::move(fn), std::move(args)); + } + + std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state) + { + return call(pos, state.s.lessThan, !less); + } + + std::unique_ptr<Expr> concatStrings(PosIdx pos) + { + std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> args(2); + args[1] = popExpr(); + args[0] = popExpr(); + return std::make_unique<ExprConcatStrings>(pos, false, std::move(args)); + } + + std::unique_ptr<Expr> negate(PosIdx pos, State & state) + { + std::vector<std::unique_ptr<Expr>> args(2); + args[0] = std::make_unique<ExprInt>(0); + args[1] = popExprOnly(); + return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(state.s.sub), std::move(args)); + } + + std::pair<PosIdx, std::unique_ptr<Expr>> applyOp(PosIdx pos, auto & op, State & state) { + using Op = grammar::v1::op; + + auto not_ = [] (auto e) { + return std::make_unique<ExprOpNot>(std::move(e)); + }; + + return { + pos, + (overloaded { + [&] (Op::implies) { return applyBinary<ExprOpImpl>(pos); }, + [&] (Op::or_) { return applyBinary<ExprOpOr>(pos); }, + [&] (Op::and_) { return applyBinary<ExprOpAnd>(pos); }, + [&] (Op::equals) { return applyBinary<ExprOpEq>(pos); }, + [&] (Op::not_equals) { return applyBinary<ExprOpNEq>(pos); }, + [&] (Op::less) { return order(pos, true, state); }, + [&] (Op::greater_eq) { return not_(order(pos, true, state)); }, + [&] (Op::greater) { return order(pos, false, state); }, + [&] (Op::less_eq) { return not_(order(pos, false, state)); }, + [&] (Op::update) { return applyBinary<ExprOpUpdate>(pos); }, + [&] (Op::not_) { return applyUnary<ExprOpNot>(); }, + [&] (Op::plus) { return concatStrings(pos); }, + [&] (Op::minus) { return call(pos, state.s.sub); }, + [&] (Op::mul) { return call(pos, state.s.mul); }, + [&] (Op::div) { return call(pos, state.s.div); }, + [&] (Op::concat) { return applyBinary<ExprOpConcatLists>(pos); }, + [&] (has_attr & a) { return applyUnary<ExprOpHasAttr>(std::move(a.path)); }, + [&] (Op::unary_minus) { return negate(pos, state); }, + [&] (Op::pipe_right) { return pipe(pos, state, true); }, + [&] (Op::pipe_left) { return pipe(pos, state); }, + })(op) + }; + } + + // always_inline is needed, otherwise pushOp slows down considerably + [[noreturn, gnu::always_inline]] + static void badOperator(PosIdx pos, State & state) + { + throw ParseError({ + .msg = HintFmt("syntax error, unexpected operator"), + .pos = state.positions[pos] + }); + } + + template<typename Expr, typename... Args> + Expr & pushExpr(PosIdx pos, Args && ... args) + { + auto p = std::make_unique<Expr>(std::forward<Args>(args)...); + auto & result = *p; + exprs.emplace_back(pos, std::move(p)); + return result; + } +}; + +struct SubexprState { +private: + ExprState * up; + +public: + explicit SubexprState(ExprState & up, auto &...) : up(&up) {} + operator ExprState &() { return *up; } + ExprState * operator->() { return up; } +}; + + + +template<typename Rule> +struct BuildAST : grammar::v1::nothing<Rule> {}; + +struct LambdaState : SubexprState { + using SubexprState::SubexprState; + + Symbol arg; + std::unique_ptr<Formals> formals; +}; + +struct FormalsState : SubexprState { + using SubexprState::SubexprState; + + Formals formals{}; + Formal formal{}; +}; + +template<> struct BuildAST<grammar::v1::formal::name> { + static void apply(const auto & in, FormalsState & s, State & ps) { + s.formal = { + .pos = ps.at(in), + .name = ps.symbols.create(in.string_view()), + }; + } +}; + +template<> struct BuildAST<grammar::v1::formal> { + static void apply0(FormalsState & s, State &) { + s.formals.formals.emplace_back(std::move(s.formal)); + } +}; + +template<> struct BuildAST<grammar::v1::formal::default_value> { + static void apply0(FormalsState & s, State & ps) { + s.formal.def = s->popExprOnly(); + } +}; + +template<> struct BuildAST<grammar::v1::formals::ellipsis> { + static void apply0(FormalsState & s, State &) { + s.formals.ellipsis = true; + } +}; + +template<> struct BuildAST<grammar::v1::formals> : change_head<FormalsState> { + static void success0(FormalsState & f, LambdaState & s, State &) { + s.formals = std::make_unique<Formals>(std::move(f.formals)); + } +}; + +struct AttrState : SubexprState { + using SubexprState::SubexprState; + + std::vector<AttrName> attrs; + + template <typename T> + void pushAttr(T && attr, PosIdx) { attrs.emplace_back(std::forward<T>(attr)); } +}; + +template<> struct BuildAST<grammar::v1::attr::simple> { + static void apply(const auto & in, auto & s, State & ps) { + s.pushAttr(ps.symbols.create(in.string_view()), ps.at(in)); + } +}; + +template<> struct BuildAST<grammar::v1::attr::string> { + static void apply(const auto & in, auto & s, State & ps) { + auto e = s->popExprOnly(); + if (auto str = dynamic_cast<ExprString *>(e.get())) + s.pushAttr(ps.symbols.create(str->s), ps.at(in)); + else + s.pushAttr(std::move(e), ps.at(in)); + } +}; + +template<> struct BuildAST<grammar::v1::attr::expr> : BuildAST<grammar::v1::attr::string> {}; + +struct BindingsState : SubexprState { + using SubexprState::SubexprState; + + ExprAttrs attrs; + AttrPath path; + std::unique_ptr<Expr> value; +}; + +struct InheritState : SubexprState { + using SubexprState::SubexprState; + + std::vector<std::pair<AttrName, PosIdx>> attrs; + std::unique_ptr<Expr> from; + PosIdx fromPos; + + template <typename T> + void pushAttr(T && attr, PosIdx pos) { attrs.emplace_back(std::forward<T>(attr), pos); } +}; + +template<> struct BuildAST<grammar::v1::inherit::from> { + static void apply(const auto & in, InheritState & s, State & ps) { + s.from = s->popExprOnly(); + s.fromPos = ps.at(in); + } +}; + +template<> struct BuildAST<grammar::v1::inherit> : change_head<InheritState> { + static void success0(InheritState & s, BindingsState & b, State & ps) { + auto & attrs = b.attrs.attrs; + // TODO this should not reuse generic attrpath rules. + for (auto & [i, iPos] : s.attrs) { + if (i.symbol) + continue; + if (auto str = dynamic_cast<ExprString *>(i.expr.get())) + i = AttrName(ps.symbols.create(str->s)); + else { + throw ParseError({ + .msg = HintFmt("dynamic attributes not allowed in inherit"), + .pos = ps.positions[iPos] + }); + } + } + if (s.from != nullptr) { + if (!b.attrs.inheritFromExprs) + b.attrs.inheritFromExprs = std::make_unique<std::vector<ref<Expr>>>(); + auto fromExpr = ref<Expr>(std::move(s.from)); + b.attrs.inheritFromExprs->push_back(fromExpr); + for (auto & [i, iPos] : s.attrs) { + if (attrs.find(i.symbol) != attrs.end()) + ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); + auto inheritFrom = std::make_unique<ExprInheritFrom>( + s.fromPos, + b.attrs.inheritFromExprs->size() - 1, + fromExpr + ); + attrs.emplace( + i.symbol, + ExprAttrs::AttrDef( + std::make_unique<ExprSelect>(iPos, std::move(inheritFrom), i.symbol), + iPos, + ExprAttrs::AttrDef::Kind::InheritedFrom)); + } + } else { + for (auto & [i, iPos] : s.attrs) { + if (attrs.find(i.symbol) != attrs.end()) + ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); + attrs.emplace( + i.symbol, + ExprAttrs::AttrDef( + std::make_unique<ExprVar>(iPos, i.symbol), + iPos, + ExprAttrs::AttrDef::Kind::Inherited)); + } + } + } +}; + +template<> struct BuildAST<grammar::v1::binding::path> : change_head<AttrState> { + static void success0(AttrState & a, BindingsState & s, State & ps) { + s.path = std::move(a.attrs); + } +}; + +template<> struct BuildAST<grammar::v1::binding::value> { + static void apply0(BindingsState & s, State & ps) { + s.value = s->popExprOnly(); + } +}; + +template<> struct BuildAST<grammar::v1::binding> { + static void apply(const auto & in, BindingsState & s, State & ps) { + ps.addAttr(&s.attrs, std::move(s.path), std::move(s.value), ps.at(in)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::id> { + static void apply(const auto & in, ExprState & s, State & ps) { + if (in.string_view() == "__curPos") + s.pushExpr<ExprPos>(ps.at(in), ps.at(in)); + else + s.pushExpr<ExprVar>(ps.at(in), ps.at(in), ps.symbols.create(in.string_view())); + } +}; + +template<> struct BuildAST<grammar::v1::expr::int_> { + static void apply(const auto & in, ExprState & s, State & ps) { + int64_t v; + if (std::from_chars(in.begin(), in.end(), v).ec != std::errc{}) { + throw ParseError({ + .msg = HintFmt("invalid integer '%1%'", in.string_view()), + .pos = ps.positions[ps.at(in)], + }); + } + s.pushExpr<ExprInt>(noPos, v); + } +}; + +template<> struct BuildAST<grammar::v1::expr::float_> { + static void apply(const auto & in, ExprState & s, State & ps) { + // copy the input into a temporary string so we can call stod. + // can't use from_chars because libc++ (thus darwin) does not have it, + // and floats are not performance-sensitive anyway. if they were you'd + // be in much bigger trouble than this. + // + // we also get to do a locale-save dance because stod is locale-aware and + // something (a plugin?) may have called setlocale or uselocale. + static struct locale_hack { + locale_t posix; + locale_hack(): posix(newlocale(LC_ALL_MASK, "POSIX", 0)) + { + if (posix == 0) + throw SysError("could not get POSIX locale"); + } + } locale; + + auto tmp = in.string(); + double v = [&] { + auto oldLocale = uselocale(locale.posix); + Finally resetLocale([=] { uselocale(oldLocale); }); + try { + return std::stod(tmp); + } catch (...) { + throw ParseError({ + .msg = HintFmt("invalid float '%1%'", in.string_view()), + .pos = ps.positions[ps.at(in)], + }); + } + }(); + s.pushExpr<ExprFloat>(noPos, v); + } +}; + +struct StringState : SubexprState { + using SubexprState::SubexprState; + + std::string currentLiteral; + PosIdx currentPos; + std::vector<std::pair<nix::PosIdx, std::unique_ptr<Expr>>> parts; + + void append(PosIdx pos, std::string_view s) + { + if (currentLiteral.empty()) + currentPos = pos; + currentLiteral += s; + } + + // FIXME this truncates strings on NUL for compat with the old parser. ideally + // we should use the decomposition the g gives us instead of iterating over + // the entire string again. + static void unescapeStr(std::string & str) + { + char * s = str.data(); + char * t = s; + char c; + while ((c = *s++)) { + if (c == '\\') { + c = *s++; + if (c == 'n') *t = '\n'; + else if (c == 'r') *t = '\r'; + else if (c == 't') *t = '\t'; + else *t = c; + } + else if (c == '\r') { + /* Normalise CR and CR/LF into LF. */ + *t = '\n'; + if (*s == '\n') s++; /* cr/lf */ + } + else *t = c; + t++; + } + str.resize(t - str.data()); + } + + void endLiteral() + { + if (!currentLiteral.empty()) { + unescapeStr(currentLiteral); + parts.emplace_back(currentPos, std::make_unique<ExprString>(std::move(currentLiteral))); + } + } + + std::unique_ptr<Expr> finish() + { + if (parts.empty()) { + unescapeStr(currentLiteral); + return std::make_unique<ExprString>(std::move(currentLiteral)); + } else { + endLiteral(); + auto pos = parts[0].first; + return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts)); + } + } +}; + +template<typename... Content> struct BuildAST<grammar::v1::string::literal<Content...>> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::string::cr_lf> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), in.string_view()); // FIXME compat with old parser + } +}; + +template<> struct BuildAST<grammar::v1::string::interpolation> { + static void apply(const auto & in, StringState & s, State & ps) { + s.endLiteral(); + s.parts.emplace_back(ps.at(in), s->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::string::escape> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), "\\"); // FIXME compat with old parser + s.append(ps.at(in), in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::string> : change_head<StringState> { + static void success0(StringState & s, ExprState & e, State &) { + e.exprs.emplace_back(noPos, s.finish()); + } +}; + +struct IndStringState : SubexprState { + using SubexprState::SubexprState; + + std::vector<IndStringLine> lines; +}; + +template<> struct BuildAST<grammar::v1::ind_string::line_start> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.push_back(IndStringLine { in.string_view(), ps.at(in) }); + } +}; + +template<typename... Content> +struct BuildAST<grammar::v1::ind_string::literal<Content...>> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.back().parts.emplace_back(ps.at(in), in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::ind_string::interpolation> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.back().parts.emplace_back(ps.at(in), s->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::ind_string::escape> { + static void apply(const auto & in, IndStringState & s, State & ps) { + switch (*in.begin()) { + case 'n': s.lines.back().parts.emplace_back(ps.at(in), "\n"); break; + case 'r': s.lines.back().parts.emplace_back(ps.at(in), "\r"); break; + case 't': s.lines.back().parts.emplace_back(ps.at(in), "\t"); break; + default: s.lines.back().parts.emplace_back(ps.at(in), in.string_view()); break; + } + } +}; + +template<> struct BuildAST<grammar::v1::ind_string::has_content> { + static void apply(const auto & in, IndStringState & s, State & ps) { + s.lines.back().hasContent = true; + } +}; + +template<> struct BuildAST<grammar::v1::ind_string> : change_head<IndStringState> { + static void success(const auto & in, IndStringState & s, ExprState & e, State & ps) { + e.exprs.emplace_back(noPos, ps.stripIndentation(ps.at(in), std::move(s.lines))); + } +}; + +template<typename... Content> struct BuildAST<grammar::v1::path::literal<Content...>> { + static void apply(const auto & in, StringState & s, State & ps) { + s.append(ps.at(in), in.string_view()); + s.endLiteral(); + } +}; + +template<> struct BuildAST<grammar::v1::path::interpolation> : BuildAST<grammar::v1::string::interpolation> {}; + +template<> struct BuildAST<grammar::v1::path::anchor> { + static void apply(const auto & in, StringState & s, State & ps) { + Path path(absPath(in.string(), ps.basePath.path.abs())); + /* add back in the trailing '/' to the first segment */ + if (in.string_view().ends_with('/') && in.size() > 1) + path += "/"; + s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); + } +}; + +template<> struct BuildAST<grammar::v1::path::home_anchor> { + static void apply(const auto & in, StringState & s, State & ps) { + if (evalSettings.pureEval) + throw Error("the path '%s' can not be resolved in pure mode", in.string_view()); + Path path(getHome() + in.string_view().substr(1)); + s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); + } +}; + +template<> struct BuildAST<grammar::v1::path::searched_path> { + static void apply(const auto & in, StringState & s, State & ps) { + std::vector<std::unique_ptr<Expr>> args{2}; + args[0] = std::make_unique<ExprVar>(ps.s.nixPath); + args[1] = std::make_unique<ExprString>(in.string()); + s.parts.emplace_back( + ps.at(in), + std::make_unique<ExprCall>( + ps.at(in), + std::make_unique<ExprVar>(ps.s.findFile), + std::move(args))); + } +}; + +template<> struct BuildAST<grammar::v1::path> : change_head<StringState> { + template<typename E> + static void check_slash(PosIdx end, StringState & s, State & ps) { + auto e = dynamic_cast<E *>(s.parts.back().second.get()); + if (!e || !e->s.ends_with('/')) + return; + if (s.parts.size() > 1 || e->s != "/") + throw ParseError({ + .msg = HintFmt("path has a trailing slash"), + .pos = ps.positions[end], + }); + } + + static void success(const auto & in, StringState & s, ExprState & e, State & ps) { + s.endLiteral(); + check_slash<ExprPath>(ps.atEnd(in), s, ps); + check_slash<ExprString>(ps.atEnd(in), s, ps); + if (s.parts.size() == 1) { + e.exprs.emplace_back(noPos, std::move(s.parts.back().second)); + } else { + e.pushExpr<ExprConcatStrings>(ps.at(in), ps.at(in), false, std::move(s.parts)); + } + } +}; + +// strings and paths sare handled fully by the grammar-level rule for now +template<> struct BuildAST<grammar::v1::expr::string> : p::maybe_nothing {}; +template<> struct BuildAST<grammar::v1::expr::ind_string> : p::maybe_nothing {}; +template<> struct BuildAST<grammar::v1::expr::path> : p::maybe_nothing {}; + +template<> struct BuildAST<grammar::v1::expr::uri> { + static void apply(const auto & in, ExprState & s, State & ps) { + bool URLLiterals = ps.featureSettings.isEnabled(Dep::UrlLiterals); + if (!URLLiterals) + throw ParseError({ + .msg = HintFmt("URL literals are deprecated, allow using them with %s", "--extra-deprecated-features url-literals"), + .pos = ps.positions[ps.at(in)] + }); + s.pushExpr<ExprString>(ps.at(in), in.string()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::ancient_let> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + // Added 2024-09-18. Turn into an error at some point in the future. + // See the documentation on deprecated features for more details. + if (!ps.featureSettings.isEnabled(Dep::AncientLet)) + warn( + "%s found at %s. This feature is deprecated and will be removed in the future. Use %s to silence this warning.", + "let {", + ps.positions[ps.at(in)], + "--extra-deprecated-features ancient-let" + ); + + b.attrs.pos = ps.at(in); + b.attrs.recursive = true; + s.pushExpr<ExprSelect>(b.attrs.pos, b.attrs.pos, std::make_unique<ExprAttrs>(std::move(b.attrs)), ps.s.body); + } +}; + +template<> struct BuildAST<grammar::v1::expr::rec_set> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + // Before inserting new attrs, check for __override and throw an error + // (the error will initially be a warning to ease migration) + if (!featureSettings.isEnabled(Dep::RecSetOverrides) && b.attrs.attrs.contains(ps.s.overrides)) { + ps.overridesFound(ps.at(in)); + } + + b.attrs.pos = ps.at(in); + b.attrs.recursive = true; + s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::set> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + b.attrs.pos = ps.at(in); + s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); + } +}; + +using ListState = std::vector<std::unique_ptr<Expr>>; + +template<> struct BuildAST<grammar::v1::expr::list> : change_head<ListState> { + static void success(const auto & in, ListState & ls, ExprState & s, State & ps) { + auto e = std::make_unique<ExprList>(); + e->elems = std::move(ls); + s.exprs.emplace_back(ps.at(in), std::move(e)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::list::entry> : change_head<ExprState> { + static void success0(ExprState & e, ListState & s, State & ps) { + s.emplace_back(e.finish(ps).second); + } +}; + +struct SelectState : SubexprState { + using SubexprState::SubexprState; + + PosIdx pos; + ExprSelect * e = nullptr; +}; + +template<> struct BuildAST<grammar::v1::expr::select::head> { + static void apply(const auto & in, SelectState & s, State & ps) { + s.pos = ps.at(in); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select::attr> : change_head<AttrState> { + static void success0(AttrState & a, SelectState & s, State &) { + s.e = &s->pushExpr<ExprSelect>(s.pos, s.pos, s->popExprOnly(), std::move(a.attrs), nullptr); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select::attr_or> { + static void apply0(SelectState & s, State &) { + s.e->def = s->popExprOnly(); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select::as_app_or> { + static void apply(const auto & in, SelectState & s, State & ps) { + std::vector<std::unique_ptr<Expr>> args(1); + args[0] = std::make_unique<ExprVar>(ps.at(in), ps.s.or_); + s->pushExpr<ExprCall>(s.pos, s.pos, s->popExprOnly(), std::move(args)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::select> : change_head<SelectState> { + static void success0(const auto &...) {} +}; + +struct AppState : SubexprState { + using SubexprState::SubexprState; + + PosIdx pos; + ExprCall * e = nullptr; +}; + +template<> struct BuildAST<grammar::v1::expr::app::select_or_fn> { + static void apply(const auto & in, AppState & s, State & ps) { + s.pos = ps.at(in); + } +}; + +template<> struct BuildAST<grammar::v1::expr::app::first_arg> { + static void apply(auto & in, AppState & s, State & ps) { + auto arg = s->popExprOnly(), fn = s->popExprOnly(); + if ((s.e = dynamic_cast<ExprCall *>(fn.get()))) { + // TODO remove. + // AST compat with old parser, semantics are the same. + // this can happen on occasions such as `<p> <p>` or `a or b or`, + // neither of which are super worth optimizing. + s.e->args.push_back(std::move(arg)); + s->exprs.emplace_back(noPos, std::move(fn)); + } else { + std::vector<std::unique_ptr<Expr>> args{1}; + args[0] = std::move(arg); + s.e = &s->pushExpr<ExprCall>(s.pos, s.pos, std::move(fn), std::move(args)); + } + } +}; + +template<> struct BuildAST<grammar::v1::expr::app::another_arg> { + static void apply0(AppState & s, State & ps) { + s.e->args.push_back(s->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::app> : change_head<AppState> { + static void success0(const auto &...) {} +}; + +template<typename Op> struct BuildAST<grammar::v1::expr::operator_<Op>> { + static void apply(const auto & in, ExprState & s, State & ps) { + s.pushOp(ps.at(in), Op{}, ps); + } +}; +template<> struct BuildAST<grammar::v1::expr::operator_<grammar::v1::op::has_attr>> : change_head<AttrState> { + static void success(const auto & in, AttrState & a, ExprState & s, State & ps) { + s.pushOp(ps.at(in), ExprState::has_attr{{}, std::move(a.attrs)}, ps); + } +}; + +template<> struct BuildAST<grammar::v1::expr::lambda::arg> { + static void apply(const auto & in, LambdaState & s, State & ps) { + s.arg = ps.symbols.create(in.string_view()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::lambda> : change_head<LambdaState> { + static void success(const auto & in, LambdaState & l, ExprState & s, State & ps) { + if (l.formals) + l.formals = ps.validateFormals(std::move(l.formals), ps.at(in), l.arg); + s.pushExpr<ExprLambda>(ps.at(in), ps.at(in), l.arg, std::move(l.formals), l->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::assert_> { + static void apply(const auto & in, ExprState & s, State & ps) { + auto body = s.popExprOnly(), cond = s.popExprOnly(); + s.pushExpr<ExprAssert>(ps.at(in), ps.at(in), std::move(cond), std::move(body)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::with> { + static void apply(const auto & in, ExprState & s, State & ps) { + auto body = s.popExprOnly(), scope = s.popExprOnly(); + s.pushExpr<ExprWith>(ps.at(in), ps.at(in), std::move(scope), std::move(body)); + } +}; + +template<> struct BuildAST<grammar::v1::expr::let> : change_head<BindingsState> { + static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { + if (!b.attrs.dynamicAttrs.empty()) + throw ParseError({ + .msg = HintFmt("dynamic attributes not allowed in let"), + .pos = ps.positions[ps.at(in)] + }); + + s.pushExpr<ExprLet>(ps.at(in), std::make_unique<ExprAttrs>(std::move(b.attrs)), b->popExprOnly()); + } +}; + +template<> struct BuildAST<grammar::v1::expr::if_> { + static void apply(const auto & in, ExprState & s, State & ps) { + auto else_ = s.popExprOnly(), then = s.popExprOnly(), cond = s.popExprOnly(); + s.pushExpr<ExprIf>(ps.at(in), ps.at(in), std::move(cond), std::move(then), std::move(else_)); + } +}; + +template<> struct BuildAST<grammar::v1::expr> : change_head<ExprState> { + static void success0(ExprState & inner, ExprState & outer, State & ps) { + outer.exprs.push_back(inner.finish(ps)); + } +}; + +} +} diff --git a/src/libexpr/parser/parser.cc b/src/libexpr/parser/parser.cc index 17463056c..896a54981 100644 --- a/src/libexpr/parser/parser.cc +++ b/src/libexpr/parser/parser.cc @@ -14,857 +14,11 @@ #include <charconv> #include <memory> -// flip this define when doing parser development to enable some g checks. -#if 0 -#include <tao/pegtl/contrib/analyze.hpp> -#define ANALYZE_GRAMMAR \ - ([] { \ - const std::size_t issues = tao::pegtl::analyze<grammar::root>(); \ - assert(issues == 0); \ - })() -#else -#define ANALYZE_GRAMMAR ((void) 0) -#endif - -namespace p = tao::pegtl; - -namespace nix::parser { -namespace { - -template<typename> -inline constexpr const char * error_message = nullptr; - -#define error_message_for(...) \ - template<> inline constexpr auto error_message<__VA_ARGS__> - -error_message_for(p::one<'{'>) = "expecting '{'"; -error_message_for(p::one<'}'>) = "expecting '}'"; -error_message_for(p::one<'"'>) = "expecting '\"'"; -error_message_for(p::one<';'>) = "expecting ';'"; -error_message_for(p::one<')'>) = "expecting ')'"; -error_message_for(p::one<']'>) = "expecting ']'"; -error_message_for(p::one<':'>) = "expecting ':'"; -error_message_for(p::string<'\'', '\''>) = "expecting \"''\""; -error_message_for(p::any) = "expecting any character"; -error_message_for(grammar::eof) = "expecting end of file"; -error_message_for(grammar::seps) = "expecting separators"; -error_message_for(grammar::path::forbid_prefix_triple_slash) = "too many slashes in path"; -error_message_for(grammar::path::forbid_prefix_double_slash_no_interp) = "path has a trailing slash"; -error_message_for(grammar::expr) = "expecting expression"; -error_message_for(grammar::expr::unary) = "expecting expression"; -error_message_for(grammar::binding::equal) = "expecting '='"; -error_message_for(grammar::expr::lambda::arg) = "expecting identifier"; -error_message_for(grammar::formals) = "expecting formals"; -error_message_for(grammar::attrpath) = "expecting attribute path"; -error_message_for(grammar::expr::select) = "expecting selection expression"; -error_message_for(grammar::t::kw_then) = "expecting 'then'"; -error_message_for(grammar::t::kw_else) = "expecting 'else'"; -error_message_for(grammar::t::kw_in) = "expecting 'in'"; - -struct SyntaxErrors -{ - template<typename Rule> - static constexpr auto message = error_message<Rule>; - - template<typename Rule> - static constexpr bool raise_on_failure = false; -}; - -template<typename Rule> -struct Control : p::must_if<SyntaxErrors>::control<Rule> -{ - template<typename ParseInput, typename... States> - [[noreturn]] static void raise(const ParseInput & in, States &&... st) - { - if (in.empty()) { - std::string expected; - if constexpr (constexpr auto msg = error_message<Rule>) - expected = fmt(", %s", msg); - throw p::parse_error("unexpected end of file" + expected, in); - } - p::must_if<SyntaxErrors>::control<Rule>::raise(in, st...); - } -}; - -struct ExprState - : grammar:: - operator_semantics<ExprState, PosIdx, AttrPath, std::pair<PosIdx, std::unique_ptr<Expr>>> -{ - std::unique_ptr<Expr> popExprOnly() { - return std::move(popExpr().second); - } - - template<typename Op, typename... Args> - std::unique_ptr<Expr> applyUnary(Args &&... args) { - return std::make_unique<Op>(popExprOnly(), std::forward<Args>(args)...); - } - - template<typename Op> - std::unique_ptr<Expr> applyBinary(PosIdx pos) { - auto right = popExprOnly(), left = popExprOnly(); - return std::make_unique<Op>(pos, std::move(left), std::move(right)); - } - - std::unique_ptr<Expr> call(PosIdx pos, Symbol fn, bool flip = false) - { - std::vector<std::unique_ptr<Expr>> args(2); - args[flip ? 0 : 1] = popExprOnly(); - args[flip ? 1 : 0] = popExprOnly(); - return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(fn), std::move(args)); - } - - std::unique_ptr<Expr> pipe(PosIdx pos, State & state, bool flip = false) - { - if (!state.featureSettings.isEnabled(Xp::PipeOperator)) - throw ParseError({ - .msg = HintFmt("Pipe operator is disabled"), - .pos = state.positions[pos] - }); - - // Reverse the order compared to normal function application: arg |> fn - std::unique_ptr<Expr> fn, arg; - if (flip) { - fn = popExprOnly(); - arg = popExprOnly(); - } else { - arg = popExprOnly(); - fn = popExprOnly(); - } - std::vector<std::unique_ptr<Expr>> args{1}; - args[0] = std::move(arg); - - return std::make_unique<ExprCall>(pos, std::move(fn), std::move(args)); - } - - std::unique_ptr<Expr> order(PosIdx pos, bool less, State & state) - { - return call(pos, state.s.lessThan, !less); - } - - std::unique_ptr<Expr> concatStrings(PosIdx pos) - { - std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> args(2); - args[1] = popExpr(); - args[0] = popExpr(); - return std::make_unique<ExprConcatStrings>(pos, false, std::move(args)); - } - - std::unique_ptr<Expr> negate(PosIdx pos, State & state) - { - std::vector<std::unique_ptr<Expr>> args(2); - args[0] = std::make_unique<ExprInt>(0); - args[1] = popExprOnly(); - return std::make_unique<ExprCall>(pos, std::make_unique<ExprVar>(state.s.sub), std::move(args)); - } - - std::pair<PosIdx, std::unique_ptr<Expr>> applyOp(PosIdx pos, auto & op, State & state) { - using Op = grammar::op; - - auto not_ = [] (auto e) { - return std::make_unique<ExprOpNot>(std::move(e)); - }; - - return { - pos, - (overloaded { - [&] (Op::implies) { return applyBinary<ExprOpImpl>(pos); }, - [&] (Op::or_) { return applyBinary<ExprOpOr>(pos); }, - [&] (Op::and_) { return applyBinary<ExprOpAnd>(pos); }, - [&] (Op::equals) { return applyBinary<ExprOpEq>(pos); }, - [&] (Op::not_equals) { return applyBinary<ExprOpNEq>(pos); }, - [&] (Op::less) { return order(pos, true, state); }, - [&] (Op::greater_eq) { return not_(order(pos, true, state)); }, - [&] (Op::greater) { return order(pos, false, state); }, - [&] (Op::less_eq) { return not_(order(pos, false, state)); }, - [&] (Op::update) { return applyBinary<ExprOpUpdate>(pos); }, - [&] (Op::not_) { return applyUnary<ExprOpNot>(); }, - [&] (Op::plus) { return concatStrings(pos); }, - [&] (Op::minus) { return call(pos, state.s.sub); }, - [&] (Op::mul) { return call(pos, state.s.mul); }, - [&] (Op::div) { return call(pos, state.s.div); }, - [&] (Op::concat) { return applyBinary<ExprOpConcatLists>(pos); }, - [&] (has_attr & a) { return applyUnary<ExprOpHasAttr>(std::move(a.path)); }, - [&] (Op::unary_minus) { return negate(pos, state); }, - [&] (Op::pipe_right) { return pipe(pos, state, true); }, - [&] (Op::pipe_left) { return pipe(pos, state); }, - })(op) - }; - } - - // always_inline is needed, otherwise pushOp slows down considerably - [[noreturn, gnu::always_inline]] - static void badOperator(PosIdx pos, State & state) - { - throw ParseError({ - .msg = HintFmt("syntax error, unexpected operator"), - .pos = state.positions[pos] - }); - } - - template<typename Expr, typename... Args> - Expr & pushExpr(PosIdx pos, Args && ... args) - { - auto p = std::make_unique<Expr>(std::forward<Args>(args)...); - auto & result = *p; - exprs.emplace_back(pos, std::move(p)); - return result; - } -}; - -struct SubexprState { -private: - ExprState * up; - -public: - explicit SubexprState(ExprState & up, auto &...) : up(&up) {} - operator ExprState &() { return *up; } - ExprState * operator->() { return up; } -}; - - - -template<typename Rule> -struct BuildAST : grammar::nothing<Rule> {}; - -struct LambdaState : SubexprState { - using SubexprState::SubexprState; - - Symbol arg; - std::unique_ptr<Formals> formals; -}; - -struct FormalsState : SubexprState { - using SubexprState::SubexprState; - - Formals formals{}; - Formal formal{}; -}; - -template<> struct BuildAST<grammar::formal::name> { - static void apply(const auto & in, FormalsState & s, State & ps) { - s.formal = { - .pos = ps.at(in), - .name = ps.symbols.create(in.string_view()), - }; - } -}; - -template<> struct BuildAST<grammar::formal> { - static void apply0(FormalsState & s, State &) { - s.formals.formals.emplace_back(std::move(s.formal)); - } -}; - -template<> struct BuildAST<grammar::formal::default_value> { - static void apply0(FormalsState & s, State & ps) { - s.formal.def = s->popExprOnly(); - } -}; - -template<> struct BuildAST<grammar::formals::ellipsis> { - static void apply0(FormalsState & s, State &) { - s.formals.ellipsis = true; - } -}; - -template<> struct BuildAST<grammar::formals> : change_head<FormalsState> { - static void success0(FormalsState & f, LambdaState & s, State &) { - s.formals = std::make_unique<Formals>(std::move(f.formals)); - } -}; - -struct AttrState : SubexprState { - using SubexprState::SubexprState; - - std::vector<AttrName> attrs; - - template <typename T> - void pushAttr(T && attr, PosIdx) { attrs.emplace_back(std::forward<T>(attr)); } -}; - -template<> struct BuildAST<grammar::attr::simple> { - static void apply(const auto & in, auto & s, State & ps) { - s.pushAttr(ps.symbols.create(in.string_view()), ps.at(in)); - } -}; - -template<> struct BuildAST<grammar::attr::string> { - static void apply(const auto & in, auto & s, State & ps) { - auto e = s->popExprOnly(); - if (auto str = dynamic_cast<ExprString *>(e.get())) - s.pushAttr(ps.symbols.create(str->s), ps.at(in)); - else - s.pushAttr(std::move(e), ps.at(in)); - } -}; - -template<> struct BuildAST<grammar::attr::expr> : BuildAST<grammar::attr::string> {}; - -struct BindingsState : SubexprState { - using SubexprState::SubexprState; - - ExprAttrs attrs; - AttrPath path; - std::unique_ptr<Expr> value; -}; - -struct InheritState : SubexprState { - using SubexprState::SubexprState; - - std::vector<std::pair<AttrName, PosIdx>> attrs; - std::unique_ptr<Expr> from; - PosIdx fromPos; - - template <typename T> - void pushAttr(T && attr, PosIdx pos) { attrs.emplace_back(std::forward<T>(attr), pos); } -}; - -template<> struct BuildAST<grammar::inherit::from> { - static void apply(const auto & in, InheritState & s, State & ps) { - s.from = s->popExprOnly(); - s.fromPos = ps.at(in); - } -}; - -template<> struct BuildAST<grammar::inherit> : change_head<InheritState> { - static void success0(InheritState & s, BindingsState & b, State & ps) { - auto & attrs = b.attrs.attrs; - // TODO this should not reuse generic attrpath rules. - for (auto & [i, iPos] : s.attrs) { - if (i.symbol) - continue; - if (auto str = dynamic_cast<ExprString *>(i.expr.get())) - i = AttrName(ps.symbols.create(str->s)); - else { - throw ParseError({ - .msg = HintFmt("dynamic attributes not allowed in inherit"), - .pos = ps.positions[iPos] - }); - } - } - if (s.from != nullptr) { - if (!b.attrs.inheritFromExprs) - b.attrs.inheritFromExprs = std::make_unique<std::vector<ref<Expr>>>(); - auto fromExpr = ref<Expr>(std::move(s.from)); - b.attrs.inheritFromExprs->push_back(fromExpr); - for (auto & [i, iPos] : s.attrs) { - if (attrs.find(i.symbol) != attrs.end()) - ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); - auto inheritFrom = std::make_unique<ExprInheritFrom>( - s.fromPos, - b.attrs.inheritFromExprs->size() - 1, - fromExpr - ); - attrs.emplace( - i.symbol, - ExprAttrs::AttrDef( - std::make_unique<ExprSelect>(iPos, std::move(inheritFrom), i.symbol), - iPos, - ExprAttrs::AttrDef::Kind::InheritedFrom)); - } - } else { - for (auto & [i, iPos] : s.attrs) { - if (attrs.find(i.symbol) != attrs.end()) - ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); - attrs.emplace( - i.symbol, - ExprAttrs::AttrDef( - std::make_unique<ExprVar>(iPos, i.symbol), - iPos, - ExprAttrs::AttrDef::Kind::Inherited)); - } - } - } -}; - -template<> struct BuildAST<grammar::binding::path> : change_head<AttrState> { - static void success0(AttrState & a, BindingsState & s, State & ps) { - s.path = std::move(a.attrs); - } -}; - -template<> struct BuildAST<grammar::binding::value> { - static void apply0(BindingsState & s, State & ps) { - s.value = s->popExprOnly(); - } -}; - -template<> struct BuildAST<grammar::binding> { - static void apply(const auto & in, BindingsState & s, State & ps) { - ps.addAttr(&s.attrs, std::move(s.path), std::move(s.value), ps.at(in)); - } -}; - -template<> struct BuildAST<grammar::expr::id> { - static void apply(const auto & in, ExprState & s, State & ps) { - if (in.string_view() == "__curPos") - s.pushExpr<ExprPos>(ps.at(in), ps.at(in)); - else - s.pushExpr<ExprVar>(ps.at(in), ps.at(in), ps.symbols.create(in.string_view())); - } -}; - -template<> struct BuildAST<grammar::expr::int_> { - static void apply(const auto & in, ExprState & s, State & ps) { - int64_t v; - if (std::from_chars(in.begin(), in.end(), v).ec != std::errc{}) { - throw ParseError({ - .msg = HintFmt("invalid integer '%1%'", in.string_view()), - .pos = ps.positions[ps.at(in)], - }); - } - s.pushExpr<ExprInt>(noPos, v); - } -}; - -template<> struct BuildAST<grammar::expr::float_> { - static void apply(const auto & in, ExprState & s, State & ps) { - // copy the input into a temporary string so we can call stod. - // can't use from_chars because libc++ (thus darwin) does not have it, - // and floats are not performance-sensitive anyway. if they were you'd - // be in much bigger trouble than this. - // - // we also get to do a locale-save dance because stod is locale-aware and - // something (a plugin?) may have called setlocale or uselocale. - static struct locale_hack { - locale_t posix; - locale_hack(): posix(newlocale(LC_ALL_MASK, "POSIX", 0)) - { - if (posix == 0) - throw SysError("could not get POSIX locale"); - } - } locale; - - auto tmp = in.string(); - double v = [&] { - auto oldLocale = uselocale(locale.posix); - Finally resetLocale([=] { uselocale(oldLocale); }); - try { - return std::stod(tmp); - } catch (...) { - throw ParseError({ - .msg = HintFmt("invalid float '%1%'", in.string_view()), - .pos = ps.positions[ps.at(in)], - }); - } - }(); - s.pushExpr<ExprFloat>(noPos, v); - } -}; - -struct StringState : SubexprState { - using SubexprState::SubexprState; - - std::string currentLiteral; - PosIdx currentPos; - std::vector<std::pair<nix::PosIdx, std::unique_ptr<Expr>>> parts; - - void append(PosIdx pos, std::string_view s) - { - if (currentLiteral.empty()) - currentPos = pos; - currentLiteral += s; - } - - // FIXME this truncates strings on NUL for compat with the old parser. ideally - // we should use the decomposition the g gives us instead of iterating over - // the entire string again. - static void unescapeStr(std::string & str) - { - char * s = str.data(); - char * t = s; - char c; - while ((c = *s++)) { - if (c == '\\') { - c = *s++; - if (c == 'n') *t = '\n'; - else if (c == 'r') *t = '\r'; - else if (c == 't') *t = '\t'; - else *t = c; - } - else if (c == '\r') { - /* Normalise CR and CR/LF into LF. */ - *t = '\n'; - if (*s == '\n') s++; /* cr/lf */ - } - else *t = c; - t++; - } - str.resize(t - str.data()); - } - - void endLiteral() - { - if (!currentLiteral.empty()) { - unescapeStr(currentLiteral); - parts.emplace_back(currentPos, std::make_unique<ExprString>(std::move(currentLiteral))); - } - } - - std::unique_ptr<Expr> finish() - { - if (parts.empty()) { - unescapeStr(currentLiteral); - return std::make_unique<ExprString>(std::move(currentLiteral)); - } else { - endLiteral(); - auto pos = parts[0].first; - return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts)); - } - } -}; - -template<typename... Content> struct BuildAST<grammar::string::literal<Content...>> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), in.string_view()); - } -}; - -template<> struct BuildAST<grammar::string::cr_lf> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), in.string_view()); // FIXME compat with old parser - } -}; - -template<> struct BuildAST<grammar::string::interpolation> { - static void apply(const auto & in, StringState & s, State & ps) { - s.endLiteral(); - s.parts.emplace_back(ps.at(in), s->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::string::escape> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), "\\"); // FIXME compat with old parser - s.append(ps.at(in), in.string_view()); - } -}; - -template<> struct BuildAST<grammar::string> : change_head<StringState> { - static void success0(StringState & s, ExprState & e, State &) { - e.exprs.emplace_back(noPos, s.finish()); - } -}; - -struct IndStringState : SubexprState { - using SubexprState::SubexprState; - - std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> parts; -}; - -template<bool Indented, typename... Content> -struct BuildAST<grammar::ind_string::literal<Indented, Content...>> { - static void apply(const auto & in, IndStringState & s, State & ps) { - s.parts.emplace_back(ps.at(in), StringToken{in.string_view(), Indented}); - } -}; - -template<> struct BuildAST<grammar::ind_string::interpolation> { - static void apply(const auto & in, IndStringState & s, State & ps) { - s.parts.emplace_back(ps.at(in), s->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::ind_string::escape> { - static void apply(const auto & in, IndStringState & s, State & ps) { - switch (*in.begin()) { - case 'n': s.parts.emplace_back(ps.at(in), StringToken{"\n"}); break; - case 'r': s.parts.emplace_back(ps.at(in), StringToken{"\r"}); break; - case 't': s.parts.emplace_back(ps.at(in), StringToken{"\t"}); break; - default: s.parts.emplace_back(ps.at(in), StringToken{in.string_view()}); break; - } - } -}; - -template<> struct BuildAST<grammar::ind_string> : change_head<IndStringState> { - static void success(const auto & in, IndStringState & s, ExprState & e, State & ps) { - e.exprs.emplace_back(noPos, ps.stripIndentation(ps.at(in), std::move(s.parts))); - } -}; - -template<typename... Content> struct BuildAST<grammar::path::literal<Content...>> { - static void apply(const auto & in, StringState & s, State & ps) { - s.append(ps.at(in), in.string_view()); - s.endLiteral(); - } -}; - -template<> struct BuildAST<grammar::path::interpolation> : BuildAST<grammar::string::interpolation> {}; - -template<> struct BuildAST<grammar::path::anchor> { - static void apply(const auto & in, StringState & s, State & ps) { - Path path(absPath(in.string(), ps.basePath.path.abs())); - /* add back in the trailing '/' to the first segment */ - if (in.string_view().ends_with('/') && in.size() > 1) - path += "/"; - s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); - } -}; - -template<> struct BuildAST<grammar::path::home_anchor> { - static void apply(const auto & in, StringState & s, State & ps) { - if (evalSettings.pureEval) - throw Error("the path '%s' can not be resolved in pure mode", in.string_view()); - Path path(getHome() + in.string_view().substr(1)); - s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); - } -}; - -template<> struct BuildAST<grammar::path::searched_path> { - static void apply(const auto & in, StringState & s, State & ps) { - std::vector<std::unique_ptr<Expr>> args{2}; - args[0] = std::make_unique<ExprVar>(ps.s.nixPath); - args[1] = std::make_unique<ExprString>(in.string()); - s.parts.emplace_back( - ps.at(in), - std::make_unique<ExprCall>( - ps.at(in), - std::make_unique<ExprVar>(ps.s.findFile), - std::move(args))); - } -}; - -template<> struct BuildAST<grammar::path> : change_head<StringState> { - template<typename E> - static void check_slash(PosIdx end, StringState & s, State & ps) { - auto e = dynamic_cast<E *>(s.parts.back().second.get()); - if (!e || !e->s.ends_with('/')) - return; - if (s.parts.size() > 1 || e->s != "/") - throw ParseError({ - .msg = HintFmt("path has a trailing slash"), - .pos = ps.positions[end], - }); - } - - static void success(const auto & in, StringState & s, ExprState & e, State & ps) { - s.endLiteral(); - check_slash<ExprPath>(ps.atEnd(in), s, ps); - check_slash<ExprString>(ps.atEnd(in), s, ps); - if (s.parts.size() == 1) { - e.exprs.emplace_back(noPos, std::move(s.parts.back().second)); - } else { - e.pushExpr<ExprConcatStrings>(ps.at(in), ps.at(in), false, std::move(s.parts)); - } - } -}; - -// strings and paths sare handled fully by the grammar-level rule for now -template<> struct BuildAST<grammar::expr::string> : p::maybe_nothing {}; -template<> struct BuildAST<grammar::expr::ind_string> : p::maybe_nothing {}; -template<> struct BuildAST<grammar::expr::path> : p::maybe_nothing {}; - -template<> struct BuildAST<grammar::expr::uri> { - static void apply(const auto & in, ExprState & s, State & ps) { - bool URLLiterals = ps.featureSettings.isEnabled(Dep::UrlLiterals); - if (!URLLiterals) - throw ParseError({ - .msg = HintFmt("URL literals are deprecated, allow using them with --extra-deprecated-features=url-literals"), - .pos = ps.positions[ps.at(in)] - }); - s.pushExpr<ExprString>(ps.at(in), in.string()); - } -}; - -template<> struct BuildAST<grammar::expr::ancient_let> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - // Added 2024-09-18. Turn into an error at some point in the future. - // See the documentation on deprecated features for more details. - if (!ps.featureSettings.isEnabled(Dep::AncientLet)) - warn( - "%s found at %s. This feature is deprecated and will be removed in the future. Use %s to silence this warning.", - "let {", - ps.positions[ps.at(in)], - "--extra-deprecated-features ancient-let" - ); - - b.attrs.pos = ps.at(in); - b.attrs.recursive = true; - s.pushExpr<ExprSelect>(b.attrs.pos, b.attrs.pos, std::make_unique<ExprAttrs>(std::move(b.attrs)), ps.s.body); - } -}; - -template<> struct BuildAST<grammar::expr::rec_set> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - // Before inserting new attrs, check for __override and throw an error - // (the error will initially be a warning to ease migration) - if (!featureSettings.isEnabled(Dep::RecSetOverrides) && b.attrs.attrs.contains(ps.s.overrides)) { - ps.overridesFound(ps.at(in)); - } - - b.attrs.pos = ps.at(in); - b.attrs.recursive = true; - s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); - } -}; - -template<> struct BuildAST<grammar::expr::set> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - b.attrs.pos = ps.at(in); - s.pushExpr<ExprAttrs>(b.attrs.pos, std::move(b.attrs)); - } -}; - -using ListState = std::vector<std::unique_ptr<Expr>>; - -template<> struct BuildAST<grammar::expr::list> : change_head<ListState> { - static void success(const auto & in, ListState & ls, ExprState & s, State & ps) { - auto e = std::make_unique<ExprList>(); - e->elems = std::move(ls); - s.exprs.emplace_back(ps.at(in), std::move(e)); - } -}; - -template<> struct BuildAST<grammar::expr::list::entry> : change_head<ExprState> { - static void success0(ExprState & e, ListState & s, State & ps) { - s.emplace_back(e.finish(ps).second); - } -}; - -struct SelectState : SubexprState { - using SubexprState::SubexprState; - - PosIdx pos; - ExprSelect * e = nullptr; -}; - -template<> struct BuildAST<grammar::expr::select::head> { - static void apply(const auto & in, SelectState & s, State & ps) { - s.pos = ps.at(in); - } -}; - -template<> struct BuildAST<grammar::expr::select::attr> : change_head<AttrState> { - static void success0(AttrState & a, SelectState & s, State &) { - s.e = &s->pushExpr<ExprSelect>(s.pos, s.pos, s->popExprOnly(), std::move(a.attrs), nullptr); - } -}; - -template<> struct BuildAST<grammar::expr::select::attr_or> { - static void apply0(SelectState & s, State &) { - s.e->def = s->popExprOnly(); - } -}; - -template<> struct BuildAST<grammar::expr::select::as_app_or> { - static void apply(const auto & in, SelectState & s, State & ps) { - std::vector<std::unique_ptr<Expr>> args(1); - args[0] = std::make_unique<ExprVar>(ps.at(in), ps.s.or_); - s->pushExpr<ExprCall>(s.pos, s.pos, s->popExprOnly(), std::move(args)); - } -}; - -template<> struct BuildAST<grammar::expr::select> : change_head<SelectState> { - static void success0(const auto &...) {} -}; - -struct AppState : SubexprState { - using SubexprState::SubexprState; - - PosIdx pos; - ExprCall * e = nullptr; -}; - -template<> struct BuildAST<grammar::expr::app::select_or_fn> { - static void apply(const auto & in, AppState & s, State & ps) { - s.pos = ps.at(in); - } -}; - -template<> struct BuildAST<grammar::expr::app::first_arg> { - static void apply(auto & in, AppState & s, State & ps) { - auto arg = s->popExprOnly(), fn = s->popExprOnly(); - if ((s.e = dynamic_cast<ExprCall *>(fn.get()))) { - // TODO remove. - // AST compat with old parser, semantics are the same. - // this can happen on occasions such as `<p> <p>` or `a or b or`, - // neither of which are super worth optimizing. - s.e->args.push_back(std::move(arg)); - s->exprs.emplace_back(noPos, std::move(fn)); - } else { - std::vector<std::unique_ptr<Expr>> args{1}; - args[0] = std::move(arg); - s.e = &s->pushExpr<ExprCall>(s.pos, s.pos, std::move(fn), std::move(args)); - } - } -}; - -template<> struct BuildAST<grammar::expr::app::another_arg> { - static void apply0(AppState & s, State & ps) { - s.e->args.push_back(s->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::expr::app> : change_head<AppState> { - static void success0(const auto &...) {} -}; - -template<typename Op> struct BuildAST<grammar::expr::operator_<Op>> { - static void apply(const auto & in, ExprState & s, State & ps) { - s.pushOp(ps.at(in), Op{}, ps); - } -}; -template<> struct BuildAST<grammar::expr::operator_<grammar::op::has_attr>> : change_head<AttrState> { - static void success(const auto & in, AttrState & a, ExprState & s, State & ps) { - s.pushOp(ps.at(in), ExprState::has_attr{{}, std::move(a.attrs)}, ps); - } -}; - -template<> struct BuildAST<grammar::expr::lambda::arg> { - static void apply(const auto & in, LambdaState & s, State & ps) { - s.arg = ps.symbols.create(in.string_view()); - } -}; - -template<> struct BuildAST<grammar::expr::lambda> : change_head<LambdaState> { - static void success(const auto & in, LambdaState & l, ExprState & s, State & ps) { - if (l.formals) - l.formals = ps.validateFormals(std::move(l.formals), ps.at(in), l.arg); - s.pushExpr<ExprLambda>(ps.at(in), ps.at(in), l.arg, std::move(l.formals), l->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::expr::assert_> { - static void apply(const auto & in, ExprState & s, State & ps) { - auto body = s.popExprOnly(), cond = s.popExprOnly(); - s.pushExpr<ExprAssert>(ps.at(in), ps.at(in), std::move(cond), std::move(body)); - } -}; - -template<> struct BuildAST<grammar::expr::with> { - static void apply(const auto & in, ExprState & s, State & ps) { - auto body = s.popExprOnly(), scope = s.popExprOnly(); - s.pushExpr<ExprWith>(ps.at(in), ps.at(in), std::move(scope), std::move(body)); - } -}; - -template<> struct BuildAST<grammar::expr::let> : change_head<BindingsState> { - static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { - if (!b.attrs.dynamicAttrs.empty()) - throw ParseError({ - .msg = HintFmt("dynamic attributes not allowed in let"), - .pos = ps.positions[ps.at(in)] - }); - - s.pushExpr<ExprLet>(ps.at(in), std::make_unique<ExprAttrs>(std::move(b.attrs)), b->popExprOnly()); - } -}; - -template<> struct BuildAST<grammar::expr::if_> { - static void apply(const auto & in, ExprState & s, State & ps) { - auto else_ = s.popExprOnly(), then = s.popExprOnly(), cond = s.popExprOnly(); - s.pushExpr<ExprIf>(ps.at(in), ps.at(in), std::move(cond), std::move(then), std::move(else_)); - } -}; - -template<> struct BuildAST<grammar::expr> : change_head<ExprState> { - static void success0(ExprState & inner, ExprState & outer, State & ps) { - outer.exprs.push_back(inner.finish(ps)); - } -}; - -} -} +// Linter complains that this is a "suspicious include of file with '.cc' extension". +// While that is correct and generally not great, it is one of the less bad options to pick +// in terms of diff noise. +// NOLINTNEXTLINE(bugprone-suspicious-include) +#include "parser-impl1.inc.cc" namespace nix { @@ -884,7 +38,6 @@ Expr * EvalState::parse( exprSymbols, featureSettings, }; - parser::ExprState x; assert(length >= 2); assert(text[length - 1] == 0); @@ -893,7 +46,12 @@ Expr * EvalState::parse( p::string_input<p::tracking_mode::lazy> inp{std::string_view{text, length}, "input"}; try { - p::parse<parser::grammar::root, parser::BuildAST, parser::Control>(inp, x, s); + parser::v1::ExprState x; + p::parse<parser::grammar::v1::root, parser::v1::BuildAST, parser::v1::Control>(inp, x, s); + + auto [_pos, result] = x.finish(s); + result->bindVars(*this, staticEnv); + return result.release(); } catch (p::parse_error & e) { auto pos = e.positions().back(); throw ParseError({ @@ -901,10 +59,6 @@ Expr * EvalState::parse( .pos = positions[s.positions.add(s.origin, pos.byte)] }); } - - auto [_pos, result] = x.finish(s); - result->bindVars(*this, staticEnv); - return result.release(); } } diff --git a/src/libexpr/parser/state.hh b/src/libexpr/parser/state.hh index 1d57e2f5f..b969f73e4 100644 --- a/src/libexpr/parser/state.hh +++ b/src/libexpr/parser/state.hh @@ -6,11 +6,21 @@ namespace nix::parser { -struct StringToken -{ - std::string_view s; - bool hasIndentation; - operator std::string_view() const { return s; } +struct IndStringLine { + // String containing only the leading whitespace of the line. May be empty. + std::string_view indentation; + // Position of the line start (before the indentation) + PosIdx pos; + + // Whether the line contains anything besides indentation and line break + bool hasContent = false; + + std::vector< + std::pair< + PosIdx, + std::variant<std::unique_ptr<Expr>, std::string_view> + > + > parts = {}; }; struct State @@ -27,8 +37,7 @@ struct State void overridesFound(const PosIdx pos); void addAttr(ExprAttrs * attrs, AttrPath && attrPath, std::unique_ptr<Expr> e, const PosIdx pos); std::unique_ptr<Formals> validateFormals(std::unique_ptr<Formals> formals, PosIdx pos = noPos, Symbol arg = {}); - std::unique_ptr<Expr> stripIndentation(const PosIdx pos, - std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es); + std::unique_ptr<Expr> stripIndentation(const PosIdx pos, std::vector<IndStringLine> && line); // lazy positioning means we don't get byte offsets directly, in.position() would work // but also requires line and column (which is expensive) @@ -182,98 +191,87 @@ inline std::unique_ptr<Formals> State::validateFormals(std::unique_ptr<Formals> return formals; } -inline std::unique_ptr<Expr> State::stripIndentation(const PosIdx pos, - std::vector<std::pair<PosIdx, std::variant<std::unique_ptr<Expr>, StringToken>>> && es) +inline std::unique_ptr<Expr> State::stripIndentation( + const PosIdx pos, + std::vector<IndStringLine> && lines) { - if (es.empty()) return std::make_unique<ExprString>(""); + /* If the only line is whitespace-only, directly return empty string. + * The rest of the code relies on the final string not being empty. + */ + if (lines.size() == 1 && lines.front().parts.empty()) { + return std::make_unique<ExprString>(""); + } - /* Figure out the minimum indentation. Note that by design - whitespace-only final lines are not taken into account. (So - the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ - bool atStartOfLine = true; /* = seen only whitespace in the current line */ + /* If the last line only contains whitespace, trim it to not cause excessive whitespace. + * (Other whitespace-only lines get stripped only of the common indentation, and excess + * whitespace becomes part of the string.) + */ + if (lines.back().parts.empty()) { + lines.back().indentation = {}; + } + + /* Figure out the minimum indentation. Note that by design + whitespace-only lines are not taken into account. */ size_t minIndent = 1000000; - size_t curIndent = 0; - for (auto & [i_pos, i] : es) { - auto * str = std::get_if<StringToken>(&i); - if (!str || !str->hasIndentation) { - /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ - if (atStartOfLine) { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - continue; - } - for (size_t j = 0; j < str->s.size(); ++j) { - if (atStartOfLine) { - if (str->s[j] == ' ') - curIndent++; - else if (str->s[j] == '\n') { - /* Empty line, doesn't influence minimum - indentation. */ - curIndent = 0; - } else { - atStartOfLine = false; - if (curIndent < minIndent) minIndent = curIndent; - } - } else if (str->s[j] == '\n') { - atStartOfLine = true; - curIndent = 0; - } + for (auto & line : lines) { + if (line.hasContent) { + minIndent = std::min(minIndent, line.indentation.size()); } } /* Strip spaces from each line. */ - std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> es2; - atStartOfLine = true; - size_t curDropped = 0; - size_t n = es.size(); - auto i = es.begin(); - const auto trimExpr = [&] (std::unique_ptr<Expr> e) { - atStartOfLine = false; - curDropped = 0; - es2.emplace_back(i->first, std::move(e)); - }; - const auto trimString = [&] (const StringToken t) { - std::string s2; - for (size_t j = 0; j < t.s.size(); ++j) { - if (atStartOfLine) { - if (t.s[j] == ' ') { - if (curDropped++ >= minIndent) - s2 += t.s[j]; - } - else if (t.s[j] == '\n') { - curDropped = 0; - s2 += t.s[j]; - } else { - atStartOfLine = false; - curDropped = 0; - s2 += t.s[j]; - } - } else { - s2 += t.s[j]; - if (t.s[j] == '\n') atStartOfLine = true; - } - } + for (auto & line : lines) { + line.indentation.remove_prefix(std::min(minIndent, line.indentation.size())); + } + + /* Concat the parts together again */ + + std::vector<std::pair<PosIdx, std::unique_ptr<Expr>>> parts; + /* Accumulator for merging intermediates */ + PosIdx merged_pos; + std::string merged = ""; - /* Remove the last line if it is empty and consists only of - spaces. */ - if (n == 1) { - std::string::size_type p = s2.find_last_of('\n'); - if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) - s2 = std::string(s2, 0, p + 1); + auto push_merged = [&] (PosIdx i_pos, std::string_view str) { + if (merged.empty()) { + merged_pos = i_pos; } + merged += str; + }; - es2.emplace_back(i->first, std::make_unique<ExprString>(std::move(s2))); + auto flush_merged = [&] () { + if (!merged.empty()) { + parts.emplace_back(merged_pos, std::make_unique<ExprString>(std::string(merged))); + merged.clear(); + } }; - for (; i != es.end(); ++i, --n) { - std::visit(overloaded { trimExpr, trimString }, std::move(i->second)); + + for (auto && [li, line] : enumerate(lines)) { + push_merged(line.pos, line.indentation); + + for (auto & val : line.parts) { + auto &[i_pos, item] = val; + + std::visit(overloaded{ + [&](std::string_view str) { + push_merged(i_pos, str); + }, + [&](std::unique_ptr<Expr> expr) { + flush_merged(); + parts.emplace_back(i_pos, std::move(expr)); + }, + }, std::move(item)); + } } - /* If this is a single string, then don't do a concatenation. */ - if (es2.size() == 1 && dynamic_cast<ExprString *>(es2[0].second.get())) { - return std::move(es2[0].second); + flush_merged(); + + /* If this is a single string, then don't do a concatenation. + * (If it's a single expression, still do the ConcatStrings to properly force it being a string.) + */ + if (parts.size() == 1 && dynamic_cast<ExprString *>(parts[0].second.get())) { + return std::move(parts[0].second); } - return std::make_unique<ExprConcatStrings>(pos, true, std::move(es2)); + return std::make_unique<ExprConcatStrings>(pos, true, std::move(parts)); } } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index b0e14a26e..c98fe2a03 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -394,7 +394,8 @@ static RegisterPrimOp primop_fetchGit({ [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References By default, the `ref` value is prefixed with `refs/heads/`. - As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`. + As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/` or + if `ref` looks like a commit hash for backwards compatibility with CppNix 2.3. - `submodules` (default: `false`) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7d16d3f57..da60bf331 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,3 +1,4 @@ +#include "error.hh" #include "fetchers.hh" #include "cache.hh" #include "globals.hh" @@ -257,6 +258,28 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co } } // end namespace +static std::optional<Path> resolveRefToCachePath( + Input & input, + const Path & cacheDir, + std::vector<Path> & gitRefFileCandidates, + std::function<bool(const Path&)> condition) +{ + if (input.getRef()->starts_with("refs/")) { + Path fullpath = cacheDir + "/" + *input.getRef(); + if (condition(fullpath)) { + return fullpath; + } + } + + for (auto & candidate : gitRefFileCandidates) { + if (condition(candidate)) { + return candidate; + } + } + + return std::nullopt; +} + struct GitInputScheme : InputScheme { std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override @@ -539,10 +562,13 @@ struct GitInputScheme : InputScheme runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); } - Path localRefFile = - input.getRef()->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input.getRef() - : cacheDir + "/refs/heads/" + *input.getRef(); + std::vector<Path> gitRefFileCandidates; + for (auto & infix : {"", "tags/", "heads/"}) { + Path p = cacheDir + "/refs/" + infix + *input.getRef(); + gitRefFileCandidates.push_back(p); + } + + Path localRefFile; bool doFetch; time_t now = time(0); @@ -564,29 +590,70 @@ struct GitInputScheme : InputScheme if (allRefs) { doFetch = true; } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - !isCacheFileWithinTtl(now, st); + std::function<bool(const Path&)> condition; + condition = [&now](const Path & path) { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + return stat(path.c_str(), &st) == 0 && + isCacheFileWithinTtl(now, st); + }; + if (auto result = resolveRefToCachePath( + input, + cacheDir, + gitRefFileCandidates, + condition + )) { + localRefFile = *result; + doFetch = false; + } else { + doFetch = true; + } } } + // When having to fetch, we don't know `localRefFile` yet. + // Because git needs to figure out what we're fetching + // (i.e. is it a rev? a branch? a tag?) if (doFetch) { Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. + auto ref = input.getRef(); + std::string fetchRef; + if (allRefs) { + fetchRef = "refs/*"; + } else if ( + ref->starts_with("refs/") + || *ref == "HEAD" + || std::regex_match(*ref, revRegex)) + { + fetchRef = *ref; + } else { + fetchRef = "refs/*/" + *ref; + } + try { - auto ref = input.getRef(); - auto fetchRef = allRefs - ? "refs/*" - : ref->compare(0, 5, "refs/") == 0 - ? *ref - : ref == "HEAD" - ? *ref - : "refs/heads/" + *ref; - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, true); + Finally finally([&]() { + if (auto p = resolveRefToCachePath( + input, + cacheDir, + gitRefFileCandidates, + pathExists + )) { + localRefFile = *p; + } + }); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, { + "-C", repoDir, + "--git-dir", gitDir, + "fetch", + "--quiet", + "--force", + "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) + }, true); } catch (Error & e) { if (!pathExists(localRefFile)) throw; warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); diff --git a/src/libmain/crash-handler.cc b/src/libmain/crash-handler.cc new file mode 100644 index 000000000..3f1b9f7d8 --- /dev/null +++ b/src/libmain/crash-handler.cc @@ -0,0 +1,41 @@ +#include "crash-handler.hh" +#include "fmt.hh" + +#include <boost/core/demangle.hpp> +#include <exception> + +namespace nix { + +namespace { +void onTerminate() +{ + std::cerr << "Lix crashed. This is a bug. We would appreciate if you report it along with what caused it at https://git.lix.systems/lix-project/lix/issues with the following information included:\n\n"; + try { + std::exception_ptr eptr = std::current_exception(); + if (eptr) { + std::rethrow_exception(eptr); + } else { + std::cerr << "std::terminate() called without exception\n"; + } + } catch (const std::exception & ex) { + std::cerr << "Exception: " << boost::core::demangle(typeid(ex).name()) << ": " << ex.what() << "\n"; + } catch (...) { + std::cerr << "Unknown exception! Spooky.\n"; + } + + std::cerr << "Stack trace:\n"; + nix::printStackTrace(); + + std::abort(); +} +} + +void registerCrashHandler() +{ + // DO NOT use this for signals. Boost stacktrace is very much not + // async-signal-safe, and in a world with ASLR, addr2line is pointless. + // + // If you want signals, set up a minidump system and do it out-of-process. + std::set_terminate(onTerminate); +} +} diff --git a/src/libmain/crash-handler.hh b/src/libmain/crash-handler.hh new file mode 100644 index 000000000..4c5641b8c --- /dev/null +++ b/src/libmain/crash-handler.hh @@ -0,0 +1,21 @@ +#pragma once +/// @file Crash handler for Lix that prints back traces (hopefully in instances where it is not just going to crash the process itself). +/* + * Author's note: This will probably be partially/fully supplanted by a + * minidump writer like the following once we get our act together on crashes a + * little bit more: + * https://github.com/rust-minidump/minidump-writer + * https://github.com/EmbarkStudios/crash-handling + * (out of process implementation *should* be able to be done on-demand) + * + * Such an out-of-process implementation could then both make minidumps and + * print stack traces for arbitrarily messed-up process states such that we can + * safely give out backtraces for SIGSEGV and other deadly signals. + */ + +namespace nix { + +/** Registers the Lix crash handler for std::terminate (currently; will support more crashes later). See also detectStackOverflow(). */ +void registerCrashHandler(); + +} diff --git a/src/libmain/meson.build b/src/libmain/meson.build index a7cce287c..a1a888c16 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -1,5 +1,6 @@ libmain_sources = files( 'common-args.cc', + 'crash-handler.cc', 'loggers.cc', 'progress-bar.cc', 'shared.cc', @@ -8,6 +9,7 @@ libmain_sources = files( libmain_headers = files( 'common-args.hh', + 'crash-handler.hh', 'loggers.hh', 'progress-bar.hh', 'shared.hh', diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index bc9548e09..029b457b1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,3 +1,4 @@ +#include "crash-handler.hh" #include "globals.hh" #include "shared.hh" #include "store-api.hh" @@ -118,6 +119,8 @@ static void sigHandler(int signo) { } void initNix() { + registerCrashHandler(); + /* Turn on buffering for cerr. */ static char buf[1024]; std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); @@ -335,12 +338,15 @@ int handleExceptions(const std::string & programName, std::function<void()> fun) } catch (BaseError & e) { logError(e.info()); return e.info().status; - } catch (std::bad_alloc & e) { + } catch (const std::bad_alloc & e) { printError(error + "out of memory"); return 1; - } catch (std::exception & e) { - printError(error + e.what()); - return 1; + } catch (const std::exception & e) { + // Random exceptions bubbling into main are cause for bug reports, crash + std::terminate(); + } catch (...) { + // Explicitly do not tolerate non-std exceptions escaping. + std::terminate(); } return 0; @@ -389,7 +395,7 @@ RunPager::~RunPager() pid.wait(); } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index b41efe567..49b72a54e 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -111,7 +111,7 @@ struct PrintFreed /** - * Install a SIGSEGV handler to detect stack overflows. + * Install a SIGSEGV handler to detect stack overflows. See also registerCrashHandler(). */ void detectStackOverflow(); diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 9634fb944..846c6c9b9 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -47,7 +47,7 @@ struct BuildResult * @todo This should be an entire ErrorInfo object, not just a * string, for richer information. */ - std::string errorMsg; + std::string errorMsg = {}; std::string toString() const { auto strStatus = [&]() { @@ -90,7 +90,7 @@ struct BuildResult * For derivations, a mapping from the names of the wanted outputs * to actual paths. */ - SingleDrvOutputs builtOutputs; + SingleDrvOutputs builtOutputs = {}; /** * The start/stop times of the build (or one of the rounds, if it diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 827c9f541..96140e10b 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -11,7 +11,13 @@ #include "drv-output-substitution-goal.hh" #include "strings.hh" +#include <boost/outcome/try.hpp> #include <fstream> +#include <kj/array.h> +#include <kj/async-unix.h> +#include <kj/async.h> +#include <kj/debug.h> +#include <kj/vector.h> #include <sys/types.h> #include <sys/socket.h> #include <sys/un.h> @@ -65,7 +71,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -85,7 +90,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique<Derivation>(drv); - state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); @@ -103,17 +107,7 @@ DerivationGoal::~DerivationGoal() noexcept(false) { /* Careful: we should never ever throw an exception from a destructor. */ - try { closeLogFile(); } catch (...) { ignoreException(); } -} - - -std::string DerivationGoal::key() -{ - /* Ensure that derivations get built in order of their name, - i.e. a derivation named "aardvark" always comes before - "baboon". And substitution goals always happen before - derivation goals (due to "b$"). */ - return "b$" + std::string(drvPath.name()) + "$" + worker.store.printStorePath(drvPath); + try { closeLogFile(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -124,20 +118,24 @@ void DerivationGoal::killChild() } -Goal::Finished DerivationGoal::timedOut(Error && ex) +Goal::WorkResult DerivationGoal::timedOut(Error && ex) { killChild(); return done(BuildResult::TimedOut, {}, std::move(ex)); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::work(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::workImpl() noexcept { - return (this->*state)(inBuildSlot); + return useDerivation ? getDerivation() : haveDerivation(); } -void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) +bool DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) { + if (isDone) { + return false; + } + auto newWanted = wantedOutputs.union_(outputs); switch (needRestart) { case NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed: @@ -154,10 +152,11 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) break; }; wantedOutputs = newWanted; + return true; } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::getDerivation(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::getDerivation() noexcept try { trace("init"); @@ -165,18 +164,17 @@ try { exists. If it doesn't, it may be created through a substitute. */ if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - return loadDerivation(inBuildSlot); + co_return co_await loadDerivation(); } - - state = &DerivationGoal::loadDerivation; - return {WaitForGoals{{worker.goalFactory().makePathSubstitutionGoal(drvPath)}}}; + (co_await waitForGoals(worker.goalFactory().makePathSubstitutionGoal(drvPath))).value(); + co_return co_await loadDerivation(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::loadDerivation(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::loadDerivation() noexcept try { trace("loading derivation"); @@ -207,13 +205,13 @@ try { } assert(drv); - return haveDerivation(inBuildSlot); + return haveDerivation(); } catch (...) { return {std::current_exception()}; } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::haveDerivation(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::haveDerivation() noexcept try { trace("have derivation"); @@ -241,7 +239,7 @@ try { }); } - return gaveUpOnSubstitution(inBuildSlot); + co_return co_await gaveUpOnSubstitution(); } for (auto & i : drv->outputsAndOptPaths(worker.store)) @@ -263,19 +261,19 @@ try { /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - return {done(BuildResult::AlreadyValid, std::move(validOutputs))}; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* We are first going to try to create the invalid output paths through substitutes. If that doesn't work, we'll build them. */ - WaitForGoals result; + kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies; if (settings.useSubstitutes) { if (parsedDrv->substitutesAllowed()) { for (auto & [outputName, status] : initialOutputs) { if (!status.wanted) continue; if (!status.known) - result.goals.insert( + dependencies.add( worker.goalFactory().makeDrvOutputSubstitutionGoal( DrvOutput{status.outputHash, outputName}, buildMode == bmRepair ? Repair : NoRepair @@ -283,7 +281,7 @@ try { ); else { auto * cap = getDerivationCA(*drv); - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal( + dependencies.add(worker.goalFactory().makePathSubstitutionGoal( status.known->path, buildMode == bmRepair ? Repair : NoRepair, cap ? std::optional { *cap } : std::nullopt)); @@ -294,17 +292,15 @@ try { } } - if (result.goals.empty()) { /* to prevent hang (no wake-up event) */ - return outputsSubstitutionTried(inBuildSlot); - } else { - state = &DerivationGoal::outputsSubstitutionTried; - return {std::move(result)}; + if (!dependencies.empty()) { /* to prevent hang (no wake-up event) */ + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await outputsSubstitutionTried(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::outputsSubstitutionTried(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::outputsSubstitutionTried() noexcept try { trace("all outputs substituted (maybe)"); @@ -354,7 +350,7 @@ try { if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; - return haveDerivation(inBuildSlot); + return haveDerivation(); } auto [allValid, validOutputs] = checkPathValidity(); @@ -370,7 +366,7 @@ try { worker.store.printStorePath(drvPath)); /* Nothing to wait for; tail call */ - return gaveUpOnSubstitution(inBuildSlot); + return gaveUpOnSubstitution(); } catch (...) { return {std::current_exception()}; } @@ -378,9 +374,9 @@ try { /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::gaveUpOnSubstitution(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::gaveUpOnSubstitution() noexcept try { - WaitForGoals result; + kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies; /* At this point we are building all outputs, so if more are wanted there is no need to restart. */ @@ -393,7 +389,7 @@ try { addWaiteeDerivedPath = [&](ref<SingleDerivedPath> inputDrv, const DerivedPathMap<StringSet>::ChildNode & inputNode) { if (!inputNode.value.empty()) - result.goals.insert(worker.goalFactory().makeGoal( + dependencies.add(worker.goalFactory().makeGoal( DerivedPath::Built { .drvPath = inputDrv, .outputs = inputNode.value, @@ -438,17 +434,15 @@ try { if (!settings.useSubstitutes) throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled", worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); } - if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ - return inputsRealised(inBuildSlot); - } else { - state = &DerivationGoal::inputsRealised; - return {result}; + if (!dependencies.empty()) {/* to prevent hang (no wake-up event) */ + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await inputsRealised(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } @@ -488,7 +482,7 @@ try { } /* Check each path (slow!). */ - WaitForGoals result; + kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies; for (auto & i : outputClosure) { if (worker.pathContentsGood(i)) continue; printError( @@ -496,9 +490,9 @@ try { worker.store.printStorePath(i), worker.store.printStorePath(drvPath)); auto drvPath2 = outputsToDrv.find(i); if (drvPath2 == outputsToDrv.end()) - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i, Repair)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i, Repair)); else - result.goals.insert(worker.goalFactory().makeGoal( + dependencies.add(worker.goalFactory().makeGoal( DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath2->second), .outputs = OutputsSpec::All { }, @@ -506,18 +500,18 @@ try { bmRepair)); } - if (result.goals.empty()) { - return {done(BuildResult::AlreadyValid, assertPathValidity())}; + if (dependencies.empty()) { + co_return done(BuildResult::AlreadyValid, assertPathValidity()); } - state = &DerivationGoal::closureRepaired; - return {result}; + (co_await waitForGoals(dependencies.releaseAsArray())).value(); + co_return co_await closureRepaired(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::closureRepaired(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::closureRepaired() noexcept try { trace("closure repaired"); if (nrFailed > 0) @@ -529,14 +523,14 @@ try { } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::inputsRealised(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::inputsRealised() noexcept try { trace("all inputs realised"); if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - return {done( + co_return done( BuildResult::DependencyFailed, {}, Error( @@ -544,12 +538,12 @@ try { nrFailed, worker.store.printStorePath(drvPath) ) - )}; + ); } if (retrySubstitution == RetrySubstitution::YesNeed) { retrySubstitution = RetrySubstitution::AlreadyRetried; - return haveDerivation(inBuildSlot); + co_return co_await haveDerivation(); } /* Gather information necessary for computing the closure and/or @@ -611,11 +605,12 @@ try { worker.store.printStorePath(pathResolved), }); - resolvedDrvGoal = worker.goalFactory().makeDerivationGoal( + auto dependency = worker.goalFactory().makeDerivationGoal( pathResolved, wantedOutputs, buildMode); + resolvedDrvGoal = dependency.first; - state = &DerivationGoal::resolvedFinished; - return {WaitForGoals{{resolvedDrvGoal}}}; + (co_await waitForGoals(std::move(dependency))).value(); + co_return co_await resolvedFinished(); } std::function<void(const StorePath &, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths; @@ -679,10 +674,9 @@ try { /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ - state = &DerivationGoal::tryToBuild; - return tryToBuild(inBuildSlot); + co_return co_await tryToBuild(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } void DerivationGoal::started() @@ -698,8 +692,9 @@ void DerivationGoal::started() mcRunningBuilds = worker.runningBuilds.addTemporarily(1); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::tryToBuild(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::tryToBuild() noexcept try { +retry: trace("trying to build"); /* Obtain locks on all output paths, if the paths are known a priori. @@ -733,7 +728,9 @@ try { if (!actLock) actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); - return {WaitForAWhile{}}; + co_await waitForAWhile(); + // we can loop very often, and `co_return co_await` always allocates a new frame + goto retry; } actLock.reset(); @@ -750,7 +747,7 @@ try { if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - return {done(BuildResult::AlreadyValid, std::move(validOutputs))}; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* If any of the outputs already exist but are not valid, delete @@ -770,47 +767,56 @@ try { && settings.maxBuildJobs.get() != 0; if (!buildLocally) { - auto hookReply = tryBuildHook(inBuildSlot); - auto result = std::visit( - overloaded{ - [&](HookReply::Accept & a) -> std::optional<WorkResult> { - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - buildResult.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; - started(); - return WaitForWorld{std::move(a.fds), false}; - }, - [&](HookReply::Postpone) -> std::optional<WorkResult> { - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique<Activity>(*logger, lvlTalkative, actBuildWaiting, - fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); - outputLocks.unlock(); - return WaitForAWhile{}; - }, - [&](HookReply::Decline) -> std::optional<WorkResult> { - /* We should do it ourselves. */ - return std::nullopt; - }, - }, - hookReply); - if (result) { - return {std::move(*result)}; + auto hookReply = tryBuildHook(); + switch (hookReply.index()) { + case 0: { + HookReply::Accept & a = std::get<0>(hookReply); + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + actLock.reset(); + buildResult.startTime = time(0); // inexact + started(); + auto r = co_await a.promise; + if (r.has_value()) { + co_return co_await buildDone(); + } else if (r.has_error()) { + co_return r.assume_error(); + } else { + co_return r.assume_exception(); + } + } + + case 1: { + HookReply::Decline _ [[gnu::unused]] = std::get<1>(hookReply); + break; + } + + case 2: { + HookReply::Postpone _ [[gnu::unused]] = std::get<2>(hookReply); + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique<Activity>(*logger, lvlTalkative, actBuildWaiting, + fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); + outputLocks.unlock(); + co_await waitForAWhile(); + goto retry; + } + + default: + // can't static_assert this because HookReply *subclasses* variant and std::variant_size breaks + assert(false && "unexpected hook reply"); } } actLock.reset(); - state = &DerivationGoal::tryLocalBuild; - return tryLocalBuild(inBuildSlot); + co_return co_await tryLocalBuild(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::tryLocalBuild(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::tryLocalBuild() noexcept try { throw Error( "unable to build with a primary store that isn't a local store; " @@ -857,7 +863,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) // attempt to recover movePath(oldPath, storePath); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } throw; } @@ -973,10 +979,11 @@ void runPostBuildHook( proc.getStdout()->drainInto(sink); } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::buildDone(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::buildDone() noexcept try { trace("build done"); + slotToken = {}; Finally releaseBuildUser([&](){ this->cleanupHookFinally(); }); cleanupPreChildKill(); @@ -992,9 +999,6 @@ try { buildResult.timesBuilt++; buildResult.stopTime = time(0); - /* So the child is gone now. */ - worker.childTerminated(this); - /* Close the read side of the logger pipe. */ closeReadPipes(); @@ -1095,7 +1099,7 @@ try { return {std::current_exception()}; } -kj::Promise<Result<Goal::WorkResult>> DerivationGoal::resolvedFinished(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DerivationGoal::resolvedFinished() noexcept try { trace("resolved derivation finished"); @@ -1168,7 +1172,7 @@ try { return {std::current_exception()}; } -HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) +HookReply DerivationGoal::tryBuildHook() { if (!worker.hook.available || !useDerivation) return HookReply::Decline{}; @@ -1180,7 +1184,7 @@ HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) /* Send the request to the hook. */ worker.hook.instance->sink << "try" - << (inBuildSlot ? 1 : 0) + << (slotToken.valid() ? 1 : 0) << drv->platform << worker.store.printStorePath(drvPath) << parsedDrv->getRequiredSystemFeatures(); @@ -1207,6 +1211,7 @@ HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) else { s += "\n"; writeLogsToStderr(s); + logger->log(lvlInfo, s); } } @@ -1266,12 +1271,8 @@ HookReply DerivationGoal::tryBuildHook(bool inBuildSlot) /* Create the log file and pipe. */ Path logFile = openLogFile(); - std::set<int> fds; - fds.insert(hook->fromHook.get()); - fds.insert(hook->builderOut.get()); builderOutFD = &hook->builderOut; - - return HookReply::Accept{std::move(fds)}; + return HookReply::Accept{handleChildOutput()}; } @@ -1331,23 +1332,69 @@ void DerivationGoal::closeLogFile() } -Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data) +Goal::WorkResult DerivationGoal::tooMuchLogs() { - assert(builderOutFD); + killChild(); + return done( + BuildResult::LogLimitExceeded, {}, + Error("%s killed after writing more than %d bytes of log output", + getName(), settings.maxLogSize)); +} - auto tooMuchLogs = [&] { - killChild(); - return done( - BuildResult::LogLimitExceeded, {}, - Error("%s killed after writing more than %d bytes of log output", - getName(), settings.maxLogSize)); - }; +struct DerivationGoal::InputStream final : private kj::AsyncObject +{ + int fd; + kj::UnixEventPort::FdObserver observer; + + InputStream(kj::UnixEventPort & ep, int fd) + : fd(fd) + , observer(ep, fd, kj::UnixEventPort::FdObserver::OBSERVE_READ) + { + int flags = fcntl(fd, F_GETFL); + if (flags < 0) { + throw SysError("fcntl(F_GETFL) failed on fd %i", fd); + } + if (fcntl(fd, F_SETFL, flags | O_NONBLOCK) < 0) { + throw SysError("fcntl(F_SETFL) failed on fd %i", fd); + } + } + + kj::Promise<std::string_view> read(kj::ArrayPtr<char> buffer) + { + const auto res = ::read(fd, buffer.begin(), buffer.size()); + // closing a pty endpoint causes EIO on the other endpoint. stock kj streams + // do not handle this and throw exceptions we can't ask for errno instead :( + // (we can't use `errno` either because kj may well have mangled it by now.) + if (res == 0 || (res == -1 && errno == EIO)) { + return std::string_view{}; + } + + KJ_NONBLOCKING_SYSCALL(res) {} + + if (res > 0) { + return std::string_view{buffer.begin(), static_cast<size_t>(res)}; + } + + return observer.whenBecomesReadable().then([this, buffer] { + return read(buffer); + }); + } +}; + +kj::Promise<Outcome<void, Goal::WorkResult>> DerivationGoal::handleBuilderOutput(InputStream & in) noexcept +try { + auto buf = kj::heapArray<char>(4096); + while (true) { + auto data = co_await in.read(buf); + lastChildActivity = worker.aio.provider->getTimer().now(); + + if (data.empty()) { + co_return result::success(); + } - // local & `ssh://`-builds are dealt with here. - if (fd == builderOutFD->get()) { logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - return tooMuchLogs(); + co_return tooMuchLogs(); } for (auto c : data) @@ -1362,10 +1409,22 @@ Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data } if (logSink) (*logSink)(data); - return StillAlive{}; } +} catch (...) { + co_return std::current_exception(); +} + +kj::Promise<Outcome<void, Goal::WorkResult>> DerivationGoal::handleHookOutput(InputStream & in) noexcept +try { + auto buf = kj::heapArray<char>(4096); + while (true) { + auto data = co_await in.read(buf); + lastChildActivity = worker.aio.provider->getTimer().now(); + + if (data.empty()) { + co_return result::success(); + } - if (hook && fd == hook->fromHook.get()) { for (auto c : data) if (c == '\n') { auto json = parseJSONMessage(currentHookLine); @@ -1381,7 +1440,7 @@ Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data (fields.size() > 0 ? fields[0].get<std::string>() : "") + "\n"; logSize += logLine.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { - return tooMuchLogs(); + co_return tooMuchLogs(); } (*logSink)(logLine); } else if (type == resSetPhase && ! fields.is_null()) { @@ -1405,16 +1464,83 @@ Goal::WorkResult DerivationGoal::handleChildOutput(int fd, std::string_view data } else currentHookLine += c; } - - return StillAlive{}; +} catch (...) { + co_return std::current_exception(); } +kj::Promise<Outcome<void, Goal::WorkResult>> DerivationGoal::handleChildOutput() noexcept +try { + assert(builderOutFD); + + auto builderIn = kj::heap<InputStream>(worker.aio.unixEventPort, builderOutFD->get()); + kj::Own<InputStream> hookIn; + if (hook) { + hookIn = kj::heap<InputStream>(worker.aio.unixEventPort, hook->fromHook.get()); + } -void DerivationGoal::handleEOF(int fd) + auto handlers = handleChildStreams(*builderIn, hookIn.get()).attach(std::move(builderIn), std::move(hookIn)); + + if (respectsTimeouts() && settings.buildTimeout != 0) { + handlers = handlers.exclusiveJoin( + worker.aio.provider->getTimer() + .afterDelay(settings.buildTimeout.get() * kj::SECONDS) + .then([this]() -> Outcome<void, WorkResult> { + return timedOut( + Error("%1% timed out after %2% seconds", name, settings.buildTimeout) + ); + }) + ); + } + + return handlers.then([this](auto r) -> Outcome<void, WorkResult> { + if (!currentLogLine.empty()) flushLine(); + return r; + }); +} catch (...) { + return {std::current_exception()}; +} + +kj::Promise<Outcome<void, Goal::WorkResult>> DerivationGoal::monitorForSilence() noexcept { - if (!currentLogLine.empty()) flushLine(); + while (true) { + const auto stash = lastChildActivity; + auto waitUntil = lastChildActivity + settings.maxSilentTime.get() * kj::SECONDS; + co_await worker.aio.provider->getTimer().atTime(waitUntil); + if (lastChildActivity == stash) { + co_return timedOut( + Error("%1% timed out after %2% seconds of silence", name, settings.maxSilentTime) + ); + } + } } +kj::Promise<Outcome<void, Goal::WorkResult>> +DerivationGoal::handleChildStreams(InputStream & builderIn, InputStream * hookIn) noexcept +{ + lastChildActivity = worker.aio.provider->getTimer().now(); + + auto handlers = kj::joinPromisesFailFast([&] { + kj::Vector<kj::Promise<Outcome<void, WorkResult>>> parts{2}; + + parts.add(handleBuilderOutput(builderIn)); + if (hookIn) { + parts.add(handleHookOutput(*hookIn)); + } + + return parts.releaseAsArray(); + }()); + + if (respectsTimeouts() && settings.maxSilentTime != 0) { + handlers = handlers.exclusiveJoin(monitorForSilence().then([](auto r) { + return kj::arr(std::move(r)); + })); + } + + for (auto r : co_await handlers) { + BOOST_OUTCOME_CO_TRYV(r); + } + co_return result::success(); +} void DerivationGoal::flushLine() { @@ -1555,11 +1681,13 @@ SingleDrvOutputs DerivationGoal::assertPathValidity() } -Goal::Finished DerivationGoal::done( +Goal::WorkResult DerivationGoal::done( BuildResult::Status status, SingleDrvOutputs builtOutputs, std::optional<Error> ex) { + isDone = true; + outputLocks.unlock(); buildResult.status = status; if (ex) @@ -1590,7 +1718,7 @@ Goal::Finished DerivationGoal::done( logError(ex->info()); } - return Finished{ + return WorkResult{ .exitCode = buildResult.success() ? ecSuccess : ecFailed, .result = buildResult, .ex = ex ? std::make_shared<Error>(std::move(*ex)) : nullptr, @@ -1629,5 +1757,4 @@ void DerivationGoal::waiteeDone(GoalPtr waitee) } } } - } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 020388d5a..6dd58afd2 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -8,6 +8,7 @@ #include "store-api.hh" #include "pathlocks.hh" #include "goal.hh" +#include <kj/time.h> namespace nix { @@ -17,7 +18,7 @@ struct HookInstance; struct HookReplyBase { struct [[nodiscard]] Accept { - std::set<int> fds; + kj::Promise<Outcome<void, Goal::WorkResult>> promise; }; struct [[nodiscard]] Decline {}; struct [[nodiscard]] Postpone {}; @@ -62,7 +63,7 @@ struct InitialOutputStatus { struct InitialOutput { bool wanted; Hash outputHash; - std::optional<InitialOutputStatus> known; + std::optional<InitialOutputStatus> known = {}; }; /** @@ -70,6 +71,14 @@ struct InitialOutput { */ struct DerivationGoal : public Goal { + struct InputStream; + + /** + * Whether this goal has completed. Completed goals can not be + * asked for more outputs, a new goal must be created instead. + */ + bool isDone = false; + /** * Whether to use an on-disk .drv file. */ @@ -176,6 +185,11 @@ struct DerivationGoal : public Goal std::map<std::string, InitialOutput> initialOutputs; /** + * Build result. + */ + BuildResult buildResult; + + /** * File descriptor for the log file. */ AutoCloseFD fdLogFile; @@ -213,9 +227,6 @@ struct DerivationGoal : public Goal */ std::optional<DerivationType> derivationType; - typedef kj::Promise<Result<WorkResult>> (DerivationGoal::*GoalState)(bool inBuildSlot) noexcept; - GoalState state; - BuildMode buildMode; NotifyingCounter<uint64_t>::Bump mcExpectedBuilds, mcRunningBuilds; @@ -242,37 +253,35 @@ struct DerivationGoal : public Goal BuildMode buildMode = bmNormal); virtual ~DerivationGoal() noexcept(false); - Finished timedOut(Error && ex) override; - - std::string key() override; + WorkResult timedOut(Error && ex); - kj::Promise<Result<WorkResult>> work(bool inBuildSlot) noexcept override; + kj::Promise<Result<WorkResult>> workImpl() noexcept override; /** * Add wanted outputs to an already existing derivation goal. */ - void addWantedOutputs(const OutputsSpec & outputs); + bool addWantedOutputs(const OutputsSpec & outputs); /** * The states. */ - kj::Promise<Result<WorkResult>> getDerivation(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> loadDerivation(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> haveDerivation(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> outputsSubstitutionTried(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> gaveUpOnSubstitution(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> closureRepaired(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> inputsRealised(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> tryToBuild(bool inBuildSlot) noexcept; - virtual kj::Promise<Result<WorkResult>> tryLocalBuild(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> buildDone(bool inBuildSlot) noexcept; + kj::Promise<Result<WorkResult>> getDerivation() noexcept; + kj::Promise<Result<WorkResult>> loadDerivation() noexcept; + kj::Promise<Result<WorkResult>> haveDerivation() noexcept; + kj::Promise<Result<WorkResult>> outputsSubstitutionTried() noexcept; + kj::Promise<Result<WorkResult>> gaveUpOnSubstitution() noexcept; + kj::Promise<Result<WorkResult>> closureRepaired() noexcept; + kj::Promise<Result<WorkResult>> inputsRealised() noexcept; + kj::Promise<Result<WorkResult>> tryToBuild() noexcept; + virtual kj::Promise<Result<WorkResult>> tryLocalBuild() noexcept; + kj::Promise<Result<WorkResult>> buildDone() noexcept; - kj::Promise<Result<WorkResult>> resolvedFinished(bool inBuildSlot) noexcept; + kj::Promise<Result<WorkResult>> resolvedFinished() noexcept; /** * Is the build hook willing to perform the build? */ - HookReply tryBuildHook(bool inBuildSlot); + HookReply tryBuildHook(); virtual int getChildStatus(); @@ -312,13 +321,19 @@ struct DerivationGoal : public Goal virtual void cleanupPostOutputsRegisteredModeCheck(); virtual void cleanupPostOutputsRegisteredModeNonCheck(); - /** - * Callback used by the worker to write to the log. - */ - WorkResult handleChildOutput(int fd, std::string_view data) override; - void handleEOF(int fd) override; +protected: + kj::TimePoint lastChildActivity = kj::minValue; + + kj::Promise<Outcome<void, WorkResult>> handleChildOutput() noexcept; + kj::Promise<Outcome<void, WorkResult>> + handleChildStreams(InputStream & builderIn, InputStream * hookIn) noexcept; + kj::Promise<Outcome<void, WorkResult>> handleBuilderOutput(InputStream & in) noexcept; + kj::Promise<Outcome<void, WorkResult>> handleHookOutput(InputStream & in) noexcept; + kj::Promise<Outcome<void, WorkResult>> monitorForSilence() noexcept; + WorkResult tooMuchLogs(); void flushLine(); +public: /** * Wrappers around the corresponding Store methods that first consult the * derivation. This is currently needed because when there is no drv file @@ -350,13 +365,18 @@ struct DerivationGoal : public Goal void started(); - Finished done( + WorkResult done( BuildResult::Status status, SingleDrvOutputs builtOutputs = {}, std::optional<Error> ex = {}); void waiteeDone(GoalPtr waitee) override; + virtual bool respectsTimeouts() + { + return false; + } + StorePathSet exportReferences(const StorePathSet & storePaths); JobCategory jobCategory() const override { diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 7986123cc..f04beb884 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -4,6 +4,9 @@ #include "worker.hh" #include "substitution-goal.hh" #include "signals.hh" +#include <kj/array.h> +#include <kj/async.h> +#include <kj/vector.h> namespace nix { @@ -16,33 +19,32 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( : Goal(worker, isDependency) , id(id) { - state = &DrvOutputSubstitutionGoal::init; name = fmt("substitution of '%s'", id.to_string()); trace("created"); } -kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::init(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::workImpl() noexcept try { trace("init"); /* If the derivation already exists, we’re done */ if (worker.store.queryRealisation(id)) { - return {Finished{ecSuccess, std::move(buildResult)}}; + co_return WorkResult{ecSuccess}; } subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>(); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::tryNext(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::tryNext() noexcept try { trace("trying next substituter"); - if (!inBuildSlot) { - return {WaitForSlot{}}; + if (!slotToken.valid()) { + slotToken = co_await worker.substitutions.acquire(); } maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); @@ -59,7 +61,7 @@ try { /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - return {Finished{substituterFailed ? ecFailed : ecNoSubstituters, std::move(buildResult)}}; + co_return WorkResult{substituterFailed ? ecFailed : ecNoSubstituters}; } sub = subs.front(); @@ -69,25 +71,26 @@ try { some other error occurs), so it must not touch `this`. So put the shared state in a separate refcounted object. */ downloadState = std::make_shared<DownloadState>(); - downloadState->outPipe.create(); + auto pipe = kj::newPromiseAndCrossThreadFulfiller<void>(); + downloadState->outPipe = kj::mv(pipe.fulfiller); downloadState->result = std::async(std::launch::async, [downloadState{downloadState}, id{id}, sub{sub}] { + Finally updateStats([&]() { downloadState->outPipe->fulfill(); }); ReceiveInterrupts receiveInterrupts; - Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); }); return sub->queryRealisation(id); }); - state = &DrvOutputSubstitutionGoal::realisationFetched; - return {WaitForWorld{{downloadState->outPipe.readSide.get()}, true}}; + co_await pipe.promise; + co_return co_await realisationFetched(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::realisationFetched(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::realisationFetched() noexcept try { - worker.childTerminated(this); maintainRunningSubstitutions.reset(); + slotToken = {}; try { outputInfo = downloadState->result.get(); @@ -97,10 +100,10 @@ try { } if (!outputInfo) { - return tryNext(inBuildSlot); + co_return co_await tryNext(); } - WaitForGoals result; + kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies; for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { if (depId != id) { if (auto localOutputInfo = worker.store.queryRealisation(depId); @@ -114,34 +117,31 @@ try { worker.store.printStorePath(localOutputInfo->outPath), worker.store.printStorePath(depPath) ); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } - result.goals.insert(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); + dependencies.add(worker.goalFactory().makeDrvOutputSubstitutionGoal(depId)); } } - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(outputInfo->outPath)); - if (result.goals.empty()) { - return outPathValid(inBuildSlot); - } else { - state = &DrvOutputSubstitutionGoal::outPathValid; - return {std::move(result)}; + if (!dependencies.empty()) { + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await outPathValid(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::outPathValid(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::outPathValid() noexcept try { assert(outputInfo); trace("output path substituted"); if (nrFailed > 0) { debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - return {Finished{ + return {WorkResult{ nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, - std::move(buildResult), }}; } @@ -154,22 +154,9 @@ try { kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::finished() noexcept try { trace("finished"); - return {Finished{ecSuccess, std::move(buildResult)}}; + return {WorkResult{ecSuccess}}; } catch (...) { return {std::current_exception()}; } -std::string DrvOutputSubstitutionGoal::key() -{ - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(id.to_string()); -} - -kj::Promise<Result<Goal::WorkResult>> DrvOutputSubstitutionGoal::work(bool inBuildSlot) noexcept -{ - return (this->*state)(inBuildSlot); -} - - } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index f33196665..f959e2a7b 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -45,7 +45,7 @@ class DrvOutputSubstitutionGoal : public Goal { struct DownloadState { - Pipe outPipe; + kj::Own<kj::CrossThreadPromiseFulfiller<void>> outPipe; std::future<std::shared_ptr<const Realisation>> result; }; @@ -65,20 +65,12 @@ public: std::optional<ContentAddress> ca = std::nullopt ); - typedef kj::Promise<Result<WorkResult>> (DrvOutputSubstitutionGoal::*GoalState)(bool inBuildSlot) noexcept; - GoalState state; - - kj::Promise<Result<WorkResult>> init(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> tryNext(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> realisationFetched(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> outPathValid(bool inBuildSlot) noexcept; + kj::Promise<Result<WorkResult>> tryNext() noexcept; + kj::Promise<Result<WorkResult>> realisationFetched() noexcept; + kj::Promise<Result<WorkResult>> outPathValid() noexcept; kj::Promise<Result<WorkResult>> finished() noexcept; - Finished timedOut(Error && ex) override { abort(); }; - - std::string key() override; - - kj::Promise<Result<WorkResult>> work(bool inBuildSlot) noexcept override; + kj::Promise<Result<WorkResult>> workImpl() noexcept override; JobCategory jobCategory() const override { return JobCategory::Substitution; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index a0f18a02c..808179a4d 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -17,22 +17,22 @@ void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMod Worker worker(*this, evalStore ? *evalStore : *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - Goals goals; + Worker::Targets goals; for (auto & br : reqs) - goals.insert(gf.makeGoal(br, buildMode)); + goals.emplace(gf.makeGoal(br, buildMode)); return goals; }); StringSet failed; std::shared_ptr<Error> ex; - for (auto & i : goals) { - if (i->ex) { + for (auto & [i, result] : goals) { + if (result.ex) { if (ex) - logError(i->ex->info()); + logError(result.ex->info()); else - ex = i->ex; + ex = result.ex; } - if (i->exitCode != Goal::ecSuccess) { + if (result.exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(printStorePath(i2->drvPath)); else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) @@ -60,11 +60,11 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults( std::vector<std::pair<const DerivedPath &, GoalPtr>> state; auto goals = runWorker(worker, [&](GoalFactory & gf) { - Goals goals; + Worker::Targets goals; for (const auto & req : reqs) { auto goal = gf.makeGoal(req, buildMode); - goals.insert(goal); - state.push_back({req, goal}); + state.push_back({req, goal.first}); + goals.emplace(std::move(goal)); } return goals; }); @@ -72,7 +72,7 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults( std::vector<KeyedBuildResult> results; for (auto & [req, goalPtr] : state) - results.emplace_back(goalPtr->buildResult.restrictTo(req)); + results.emplace_back(goals[goalPtr].result.restrictTo(req)); return results; } @@ -84,11 +84,13 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat Worker worker(*this, *this, aio); try { - auto goals = runWorker(worker, [&](GoalFactory & gf) -> Goals { - return Goals{gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)}; + auto goals = runWorker(worker, [&](GoalFactory & gf) { + Worker::Targets goals; + goals.emplace(gf.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All{}, buildMode)); + return goals; }); - auto goal = *goals.begin(); - return goal->buildResult.restrictTo(DerivedPath::Built { + auto [goal, result] = *goals.begin(); + return result.result.restrictTo(DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); @@ -110,14 +112,16 @@ void Store::ensurePath(const StorePath & path) Worker worker(*this, *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - return Goals{gf.makePathSubstitutionGoal(path)}; + Worker::Targets goals; + goals.emplace(gf.makePathSubstitutionGoal(path)); + return goals; }); - auto goal = *goals.begin(); + auto [goal, result] = *goals.begin(); - if (goal->exitCode != Goal::ecSuccess) { - if (goal->ex) { - goal->ex->withExitStatus(worker.failingExitStatus()); - throw std::move(*goal->ex); + if (result.exitCode != Goal::ecSuccess) { + if (result.ex) { + result.ex->withExitStatus(worker.failingExitStatus()); + throw std::move(*result.ex); } else throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); } @@ -130,24 +134,28 @@ void Store::repairPath(const StorePath & path) Worker worker(*this, *this, aio); auto goals = runWorker(worker, [&](GoalFactory & gf) { - return Goals{gf.makePathSubstitutionGoal(path, Repair)}; + Worker::Targets goals; + goals.emplace(gf.makePathSubstitutionGoal(path, Repair)); + return goals; }); - auto goal = *goals.begin(); + auto [goal, result] = *goals.begin(); - if (goal->exitCode != Goal::ecSuccess) { + if (result.exitCode != Goal::ecSuccess) { /* Since substituting the path didn't work, if we have a valid deriver, then rebuild the deriver. */ auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { worker.run([&](GoalFactory & gf) { - return Goals{gf.makeGoal( + Worker::Targets goals; + goals.emplace(gf.makeGoal( DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*info->deriver), // FIXME: Should just build the specific output we need. .outputs = OutputsSpec::All{}, }, bmRepair - )}; + )); + return goals; }); } else throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 82861ad2b..02b22b8ad 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -1,18 +1,73 @@ #include "goal.hh" +#include "async-collect.hh" +#include "worker.hh" +#include <boost/outcome/try.hpp> +#include <kj/time.h> namespace nix { -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - std::string s1 = a->key(); - std::string s2 = b->key(); - return s1 < s2; -} - - void Goal::trace(std::string_view s) { debug("%1%: %2%", name, s); } +kj::Promise<void> Goal::waitForAWhile() +{ + trace("wait for a while"); + /* If we are polling goals that are waiting for a lock, then wake + up after a few seconds at most. */ + return worker.aio.provider->getTimer().afterDelay(settings.pollInterval.get() * kj::SECONDS); +} + +kj::Promise<Result<Goal::WorkResult>> Goal::work() noexcept +try { + BOOST_OUTCOME_CO_TRY(auto result, co_await workImpl()); + + trace("done"); + + cleanup(); + + co_return std::move(result); +} catch (...) { + co_return result::failure(std::current_exception()); +} + +kj::Promise<Result<void>> +Goal::waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies) noexcept +try { + auto left = dependencies.size(); + for (auto & [dep, p] : dependencies) { + p = p.then([this, dep, &left](auto _result) -> Result<WorkResult> { + BOOST_OUTCOME_TRY(auto result, _result); + + left--; + trace(fmt("waitee '%s' done; %d left", dep->name, left)); + + if (result.exitCode != Goal::ecSuccess) ++nrFailed; + if (result.exitCode == Goal::ecNoSubstituters) ++nrNoSubstituters; + if (result.exitCode == Goal::ecIncompleteClosure) ++nrIncompleteClosure; + + return std::move(result); + }).eagerlyEvaluate(nullptr); + } + + auto collectDeps = asyncCollect(std::move(dependencies)); + + while (auto item = co_await collectDeps.next()) { + auto & [dep, _result] = *item; + BOOST_OUTCOME_CO_TRY(auto result, _result); + + waiteeDone(dep); + + if (result.exitCode == ecFailed && !settings.keepGoing) { + co_return result::success(); + } + } + + co_return result::success(); +} catch (...) { + co_return result::failure(std::current_exception()); +} + } diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 189505308..29540dcd3 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -1,10 +1,12 @@ #pragma once ///@file +#include "async-semaphore.hh" #include "result.hh" #include "types.hh" #include "store-api.hh" #include "build-result.hh" +#include <concepts> // IWYU pragma: keep #include <kj/async.h> namespace nix { @@ -19,22 +21,11 @@ class Worker; * A pointer to a goal. */ typedef std::shared_ptr<Goal> GoalPtr; -typedef std::weak_ptr<Goal> WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; /** * Set of goals. */ -typedef std::set<GoalPtr, CompareGoalPtrs> Goals; -typedef std::set<WeakGoalPtr, std::owner_less<WeakGoalPtr>> WeakGoals; - -/** - * A map of paths to goals (and the other way around). - */ -typedef std::map<StorePath, WeakGoalPtr> WeakGoalMap; +typedef std::set<GoalPtr> Goals; /** * Used as a hint to the worker on how to schedule a particular goal. For example, @@ -70,17 +61,6 @@ struct Goal const bool isDependency; /** - * Goals that this goal is waiting for. - */ - Goals waitees; - - /** - * Goals waiting for this one to finish. Must use weak pointers - * here to prevent cycles. - */ - WeakGoals waiters; - - /** * Number of goals we are/were waiting for that have failed. */ size_t nrFailed = 0; @@ -102,57 +82,37 @@ struct Goal */ std::string name; - /** - * Whether the goal is finished. - */ - std::optional<ExitCode> exitCode; - - /** - * Build result. - */ - BuildResult buildResult; +protected: + AsyncSemaphore::Token slotToken; public: - - struct [[nodiscard]] StillAlive {}; - struct [[nodiscard]] WaitForSlot {}; - struct [[nodiscard]] WaitForAWhile {}; - struct [[nodiscard]] ContinueImmediately {}; - struct [[nodiscard]] WaitForGoals { - Goals goals; - }; - struct [[nodiscard]] WaitForWorld { - std::set<int> fds; - bool inBuildSlot; - }; - struct [[nodiscard]] Finished { + struct [[nodiscard]] WorkResult { ExitCode exitCode; - BuildResult result; - std::shared_ptr<Error> ex; + BuildResult result = {}; + std::shared_ptr<Error> ex = {}; bool permanentFailure = false; bool timedOut = false; bool hashMismatch = false; bool checkMismatch = false; }; - struct [[nodiscard]] WorkResult : std::variant< - StillAlive, - WaitForSlot, - WaitForAWhile, - ContinueImmediately, - WaitForGoals, - WaitForWorld, - Finished> +protected: + kj::Promise<void> waitForAWhile(); + kj::Promise<Result<void>> + waitForGoals(kj::Array<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies) noexcept; + + template<std::derived_from<Goal>... G> + kj::Promise<Result<void>> + waitForGoals(std::pair<std::shared_ptr<G>, kj::Promise<Result<WorkResult>>>... goals) noexcept { - WorkResult() = delete; - using variant::variant; - }; + return waitForGoals( + kj::arrOf<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>>(std::move(goals)...) + ); + } - /** - * Exception containing an error message, if any. - */ - std::shared_ptr<Error> ex; + virtual kj::Promise<Result<WorkResult>> workImpl() noexcept = 0; +public: explicit Goal(Worker & worker, bool isDependency) : worker(worker) , isDependency(isDependency) @@ -163,24 +123,10 @@ public: trace("goal destroyed"); } - virtual kj::Promise<Result<WorkResult>> work(bool inBuildSlot) noexcept = 0; + kj::Promise<Result<WorkResult>> work() noexcept; virtual void waiteeDone(GoalPtr waitee) { } - virtual WorkResult handleChildOutput(int fd, std::string_view data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - } - - virtual bool respectsTimeouts() - { - return false; - } - void trace(std::string_view s); std::string getName() const @@ -188,15 +134,6 @@ public: return name; } - /** - * Callback in case of a timeout. It should wake up its waiters, - * get rid of any running child processes that are being monitored - * by the worker (important!), etc. - */ - virtual Finished timedOut(Error && ex) = 0; - - virtual std::string key() = 0; - virtual void cleanup() { } /** diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index f91a904cc..521f34917 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,4 +1,5 @@ #include "child.hh" +#include "error.hh" #include "file-system.hh" #include "globals.hh" #include "hook-instance.hh" @@ -86,7 +87,7 @@ HookInstance::~HookInstance() toHook.reset(); if (pid) pid.kill(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4baa525d9..c8c68f99f 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,4 +1,5 @@ #include "local-derivation-goal.hh" +#include "error.hh" #include "indirect-root-store.hh" #include "machines.hh" #include "store-api.hh" @@ -98,9 +99,9 @@ LocalDerivationGoal::~LocalDerivationGoal() noexcept(false) { /* Careful: we should never ever throw an exception from a destructor. */ - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } + try { deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } + try { stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -121,8 +122,6 @@ LocalStore & LocalDerivationGoal::getLocalStore() void LocalDerivationGoal::killChild() { if (pid) { - worker.childTerminated(this); - /* If we're using a build user, then there is a tricky race condition: if we kill the build user before the child has done its setuid() to the build user uid, then it won't be @@ -149,17 +148,18 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -kj::Promise<Result<Goal::WorkResult>> LocalDerivationGoal::tryLocalBuild(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> LocalDerivationGoal::tryLocalBuild() noexcept try { +retry: #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); #endif - if (!inBuildSlot) { - state = &DerivationGoal::tryToBuild; + if (!slotToken.valid()) { outputLocks.unlock(); - if (0U != settings.maxBuildJobs) { - return {WaitForSlot{}}; + if (worker.localBuilds.capacity() > 0) { + slotToken = co_await worker.localBuilds.acquire(); + co_return co_await tryToBuild(); } if (getMachines().empty()) { throw Error( @@ -214,7 +214,9 @@ try { if (!actLock) actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); - return {WaitForAWhile{}}; + co_await waitForAWhile(); + // we can loop very often, and `co_return co_await` always allocates a new frame + goto retry; } } @@ -243,24 +245,29 @@ try { try { /* Okay, we have to build. */ - auto fds = startBuilder(); - - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; + auto promise = startBuilder(); started(); - return {WaitForWorld{std::move(fds), true}}; + auto r = co_await promise; + if (r.has_value()) { + // all good so far + } else if (r.has_error()) { + co_return r.assume_error(); + } else { + co_return r.assume_exception(); + } } catch (BuildError & e) { outputLocks.unlock(); buildUser.reset(); auto report = done(BuildResult::InputRejected, {}, std::move(e)); report.permanentFailure = true; - return {std::move(report)}; + co_return report; } + + co_return co_await buildDone(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } @@ -390,7 +397,9 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() cleanupPostOutputsRegisteredModeCheck(); } -std::set<int> LocalDerivationGoal::startBuilder() +// NOTE this one isn't noexcept because it's called from places that expect +// exceptions to signal failure to launch. we should change this some time. +kj::Promise<Outcome<void, Goal::WorkResult>> LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) #if __linux__ @@ -779,7 +788,7 @@ std::set<int> LocalDerivationGoal::startBuilder() msgs.push_back(std::move(msg)); } - return {builderOutPTY.get()}; + return handleChildOutput(); } @@ -1241,7 +1250,7 @@ void LocalDerivationGoal::startDaemon() NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SysError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } }); @@ -1361,13 +1370,20 @@ void LocalDerivationGoal::runChild() bool setUser = true; - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ + /* Make the contents of netrc and the CA certificate bundle + available to builtin:fetchurl (which may run under a + different uid and/or in a sandbox). */ std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl" && !derivationType->isSandboxed()) + std::string caFileData; + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl" && !derivationType->isSandboxed()) { + try { netrcData = readFile(settings.netrcFile); - } catch (SysError &) { } + } catch (SysError &) { } + + try { + caFileData = readFile(settings.caFile); + } catch (SysError &) { } + } #if __linux__ if (useChroot) { @@ -1802,7 +1818,7 @@ void LocalDerivationGoal::runChild() e.second = rewriteStrings(e.second, inputRewrites); if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); + builtinFetchurl(drv2, netrcData, caFileData); else if (drv->builder == "builtin:buildenv") builtinBuildenv(drv2); else if (drv->builder == "builtin:unpack-channel") diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index cd040bc15..cd6ea2b55 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -182,7 +182,7 @@ struct LocalDerivationGoal : public DerivationGoal * Create a LocalDerivationGoal without an on-disk .drv file, * possibly a platform-specific subclass */ - static std::shared_ptr<LocalDerivationGoal> makeLocalDerivationGoal( + static std::unique_ptr<LocalDerivationGoal> makeLocalDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, @@ -194,7 +194,7 @@ struct LocalDerivationGoal : public DerivationGoal * Create a LocalDerivationGoal for an on-disk .drv file, * possibly a platform-specific subclass */ - static std::shared_ptr<LocalDerivationGoal> makeLocalDerivationGoal( + static std::unique_ptr<LocalDerivationGoal> makeLocalDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -213,12 +213,12 @@ struct LocalDerivationGoal : public DerivationGoal /** * The additional states. */ - kj::Promise<Result<WorkResult>> tryLocalBuild(bool inBuildSlot) noexcept override; + kj::Promise<Result<WorkResult>> tryLocalBuild() noexcept override; /** * Start building a derivation. */ - std::set<int> startBuilder(); + kj::Promise<Outcome<void, WorkResult>> startBuilder(); /** * Fill in the environment for the builder. diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index bd0ffcb9b..e0ca23a86 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -3,6 +3,8 @@ #include "nar-info.hh" #include "signals.hh" #include "finally.hh" +#include <kj/array.h> +#include <kj/vector.h> namespace nix { @@ -18,7 +20,6 @@ PathSubstitutionGoal::PathSubstitutionGoal( , repair(repair) , ca(ca) { - state = &PathSubstitutionGoal::init; name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); trace("created"); maintainExpectedSubstitutions = worker.expectedSubstitutions.addTemporarily(1); @@ -31,27 +32,21 @@ PathSubstitutionGoal::~PathSubstitutionGoal() } -Goal::Finished PathSubstitutionGoal::done( +Goal::WorkResult PathSubstitutionGoal::done( ExitCode result, BuildResult::Status status, std::optional<std::string> errorMsg) { - buildResult.status = status; + BuildResult buildResult{.status = status}; if (errorMsg) { debug(*errorMsg); buildResult.errorMsg = *errorMsg; } - return Finished{result, std::move(buildResult)}; + return WorkResult{result, std::move(buildResult)}; } -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::work(bool inBuildSlot) noexcept -{ - return (this->*state)(inBuildSlot); -} - - -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::init(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::workImpl() noexcept try { trace("init"); @@ -67,13 +62,13 @@ try { subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>(); - return tryNext(inBuildSlot); + return tryNext(); } catch (...) { return {std::current_exception()}; } -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::tryNext(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::tryNext() noexcept try { trace("trying next substituter"); @@ -89,10 +84,10 @@ try { /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - return {done( + co_return done( substituterFailed ? ecFailed : ecNoSubstituters, BuildResult::NoSubstituters, - fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath)))}; + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); } sub = subs.front(); @@ -105,26 +100,28 @@ try { if (sub->storeDir == worker.store.storeDir) assert(subPath == storePath); } else if (sub->storeDir != worker.store.storeDir) { - return tryNext(inBuildSlot); + co_return co_await tryNext(); } - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - return tryNext(inBuildSlot); - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - return tryNext(inBuildSlot); - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - return tryNext(inBuildSlot); + do { + try { + // FIXME: make async + info = sub->queryPathInfo(subPath ? *subPath : storePath); + break; + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + if (!settings.tryFallback) { + throw; + } + } catch (Error & e) { + if (settings.tryFallback) { + logError(e.info()); + } else { + throw; + } } - throw; - } + co_return co_await tryNext(); + } while (false); if (info->path != storePath) { if (info->isContentAddressed(*sub) && info->references.empty()) { @@ -134,7 +131,7 @@ try { } else { printError("asked '%s' for '%s' but got '%s'", sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } } @@ -155,28 +152,26 @@ try { { warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); - return tryNext(inBuildSlot); + co_return co_await tryNext(); } /* To maintain the closure invariant, we first have to realise the paths referenced by this one. */ - WaitForGoals result; + kj::Vector<std::pair<GoalPtr, kj::Promise<Result<WorkResult>>>> dependencies; for (auto & i : info->references) if (i != storePath) /* ignore self-references */ - result.goals.insert(worker.goalFactory().makePathSubstitutionGoal(i)); + dependencies.add(worker.goalFactory().makePathSubstitutionGoal(i)); - if (result.goals.empty()) {/* to prevent hang (no wake-up event) */ - return referencesValid(inBuildSlot); - } else { - state = &PathSubstitutionGoal::referencesValid; - return {std::move(result)}; + if (!dependencies.empty()) {/* to prevent hang (no wake-up event) */ + (co_await waitForGoals(dependencies.releaseAsArray())).value(); } + co_return co_await referencesValid(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::referencesValid(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::referencesValid() noexcept try { trace("all references realised"); @@ -191,33 +186,33 @@ try { if (i != storePath) /* ignore self-references */ assert(worker.store.isValidPath(i)); - state = &PathSubstitutionGoal::tryToRun; - return tryToRun(inBuildSlot); + return tryToRun(); } catch (...) { return {std::current_exception()}; } -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::tryToRun(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::tryToRun() noexcept try { trace("trying to run"); - if (!inBuildSlot) { - return {WaitForSlot{}}; + if (!slotToken.valid()) { + slotToken = co_await worker.substitutions.acquire(); } maintainRunningSubstitutions = worker.runningSubstitutions.addTemporarily(1); - outPipe.create(); + auto pipe = kj::newPromiseAndCrossThreadFulfiller<void>(); + outPipe = kj::mv(pipe.fulfiller); thr = std::async(std::launch::async, [this]() { + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe->fulfill(); }); + auto & fetchPath = subPath ? *subPath : storePath; try { ReceiveInterrupts receiveInterrupts; - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide.close(); }); - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); @@ -233,39 +228,39 @@ try { } }); - state = &PathSubstitutionGoal::finished; - return {WaitForWorld{{outPipe.readSide.get()}, true}}; + co_await pipe.promise; + co_return co_await finished(); } catch (...) { - return {std::current_exception()}; + co_return result::failure(std::current_exception()); } -kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::finished(bool inBuildSlot) noexcept +kj::Promise<Result<Goal::WorkResult>> PathSubstitutionGoal::finished() noexcept try { trace("substitute finished"); - worker.childTerminated(this); - - try { - thr.get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ + do { try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; + slotToken = {}; + thr.get(); + break; + } catch (std::exception & e) { + printError(e.what()); + + /* Cause the parent build to fail unless --fallback is given, + or the substitute has disappeared. The latter case behaves + the same as the substitute never having existed in the + first place. */ + try { + throw; + } catch (SubstituteGone &) { + } catch (...) { + substituterFailed = true; + } } - /* Try the next substitute. */ - state = &PathSubstitutionGoal::tryNext; - return tryNext(inBuildSlot); - } + co_return co_await tryNext(); + } while (false); worker.markContentsGood(storePath); @@ -282,15 +277,9 @@ try { worker.doneNarSize += maintainExpectedNar.delta(); maintainExpectedNar.reset(); - return {done(ecSuccess, BuildResult::Substituted)}; + co_return done(ecSuccess, BuildResult::Substituted); } catch (...) { - return {std::current_exception()}; -} - - -Goal::WorkResult PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data) -{ - return StillAlive{}; + co_return result::failure(std::current_exception()); } @@ -300,12 +289,9 @@ void PathSubstitutionGoal::cleanup() if (thr.valid()) { // FIXME: signal worker thread to quit. thr.get(); - worker.childTerminated(this); } - - outPipe.close(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 3c97b19fd..18b4262a4 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -46,7 +46,7 @@ struct PathSubstitutionGoal : public Goal /** * Pipe for the substituter's standard output. */ - Pipe outPipe; + kj::Own<kj::CrossThreadPromiseFulfiller<void>> outPipe; /** * The substituter thread. @@ -67,15 +67,12 @@ struct PathSubstitutionGoal : public Goal NotifyingCounter<uint64_t>::Bump maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef kj::Promise<Result<WorkResult>> (PathSubstitutionGoal::*GoalState)(bool inBuildSlot) noexcept; - GoalState state; - /** * Content address for recomputing store path */ std::optional<ContentAddress> ca; - Finished done( + WorkResult done( ExitCode result, BuildResult::Status status, std::optional<std::string> errorMsg = {}); @@ -90,32 +87,15 @@ public: ); ~PathSubstitutionGoal(); - Finished timedOut(Error && ex) override { abort(); }; - - /** - * We prepend "a$" to the key name to ensure substitution goals - * happen before derivation goals. - */ - std::string key() override - { - return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); - } - - kj::Promise<Result<WorkResult>> work(bool inBuildSlot) noexcept override; + kj::Promise<Result<WorkResult>> workImpl() noexcept override; /** * The states. */ - kj::Promise<Result<WorkResult>> init(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> tryNext(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> referencesValid(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> tryToRun(bool inBuildSlot) noexcept; - kj::Promise<Result<WorkResult>> finished(bool inBuildSlot) noexcept; - - /** - * Callback used by the worker to write to the log. - */ - WorkResult handleChildOutput(int fd, std::string_view data) override; + kj::Promise<Result<WorkResult>> tryNext() noexcept; + kj::Promise<Result<WorkResult>> referencesValid() noexcept; + kj::Promise<Result<WorkResult>> tryToRun() noexcept; + kj::Promise<Result<WorkResult>> finished() noexcept; /* Called by destructor, can't be overridden */ void cleanup() override final; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index ee45c7e3f..10f58f5d3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,3 +1,4 @@ +#include "async-collect.hh" #include "charptr-cast.hh" #include "worker.hh" #include "finally.hh" @@ -6,11 +7,22 @@ #include "local-derivation-goal.hh" #include "signals.hh" #include "hook-instance.hh" // IWYU pragma: keep - -#include <poll.h> +#include <boost/outcome/try.hpp> +#include <kj/vector.h> namespace nix { +namespace { +struct ErrorHandler : kj::TaskSet::ErrorHandler +{ + void taskFailed(kj::Exception && e) override + { + printError("unexpected async failure in Worker: %s", kj::str(e).cStr()); + abort(); + } +} errorHandler; +} + Worker::Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio) : act(*logger, actRealise) , actDerivations(*logger, actBuilds) @@ -18,11 +30,13 @@ Worker::Worker(Store & store, Store & evalStore, kj::AsyncIoContext & aio) , store(store) , evalStore(evalStore) , aio(aio) + /* Make sure that we are always allowed to run at least one substitution. + This prevents infinite waiting. */ + , substitutions(std::max<unsigned>(1, settings.maxSubstitutionJobs)) + , localBuilds(settings.maxBuildJobs) + , children(errorHandler) { /* Debugging: prevent recursive workers. */ - nrLocalBuilds = 0; - nrSubstitutions = 0; - lastWokenUp = steady_time_point::min(); } @@ -32,7 +46,11 @@ Worker::~Worker() goals that refer to this worker should be gone. (Otherwise we are in trouble, since goals may call childTerminated() etc. in their destructors). */ - topGoals.clear(); + children.clear(); + + derivationGoals.clear(); + drvOutputSubstitutionGoals.clear(); + substitutionGoals.clear(); assert(expectedSubstitutions == 0); assert(expectedDownloadSize == 0); @@ -40,292 +58,158 @@ Worker::~Worker() } -std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon( - const StorePath & drvPath, - const OutputsSpec & wantedOutputs, - std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal) +template<typename ID, std::derived_from<Goal> G> +std::pair<std::shared_ptr<G>, kj::Promise<Result<Goal::WorkResult>>> Worker::makeGoalCommon( + std::map<ID, CachedGoal<G>> & map, + const ID & key, + InvocableR<std::unique_ptr<G>> auto create, + InvocableR<bool, G &> auto modify +) { - std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath]; - std::shared_ptr<DerivationGoal> goal = goal_weak.lock(); - if (!goal) { - goal = mkDrvGoal(); - goal_weak = goal; - wakeUp(goal); - } else { - goal->addWantedOutputs(wantedOutputs); + auto [it, _inserted] = map.try_emplace(key); + // try twice to create the goal. we can only loop if we hit the continue, + // and then we only want to recreate the goal *once*. concurrent accesses + // to the worker are not sound, we want to catch them if at all possible. + for ([[maybe_unused]] auto _attempt : {1, 2}) { + auto & cachedGoal = it->second; + auto & goal = cachedGoal.goal; + if (!goal) { + goal = create(); + // do not start working immediately. if we are not yet running we + // may create dependencies as though they were toplevel goals, in + // which case the dependencies will not report build errors. when + // we are running we may be called for this same goal more times, + // and then we want to modify rather than recreate when possible. + auto removeWhenDone = [goal, &map, it] { + // c++ lambda coroutine capture semantics are *so* fucked up. + return [](auto goal, auto & map, auto it) -> kj::Promise<Result<Goal::WorkResult>> { + auto result = co_await goal->work(); + // a concurrent call to makeGoalCommon may have reset our + // cached goal and replaced it with a new instance. don't + // remove the goal in this case, otherwise we will crash. + if (goal == it->second.goal) { + map.erase(it); + } + co_return result; + }(goal, map, it); + }; + cachedGoal.promise = kj::evalLater(std::move(removeWhenDone)).fork(); + children.add(cachedGoal.promise.addBranch().then([this](auto _result) { + if (_result.has_value()) { + auto & result = _result.value(); + permanentFailure |= result.permanentFailure; + timedOut |= result.timedOut; + hashMismatch |= result.hashMismatch; + checkMismatch |= result.checkMismatch; + } + })); + } else { + if (!modify(*goal)) { + cachedGoal = {}; + continue; + } + } + return {goal, cachedGoal.promise.addBranch()}; } - return goal; + assert(false && "could not make a goal. possible concurrent worker access"); } -std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath, - const OutputsSpec & wantedOutputs, BuildMode buildMode) +std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> Worker::makeDerivationGoal( + const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode +) { - return makeDerivationGoalCommon( + return makeGoalCommon( + derivationGoals, drvPath, - wantedOutputs, - [&]() -> std::shared_ptr<DerivationGoal> { + [&]() -> std::unique_ptr<DerivationGoal> { return !dynamic_cast<LocalStore *>(&store) - ? std::make_shared<DerivationGoal>( + ? std::make_unique<DerivationGoal>( drvPath, wantedOutputs, *this, running, buildMode ) : LocalDerivationGoal::makeLocalDerivationGoal( drvPath, wantedOutputs, *this, running, buildMode ); - } + }, + [&](DerivationGoal & g) { return g.addWantedOutputs(wantedOutputs); } ); } -std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath, - const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) +std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> Worker::makeBasicDerivationGoal( + const StorePath & drvPath, + const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, + BuildMode buildMode +) { - return makeDerivationGoalCommon( + return makeGoalCommon( + derivationGoals, drvPath, - wantedOutputs, - [&]() -> std::shared_ptr<DerivationGoal> { + [&]() -> std::unique_ptr<DerivationGoal> { return !dynamic_cast<LocalStore *>(&store) - ? std::make_shared<DerivationGoal>( + ? std::make_unique<DerivationGoal>( drvPath, drv, wantedOutputs, *this, running, buildMode ) : LocalDerivationGoal::makeLocalDerivationGoal( drvPath, drv, wantedOutputs, *this, running, buildMode ); - } + }, + [&](DerivationGoal & g) { return g.addWantedOutputs(wantedOutputs); } ); } -std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca) +std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> +Worker::makePathSubstitutionGoal( + const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca +) { - std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path]; - auto goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared<PathSubstitutionGoal>(path, *this, running, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; + return makeGoalCommon( + substitutionGoals, + path, + [&] { return std::make_unique<PathSubstitutionGoal>(path, *this, running, repair, ca); }, + [&](auto &) { return true; } + ); } -std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca) +std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> +Worker::makeDrvOutputSubstitutionGoal( + const DrvOutput & id, RepairFlag repair, std::optional<ContentAddress> ca +) { - std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id]; - auto goal = goal_weak.lock(); // FIXME - if (!goal) { - goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, running, repair, ca); - goal_weak = goal; - wakeUp(goal); - } - return goal; + return makeGoalCommon( + drvOutputSubstitutionGoals, + id, + [&] { return std::make_unique<DrvOutputSubstitutionGoal>(id, *this, running, repair, ca); }, + [&](auto &) { return true; } + ); } -GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) +std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) -> GoalPtr { + [&](const DerivedPath::Built & bfd) -> std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> { if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath)) return makeDerivationGoal(bop->path, bfd.outputs, buildMode); else throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, - [&](const DerivedPath::Opaque & bo) -> GoalPtr { + [&](const DerivedPath::Opaque & bo) -> std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); }, }, req.raw()); } +kj::Promise<Result<Worker::Results>> Worker::updateStatistics() +try { + while (true) { + statisticsUpdateInhibitor = co_await statisticsUpdateSignal.acquire(); -template<typename K, typename G> -static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap) -{ - /* !!! inefficient */ - for (auto i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - auto j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; -} - - -void Worker::goalFinished(GoalPtr goal, Goal::Finished & f) -{ - goal->trace("done"); - assert(!goal->exitCode.has_value()); - goal->exitCode = f.exitCode; - goal->ex = f.ex; - - permanentFailure |= f.permanentFailure; - timedOut |= f.timedOut; - hashMismatch |= f.hashMismatch; - checkMismatch |= f.checkMismatch; - - for (auto & i : goal->waiters) { - if (GoalPtr waiting = i.lock()) { - assert(waiting->waitees.count(goal)); - waiting->waitees.erase(goal); - - waiting->trace(fmt("waitee '%s' done; %d left", goal->name, waiting->waitees.size())); - - if (f.exitCode != Goal::ecSuccess) ++waiting->nrFailed; - if (f.exitCode == Goal::ecNoSubstituters) ++waiting->nrNoSubstituters; - if (f.exitCode == Goal::ecIncompleteClosure) ++waiting->nrIncompleteClosure; - - if (waiting->waitees.empty() || (f.exitCode == Goal::ecFailed && !settings.keepGoing)) { - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & i : waiting->waitees) { - i->waiters.extract(waiting); - } - waiting->waitees.clear(); - - wakeUp(waiting); - } - - waiting->waiteeDone(goal); - } - } - goal->waiters.clear(); - removeGoal(goal); - goal->cleanup(); -} - -void Worker::handleWorkResult(GoalPtr goal, Goal::WorkResult how) -{ - std::visit( - overloaded{ - [&](Goal::StillAlive) {}, - [&](Goal::WaitForSlot) { waitForBuildSlot(goal); }, - [&](Goal::WaitForAWhile) { waitForAWhile(goal); }, - [&](Goal::ContinueImmediately) { wakeUp(goal); }, - [&](Goal::WaitForGoals & w) { - for (auto & dep : w.goals) { - goal->waitees.insert(dep); - dep->waiters.insert(goal); - } - }, - [&](Goal::WaitForWorld & w) { childStarted(goal, w.fds, w.inBuildSlot); }, - [&](Goal::Finished & f) { goalFinished(goal, f); }, - }, - how - ); -} - -void Worker::removeGoal(GoalPtr goal) -{ - if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal)) - nix::removeGoal(drvGoal, derivationGoals); - else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal)) - nix::removeGoal(subGoal, substitutionGoals); - else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal)) - nix::removeGoal(subGoal, drvOutputSubstitutionGoals); - else - assert(false); - - if (topGoals.find(goal) != topGoals.end()) { - topGoals.erase(goal); - /* If a top-level goal failed, then kill all other goals - (unless keepGoing was set). */ - if (goal->exitCode == Goal::ecFailed && !settings.keepGoing) - topGoals.clear(); - } -} - - -void Worker::wakeUp(GoalPtr goal) -{ - goal->trace("woken up"); - awake.insert(goal); -} - - -void Worker::childStarted(GoalPtr goal, const std::set<int> & fds, - bool inBuildSlot) -{ - Child child; - child.goal = goal; - child.goal2 = goal.get(); - child.fds = fds; - child.timeStarted = child.lastOutput = steady_time_point::clock::now(); - child.inBuildSlot = inBuildSlot; - children.emplace_back(child); - if (inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: - nrSubstitutions++; - break; - case JobCategory::Build: - nrLocalBuilds++; - break; - default: - abort(); - } - } -} - - -void Worker::childTerminated(Goal * goal) -{ - auto i = std::find_if(children.begin(), children.end(), - [&](const Child & child) { return child.goal2 == goal; }); - if (i == children.end()) return; - - if (i->inBuildSlot) { - switch (goal->jobCategory()) { - case JobCategory::Substitution: - assert(nrSubstitutions > 0); - nrSubstitutions--; - break; - case JobCategory::Build: - assert(nrLocalBuilds > 0); - nrLocalBuilds--; - break; - default: - abort(); - } - } - - children.erase(i); - - /* Wake up goals waiting for a build slot. */ - for (auto & j : wantingToBuild) { - GoalPtr goal = j.lock(); - if (goal) wakeUp(goal); - } - - wantingToBuild.clear(); -} - - -void Worker::waitForBuildSlot(GoalPtr goal) -{ - goal->trace("wait for build slot"); - bool isSubstitutionGoal = goal->jobCategory() == JobCategory::Substitution; - if ((!isSubstitutionGoal && nrLocalBuilds < settings.maxBuildJobs) || - (isSubstitutionGoal && nrSubstitutions < settings.maxSubstitutionJobs)) - wakeUp(goal); /* we can do it right away */ - else - wantingToBuild.insert(goal); -} - - -void Worker::waitForAWhile(GoalPtr goal) -{ - debug("wait for a while"); - waitingForAWhile.insert(goal); -} - - -void Worker::updateStatistics() -{ - // only update progress info while running. this notably excludes updating - // progress info while destroying, which causes the progress bar to assert - if (running && statisticsOutdated) { + // only update progress info while running. this notably excludes updating + // progress info while destroying, which causes the progress bar to assert actDerivations.progress( doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds ); @@ -338,221 +222,82 @@ void Worker::updateStatistics() act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - statisticsOutdated = false; + // limit to 50fps. that should be more than good enough for anything we do + co_await aio.provider->getTimer().afterDelay(20 * kj::MILLISECONDS); } +} catch (...) { + co_return result::failure(std::current_exception()); } -Goals Worker::run(std::function<Goals (GoalFactory &)> req) +Worker::Results Worker::run(std::function<Targets (GoalFactory &)> req) { - auto _topGoals = req(goalFactory()); + auto topGoals = req(goalFactory()); assert(!running); running = true; Finally const _stop([&] { running = false; }); - updateStatistics(); + auto onInterrupt = kj::newPromiseAndCrossThreadFulfiller<Result<Results>>(); + auto interruptCallback = createInterruptCallback([&] { + onInterrupt.fulfiller->fulfill(result::failure(std::make_exception_ptr(makeInterrupted()))); + }); - topGoals = _topGoals; + auto promise = runImpl(std::move(topGoals)) + .exclusiveJoin(updateStatistics()) + .exclusiveJoin(std::move(onInterrupt.promise)); - debug("entered goal loop"); + // TODO GC interface? + if (auto localStore = dynamic_cast<LocalStore *>(&store); localStore && settings.minFree != 0u) { + // Periodically wake up to see if we need to run the garbage collector. + promise = promise.exclusiveJoin(boopGC(*localStore)); + } - while (1) { + return promise.wait(aio.waitScope).value(); +} - checkInterrupt(); +kj::Promise<Result<Worker::Results>> Worker::runImpl(Targets topGoals) +try { + debug("entered goal loop"); - // TODO GC interface? - if (auto localStore = dynamic_cast<LocalStore *>(&store)) - localStore->autoGC(false); + kj::Vector<Targets::value_type> promises(topGoals.size()); + for (auto & gp : topGoals) { + promises.add(std::move(gp)); + } - /* Call every wake goal (in the ordering established by - CompareGoalPtrs). */ - while (!awake.empty() && !topGoals.empty()) { - Goals awake2; - for (auto & i : awake) { - GoalPtr goal = i.lock(); - if (goal) awake2.insert(goal); - } - awake.clear(); - for (auto & goal : awake2) { - checkInterrupt(); - /* Make sure that we are always allowed to run at least one substitution. - This prevents infinite waiting. */ - const bool inSlot = goal->jobCategory() == JobCategory::Substitution - ? nrSubstitutions < std::max(1U, (unsigned int) settings.maxSubstitutionJobs) - : nrLocalBuilds < settings.maxBuildJobs; - handleWorkResult(goal, goal->work(inSlot).wait(aio.waitScope).value()); - updateStatistics(); - - if (topGoals.empty()) break; // stuff may have been cancelled - } - } + Results results; - if (topGoals.empty()) break; + auto collect = AsyncCollect(promises.releaseAsArray()); + while (auto done = co_await collect.next()) { + // propagate goal exceptions outward + BOOST_OUTCOME_CO_TRY(auto result, done->second); + results.emplace(done->first, result); - /* Wait for input. */ - if (!children.empty() || !waitingForAWhile.empty()) - waitForInput(); - else { - assert(!awake.empty()); + /* If a top-level goal failed, then kill all other goals + (unless keepGoing was set). */ + if (result.exitCode == Goal::ecFailed && !settings.keepGoing) { + children.clear(); + break; } } /* If --keep-going is not set, it's possible that the main goal exited while some of its subgoals were still active. But if --keep-going *is* set, then they must all be finished now. */ - assert(!settings.keepGoing || awake.empty()); - assert(!settings.keepGoing || wantingToBuild.empty()); - assert(!settings.keepGoing || children.empty()); + assert(!settings.keepGoing || children.isEmpty()); - return _topGoals; + co_return std::move(results); +} catch (...) { + co_return result::failure(std::current_exception()); } -void Worker::waitForInput() -{ - printMsg(lvlVomit, "waiting for children"); - - /* Process output from the file descriptors attached to the - children, namely log output and output path creation commands. - We also use this to detect child termination: if we get EOF on - the logger pipe of a build, we assume that the builder has - terminated. */ - - bool useTimeout = false; - long timeout = 0; - auto before = steady_time_point::clock::now(); - - /* If we're monitoring for silence on stdout/stderr, or if there - is a build timeout, then wait for input until the first - deadline for any child. */ - auto nearest = steady_time_point::max(); // nearest deadline - if (settings.minFree.get() != 0) - // Periodicallty wake up to see if we need to run the garbage collector. - nearest = before + std::chrono::seconds(10); - for (auto & i : children) { - if (auto goal = i.goal.lock()) { - if (!goal->respectsTimeouts()) continue; - if (0 != settings.maxSilentTime) - nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); - if (0 != settings.buildTimeout) - nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); - } - } - if (nearest != steady_time_point::max()) { - timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count()); - useTimeout = true; - } - - /* If we are polling goals that are waiting for a lock, then wake - up after a few seconds at most. */ - if (!waitingForAWhile.empty()) { - useTimeout = true; - if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; - timeout = std::max(1L, - (long) std::chrono::duration_cast<std::chrono::seconds>( - lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); - } else lastWokenUp = steady_time_point::min(); - - if (useTimeout) - vomit("sleeping %d seconds", timeout); - - /* Use select() to wait for the input side of any logger pipe to - become `available'. Note that `available' (i.e., non-blocking) - includes EOF. */ - std::vector<struct pollfd> pollStatus; - std::map<int, size_t> fdToPollStatus; - for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; - } - } - - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } - - auto after = steady_time_point::clock::now(); - - /* Process all available file descriptors. FIXME: this is - O(children * fds). */ - decltype(children)::iterator i; - for (auto j = children.begin(); j != children.end(); j = i) { - i = std::next(j); - - checkInterrupt(); - - GoalPtr goal = j->goal.lock(); - assert(goal); - - if (!goal->exitCode.has_value() && - 0 != settings.maxSilentTime && - goal->respectsTimeouts() && - after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) - { - handleWorkResult( - goal, - goal->timedOut(Error( - "%1% timed out after %2% seconds of silence", - goal->getName(), - settings.maxSilentTime - )) - ); - continue; - } - - else if (!goal->exitCode.has_value() && - 0 != settings.buildTimeout && - goal->respectsTimeouts() && - after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) - { - handleWorkResult( - goal, - goal->timedOut( - Error("%1% timed out after %2% seconds", goal->getName(), settings.buildTimeout) - ) - ); - continue; - } - - std::set<int> fds2(j->fds); - std::vector<unsigned char> buffer(4096); - for (auto & k : fds2) { - const auto fdPollStatusId = get(fdToPollStatus, k); - assert(fdPollStatusId); - assert(*fdPollStatusId < pollStatus.size()); - if (pollStatus.at(*fdPollStatusId).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - handleWorkResult(goal, Goal::ContinueImmediately{}); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - std::string_view data(charptr_cast<char *>(buffer.data()), rd); - j->lastOutput = after; - handleWorkResult(goal, goal->handleChildOutput(k, data)); - } - } - } - } - - if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { - lastWokenUp = after; - for (auto & i : waitingForAWhile) { - GoalPtr goal = i.lock(); - if (goal) wakeUp(goal); - } - waitingForAWhile.clear(); +kj::Promise<Result<Worker::Results>> Worker::boopGC(LocalStore & localStore) +try { + while (true) { + co_await aio.provider->getTimer().afterDelay(10 * kj::SECONDS); + localStore.autoGC(false); } +} catch (...) { + co_return result::failure(std::current_exception()); } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 6735ea0b9..1a913ca16 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include "async-semaphore.hh" +#include "concepts.hh" #include "notifying-counter.hh" #include "types.hh" #include "lock.hh" @@ -18,37 +20,22 @@ namespace nix { struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; +class LocalStore; typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point; -/** - * A mapping used to remember for each child process to what goal it - * belongs, and file descriptors for receiving log data and output - * path creation commands. - */ -struct Child -{ - WeakGoalPtr goal; - Goal * goal2; // ugly hackery - std::set<int> fds; - bool inBuildSlot; - /** - * Time we last got output on stdout/stderr - */ - steady_time_point lastOutput; - steady_time_point timeStarted; -}; - /* Forward definition. */ struct HookInstance; class GoalFactory { public: - virtual std::shared_ptr<DerivationGoal> makeDerivationGoal( + virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> + makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal ) = 0; - virtual std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( + virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> + makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -58,12 +45,14 @@ public: /** * @ref SubstitutionGoal "substitution goal" */ - virtual std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal( + virtual std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> + makePathSubstitutionGoal( const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt ) = 0; - virtual std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal( + virtual std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> + makeDrvOutputSubstitutionGoal( const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt @@ -75,7 +64,8 @@ public: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - virtual GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0; + virtual std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> + makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0; }; // elaborate hoax to let goals access factory methods while hiding them from the public @@ -94,61 +84,27 @@ protected: */ class Worker : public WorkerBase { +public: + using Targets = std::map<GoalPtr, kj::Promise<Result<Goal::WorkResult>>>; + using Results = std::map<GoalPtr, Goal::WorkResult>; + private: bool running = false; - /* Note: the worker should only have strong pointers to the - top-level goals. */ - - /** - * The top-level goals of the worker. - */ - Goals topGoals; - - /** - * Goals that are ready to do some work. - */ - WeakGoals awake; - - /** - * Goals waiting for a build slot. - */ - WeakGoals wantingToBuild; - - /** - * Child processes currently running. - */ - std::list<Child> children; - - /** - * Number of build slots occupied. This includes local builds but does not - * include substitutions or remote builds via the build hook. - */ - unsigned int nrLocalBuilds; - - /** - * Number of substitution slots occupied. - */ - unsigned int nrSubstitutions; - + template<typename G> + struct CachedGoal + { + std::shared_ptr<G> goal; + kj::ForkedPromise<Result<Goal::WorkResult>> promise{nullptr}; + }; /** * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ - std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals; - std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals; - std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals; - - /** - * Goals sleeping for a few seconds (polling a lock). - */ - WeakGoals waitingForAWhile; - - /** - * Last time the goals in `waitingForAWhile` where woken up. - */ - steady_time_point lastWokenUp; + std::map<StorePath, CachedGoal<DerivationGoal>> derivationGoals; + std::map<StorePath, CachedGoal<PathSubstitutionGoal>> substitutionGoals; + std::map<DrvOutput, CachedGoal<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals; /** * Cache for pathContentsGood(). @@ -176,60 +132,25 @@ private: */ bool checkMismatch = false; - void goalFinished(GoalPtr goal, Goal::Finished & f); - void handleWorkResult(GoalPtr goal, Goal::WorkResult how); - - /** - * Put `goal` to sleep until a build slot becomes available (which - * might be right away). - */ - void waitForBuildSlot(GoalPtr goal); - - /** - * Wait for a few seconds and then retry this goal. Used when - * waiting for a lock held by another process. This kind of - * polling is inefficient, but POSIX doesn't really provide a way - * to wait for multiple locks in the main select() loop. - */ - void waitForAWhile(GoalPtr goal); - - /** - * Wake up a goal (i.e., there is something for it to do). - */ - void wakeUp(GoalPtr goal); - - /** - * Wait for input to become available. - */ - void waitForInput(); - - /** - * Remove a dead goal. - */ - void removeGoal(GoalPtr goal); - - /** - * Registers a running child process. `inBuildSlot` means that - * the process counts towards the jobs limit. - */ - void childStarted(GoalPtr goal, const std::set<int> & fds, - bool inBuildSlot); - /** * Pass current stats counters to the logger for progress bar updates. */ - void updateStatistics(); + kj::Promise<Result<Results>> updateStatistics(); - bool statisticsOutdated = true; + AsyncSemaphore statisticsUpdateSignal{1}; + std::optional<AsyncSemaphore::Token> statisticsUpdateInhibitor; /** * Mark statistics as outdated, such that `updateStatistics` will be called. */ void updateStatisticsLater() { - statisticsOutdated = true; + statisticsUpdateInhibitor = {}; } + kj::Promise<Result<Results>> runImpl(Targets topGoals); + kj::Promise<Result<Results>> boopGC(LocalStore & localStore); + public: const Activity act; @@ -239,7 +160,12 @@ public: Store & store; Store & evalStore; kj::AsyncIoContext & aio; + AsyncSemaphore substitutions, localBuilds; +private: + kj::TaskSet children; + +public: struct HookState { std::unique_ptr<HookInstance> instance; @@ -277,21 +203,35 @@ public: * @ref DerivationGoal "derivation goal" */ private: - std::shared_ptr<DerivationGoal> makeDerivationGoalCommon( - const StorePath & drvPath, const OutputsSpec & wantedOutputs, - std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal); - std::shared_ptr<DerivationGoal> makeDerivationGoal( + template<typename ID, std::derived_from<Goal> G> + std::pair<std::shared_ptr<G>, kj::Promise<Result<Goal::WorkResult>>> makeGoalCommon( + std::map<ID, CachedGoal<G>> & map, + const ID & key, + InvocableR<std::unique_ptr<G>> auto create, + InvocableR<bool, G &> auto modify + ); + std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> makeDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; - std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( + std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> makeBasicDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override; /** * @ref SubstitutionGoal "substitution goal" */ - std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override; - std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt) override; + std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> + makePathSubstitutionGoal( + const StorePath & storePath, + RepairFlag repair = NoRepair, + std::optional<ContentAddress> ca = std::nullopt + ) override; + std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>> + makeDrvOutputSubstitutionGoal( + const DrvOutput & id, + RepairFlag repair = NoRepair, + std::optional<ContentAddress> ca = std::nullopt + ) override; /** * Make a goal corresponding to the `DerivedPath`. @@ -299,18 +239,14 @@ private: * It will be a `DerivationGoal` for a `DerivedPath::Built` or * a `SubstitutionGoal` for a `DerivedPath::Opaque`. */ - GoalPtr makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; + std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>> + makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override; public: /** - * Unregisters a running child process. - */ - void childTerminated(Goal * goal); - - /** * Loop until the specified top-level goals have finished. */ - Goals run(std::function<Goals (GoalFactory &)> req); + Results run(std::function<Targets (GoalFactory &)> req); /*** * The exit status in case of failure. diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index d201fb3ac..e20d14b90 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -6,7 +6,7 @@ namespace nix { // TODO: make pluggable. -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); +void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData, const std::string & caFileData); void builtinUnpackChannel(const BasicDerivation & drv); } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 3fb769fe6..b28eb01d0 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -7,7 +7,7 @@ namespace nix { -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) +void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData, const std::string & caFileData) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just @@ -17,6 +17,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) writeFile(settings.netrcFile, netrcData, 0600); } + settings.caFile = "ca-certificates.crt"; + writeFile(settings.caFile, caFileData, 0600); + auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); if (i == drv.env.end()) throw Error("attribute '%s' missing", name); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 10c810e49..34b92148e 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -6,6 +6,7 @@ #include "signals.hh" #include "compression.hh" #include "strings.hh" +#include <cstddef> #if ENABLE_S3 #include <aws/core/client/ClientConfiguration.h> @@ -115,7 +116,7 @@ struct curlFileTransfer : public FileTransfer if (!done) fail(FileTransferError(Interrupted, {}, "download of '%s' was interrupted", request.uri)); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -784,8 +785,10 @@ struct curlFileTransfer : public FileTransfer size_t read(char * data, size_t len) override { - auto readPartial = [this](char * data, size_t len) { + auto readPartial = [this](char * data, size_t len) -> size_t { const auto available = std::min(len, buffered.size()); + if (available == 0u) return 0u; + memcpy(data, buffered.data(), available); buffered.remove_prefix(available); return available; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index d5903d01e..99bf80994 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -923,8 +923,8 @@ void LocalStore::autoGC(bool sync) } catch (...) { // FIXME: we could propagate the exception to the - // future, but we don't really care. - ignoreException(); + // future, but we don't really care. (what??) + ignoreExceptionInDestructor(); } }).detach(); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index ffc2543ef..f43b759d2 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -443,7 +443,7 @@ static bool initLibStoreDone = false; void assertLibStoreInitialized() { if (!initLibStoreDone) { printError("The program must call nix::initNix() before calling any libstore library functions."); - abort(); + std::terminate(); }; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1af0f54de..c3248c2c3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -481,7 +481,7 @@ LocalStore::~LocalStore() unlink(fnTempRoots.c_str()); } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -1222,7 +1222,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, try { parseDump(sink, source); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } }; diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index d0897b81f..7314e3177 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -68,11 +68,11 @@ ref<Store> Machine::openStore() const { Store::Params storeParams; if (storeUri.starts_with("ssh://")) { + storeParams["log-fd"] = "4"; storeParams["max-connections"] = "1"; } if (storeUri.starts_with("ssh://") || storeUri.starts_with("ssh-ng://")) { - storeParams["log-fd"] = "4"; if (sshKey != "") storeParams["ssh-key"] = sshKey; if (sshPublicHostKey != "") diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 7600de6e7..f228004a9 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -20,10 +20,10 @@ struct NarMember file in the NAR. */ uint64_t start = 0, size = 0; - std::string target; + std::string target = {}; /* If this is a directory, all the children of the directory. */ - std::map<std::string, NarMember> children; + std::map<std::string, NarMember> children = {}; }; struct NarAccessor : public FSAccessor diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index c60e5a85d..14381b6e0 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -31,7 +31,7 @@ struct MakeReadOnly /* This will make the path read-only. */ if (path != "") canonicaliseTimestampAndPermissions(path); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } }; diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 57e03252d..8e2da1908 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -17,7 +17,7 @@ namespace nix { struct StorePathWithOutputs { StorePath path; - std::set<std::string> outputs; + std::set<std::string> outputs = {}; std::string to_string(const Store & store) const; diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index ced0f30bb..3225857ec 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -145,7 +145,7 @@ PathLocks::~PathLocks() try { unlock(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index d06d031b5..feb0f5548 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "error.hh" #include "file-descriptor.hh" namespace nix { @@ -53,7 +54,7 @@ struct FdLock if (acquired) lockFile(fd, ltNone, false); } catch (SysError &) { - ignoreException(); + ignoreExceptionInDestructor(); } } }; diff --git a/src/libstore/platform.cc b/src/libstore/platform.cc index f2c023c82..36f8e352a 100644 --- a/src/libstore/platform.cc +++ b/src/libstore/platform.cc @@ -25,7 +25,7 @@ std::shared_ptr<LocalStore> LocalStore::makeLocalStore(const Params & params) #endif } -std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal( +std::unique_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal( const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, @@ -34,17 +34,17 @@ std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoa ) { #if __linux__ - return std::make_shared<LinuxLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique<LinuxLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); #elif __APPLE__ - return std::make_shared<DarwinLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique<DarwinLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); #elif __FreeBSD__ - return std::make_shared<FreeBSDLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique<FreeBSDLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); #else - return std::make_shared<FallbackLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); + return std::make_unique<FallbackLocalDerivationGoal>(drvPath, wantedOutputs, worker, isDependency, buildMode); #endif } -std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal( +std::unique_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal( const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, @@ -54,19 +54,19 @@ std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoa ) { #if __linux__ - return std::make_shared<LinuxLocalDerivationGoal>( + return std::make_unique<LinuxLocalDerivationGoal>( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #elif __APPLE__ - return std::make_shared<DarwinLocalDerivationGoal>( + return std::make_unique<DarwinLocalDerivationGoal>( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #elif __FreeBSD__ - return std::make_shared<FreeBSDLocalDerivationGoal>( + return std::make_unique<FreeBSDLocalDerivationGoal>( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #else - return std::make_shared<FallbackLocalDerivationGoal>( + return std::make_unique<FallbackLocalDerivationGoal>( drvPath, drv, wantedOutputs, worker, isDependency, buildMode ); #endif diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index f2b228fa0..baeb7a2c9 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -50,7 +50,7 @@ struct Realisation { DrvOutput id; StorePath outPath; - StringSet signatures; + StringSet signatures = {}; /** * The realisations that are required for the current one to be valid. @@ -58,7 +58,7 @@ struct Realisation { * When importing this realisation, the store will first check that all its * dependencies exist, and map to the correct output path */ - std::map<DrvOutput, StorePath> dependentRealisations; + std::map<DrvOutput, StorePath> dependentRealisations = {}; nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 0689ce74d..59d267873 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -29,7 +29,7 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str /* FIXME: do this asynchronously. */ writeFile(makeCacheFile(hashPart, "nar"), nar); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } @@ -41,7 +41,7 @@ ref<FSAccessor> RemoteFSAccessor::addToCache(std::string_view hashPart, std::str nlohmann::json j = listNar(narAccessor, "", true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1f94ca03f..a9f9818be 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -1,3 +1,4 @@ +#include "error.hh" #include "serialise.hh" #include "signals.hh" #include "path-with-outputs.hh" @@ -855,7 +856,7 @@ RemoteStore::Connection::~Connection() try { to.flush(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -985,7 +986,7 @@ void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sin try { std::rethrow_exception(ex); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 8d0bfcb11..7aa0b6629 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -85,7 +85,7 @@ SQLite::~SQLite() if (db && sqlite3_close(db) != SQLITE_OK) SQLiteError::throw_(db, "closing database"); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -124,7 +124,7 @@ SQLiteStmt::~SQLiteStmt() if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) SQLiteError::throw_(db, "finalizing statement '%s'", sql); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -248,7 +248,7 @@ SQLiteTxn::~SQLiteTxn() if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) SQLiteError::throw_(db, "aborting transaction"); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index fb60326c1..5c1fc0c1f 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -30,11 +30,6 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { public: - // Hack for getting remote build log output. - // Intentionally not in `SSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting<int> logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; - SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) , RemoteStoreConfig(params) @@ -49,8 +44,7 @@ public: sshPublicHostKey, // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - compress, - logFD) + compress) { } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 50d392779..18f80eef8 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -829,7 +829,7 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m { size_t left; StorePathSet valid; - std::exception_ptr exc; + std::exception_ptr exc = {}; }; Sync<State> state_(State{paths.size(), StorePathSet()}); @@ -1163,7 +1163,7 @@ std::map<StorePath, StorePath> copyPaths( // not be within our control to change that, and we might still want // to at least copy the output paths. if (e.missingFeature == Xp::CaDerivations) - ignoreException(); + ignoreExceptionExceptInterrupt(); else throw; } diff --git a/src/libutil/async-collect.hh b/src/libutil/async-collect.hh new file mode 100644 index 000000000..9e0b8bad9 --- /dev/null +++ b/src/libutil/async-collect.hh @@ -0,0 +1,101 @@ +#pragma once +/// @file + +#include <kj/async.h> +#include <kj/common.h> +#include <kj/vector.h> +#include <list> +#include <optional> +#include <type_traits> + +namespace nix { + +template<typename K, typename V> +class AsyncCollect +{ +public: + using Item = std::conditional_t<std::is_void_v<V>, K, std::pair<K, V>>; + +private: + kj::ForkedPromise<void> allPromises; + std::list<Item> results; + size_t remaining; + + kj::ForkedPromise<void> signal; + kj::Maybe<kj::Own<kj::PromiseFulfiller<void>>> notify; + + void oneDone(Item item) + { + results.emplace_back(std::move(item)); + remaining -= 1; + KJ_IF_MAYBE (n, notify) { + (*n)->fulfill(); + notify = nullptr; + } + } + + kj::Promise<void> collectorFor(K key, kj::Promise<V> promise) + { + if constexpr (std::is_void_v<V>) { + return promise.then([this, key{std::move(key)}] { oneDone(std::move(key)); }); + } else { + return promise.then([this, key{std::move(key)}](V v) { + oneDone(Item{std::move(key), std::move(v)}); + }); + } + } + + kj::ForkedPromise<void> waitForAll(kj::Array<std::pair<K, kj::Promise<V>>> & promises) + { + kj::Vector<kj::Promise<void>> wrappers; + for (auto & [key, promise] : promises) { + wrappers.add(collectorFor(std::move(key), std::move(promise))); + } + + return kj::joinPromisesFailFast(wrappers.releaseAsArray()).fork(); + } + +public: + AsyncCollect(kj::Array<std::pair<K, kj::Promise<V>>> && promises) + : allPromises(waitForAll(promises)) + , remaining(promises.size()) + , signal{nullptr} + { + } + + kj::Promise<std::optional<Item>> next() + { + if (remaining == 0 && results.empty()) { + return {std::nullopt}; + } + + if (!results.empty()) { + auto result = std::move(results.front()); + results.pop_front(); + return {{std::move(result)}}; + } + + if (notify == nullptr) { + auto pair = kj::newPromiseAndFulfiller<void>(); + notify = std::move(pair.fulfiller); + signal = pair.promise.fork(); + } + + return signal.addBranch().exclusiveJoin(allPromises.addBranch()).then([this] { + return next(); + }); + } +}; + +/** + * Collect the results of a list of promises, in order of completion. + * Once any input promise is rejected all promises that have not been + * resolved or rejected will be cancelled and the exception rethrown. + */ +template<typename K, typename V> +AsyncCollect<K, V> asyncCollect(kj::Array<std::pair<K, kj::Promise<V>>> promises) +{ + return AsyncCollect<K, V>(std::move(promises)); +} + +} diff --git a/src/libutil/async-semaphore.hh b/src/libutil/async-semaphore.hh new file mode 100644 index 000000000..f8db31a68 --- /dev/null +++ b/src/libutil/async-semaphore.hh @@ -0,0 +1,122 @@ +#pragma once +/// @file +/// @brief A semaphore implementation usable from within a KJ event loop. + +#include <cassert> +#include <kj/async.h> +#include <kj/common.h> +#include <kj/exception.h> +#include <kj/list.h> +#include <kj/source-location.h> +#include <memory> +#include <optional> + +namespace nix { + +class AsyncSemaphore +{ +public: + class [[nodiscard("destroying a semaphore guard releases the semaphore immediately")]] Token + { + struct Release + { + void operator()(AsyncSemaphore * sem) const + { + sem->unsafeRelease(); + } + }; + + std::unique_ptr<AsyncSemaphore, Release> parent; + + public: + Token() = default; + Token(AsyncSemaphore & parent, kj::Badge<AsyncSemaphore>) : parent(&parent) {} + + bool valid() const + { + return parent != nullptr; + } + }; + +private: + struct Waiter + { + kj::PromiseFulfiller<Token> & fulfiller; + kj::ListLink<Waiter> link; + kj::List<Waiter, &Waiter::link> & list; + + Waiter(kj::PromiseFulfiller<Token> & fulfiller, kj::List<Waiter, &Waiter::link> & list) + : fulfiller(fulfiller) + , list(list) + { + list.add(*this); + } + + ~Waiter() + { + if (link.isLinked()) { + list.remove(*this); + } + } + }; + + const unsigned capacity_; + unsigned used_ = 0; + kj::List<Waiter, &Waiter::link> waiters; + + void unsafeRelease() + { + used_ -= 1; + while (used_ < capacity_ && !waiters.empty()) { + used_ += 1; + auto & w = waiters.front(); + w.fulfiller.fulfill(Token{*this, {}}); + waiters.remove(w); + } + } + +public: + explicit AsyncSemaphore(unsigned capacity) : capacity_(capacity) {} + + KJ_DISALLOW_COPY_AND_MOVE(AsyncSemaphore); + + ~AsyncSemaphore() + { + assert(waiters.empty() && "destroyed a semaphore with active waiters"); + } + + std::optional<Token> tryAcquire() + { + if (used_ < capacity_) { + used_ += 1; + return Token{*this, {}}; + } else { + return {}; + } + } + + kj::Promise<Token> acquire() + { + if (auto t = tryAcquire()) { + return std::move(*t); + } else { + return kj::newAdaptedPromise<Token, Waiter>(waiters); + } + } + + unsigned capacity() const + { + return capacity_; + } + + unsigned used() const + { + return used_; + } + + unsigned available() const + { + return capacity_ - used_; + } +}; +} diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 33cda211b..3b3e46a9a 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -49,7 +49,7 @@ unsigned int getMaxCPU() auto period = cpuMaxParts[1]; if (quota != "max") return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } + } catch (Error &) { ignoreExceptionInDestructor(lvlDebug); } #endif return 0; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index a7cbfbfd0..f57e3ef7d 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -4,6 +4,7 @@ #include "position.hh" #include "terminal.hh" #include "strings.hh" +#include "signals.hh" #include <iostream> #include <optional> @@ -132,7 +133,7 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR /** * A development aid for finding missing positions, to improve error messages. Example use: * - * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 just test --suite installcheck -v functional-lang * git diff -U20 tests * */ @@ -416,7 +417,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } -void ignoreException(Verbosity lvl) +void ignoreExceptionInDestructor(Verbosity lvl) { /* Make sure no exceptions leave this function. printError() also throws when remote is closed. */ @@ -429,4 +430,15 @@ void ignoreException(Verbosity lvl) } catch (...) { } } +void ignoreExceptionExceptInterrupt(Verbosity lvl) +{ + try { + throw; + } catch (const Interrupted & e) { + throw; + } catch (std::exception & e) { + printMsg(lvl, "error (ignored): %1%", e.what()); + } +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 73c1ccadd..885a2b218 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -70,17 +70,17 @@ inline bool operator<=(const Trace& lhs, const Trace& rhs); inline bool operator>=(const Trace& lhs, const Trace& rhs); struct ErrorInfo { - Verbosity level; + Verbosity level = Verbosity::lvlError; HintFmt msg; std::shared_ptr<Pos> pos; - std::list<Trace> traces; + std::list<Trace> traces = {}; /** * Exit status. */ unsigned int status = 1; - Suggestions suggestions; + Suggestions suggestions = {}; static std::optional<std::string> programName; }; @@ -204,7 +204,22 @@ public: /** * Exception handling in destructors: print an error message, then * ignore the exception. + * + * If you're not in a destructor, you usually want to use `ignoreExceptionExceptInterrupt()`. + * + * This function might also be used in callbacks whose caller may not handle exceptions, + * but ideally we propagate the exception using an exception_ptr in such cases. + * See e.g. `PackBuilderContext` + */ +void ignoreExceptionInDestructor(Verbosity lvl = lvlError); + +/** + * Not destructor-safe. + * Print an error message, then ignore the exception. + * If the exception is an `Interrupted` exception, rethrow it. + * + * This may be used in a few places where Interrupt can't happen, but that's ok. */ -void ignoreException(Verbosity lvl = lvlError); +void ignoreExceptionExceptInterrupt(Verbosity lvl = lvlError); } diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 8385ea402..cbb2bb539 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -146,7 +146,7 @@ AutoCloseFD::~AutoCloseFD() try { close(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 1d3eba58f..c4ffb1d0c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -522,7 +522,7 @@ AutoDelete::~AutoDelete() } } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index ee3e1e2e7..5feefdf90 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -136,11 +136,17 @@ inline std::string fmt(const char * s) template<typename... Args> inline std::string fmt(const std::string & fs, const Args &... args) -{ +try { boost::format f(fs); fmt_internal::setExceptions(f); (f % ... % args); return f.str(); +} catch (boost::io::format_error & fe) { + // I don't care who catches this, we do not put up with boost format errors + // Give me a stack trace and a core dump + std::cerr << "nix::fmt threw format error. Original format string: '"; + std::cerr << fs << "'; number of arguments: " << sizeof...(args) << "\n"; + std::terminate(); } /** @@ -174,15 +180,13 @@ public: std::cerr << "HintFmt received incorrect number of format args. Original format string: '"; std::cerr << format << "'; number of arguments: " << sizeof...(args) << "\n"; // And regardless of the coredump give me a damn stacktrace. - printStackTrace(); - abort(); + std::terminate(); } } catch (boost::io::format_error & ex) { // Same thing, but for anything that happens in the member initializers. std::cerr << "HintFmt received incorrect format string. Original format string: '"; std::cerr << format << "'; number of arguments: " << sizeof...(args) << "\n"; - printStackTrace(); - abort(); + std::terminate(); } HintFmt(const HintFmt & hf) : fmt(hf.fmt) {} diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 7d9482814..7609e6e39 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -352,7 +352,7 @@ Activity::~Activity() try { logger.stopActivity(id); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index a3f21de59..afca4e021 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -53,6 +53,8 @@ libutil_headers = files( 'archive.hh', 'args/root.hh', 'args.hh', + 'async-collect.hh', + 'async-semaphore.hh', 'backed-string-view.hh', 'box_ptr.hh', 'canon-path.hh', diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh index dc09a9ba4..dd6e2978e 100644 --- a/src/libutil/processes.hh +++ b/src/libutil/processes.hh @@ -78,11 +78,11 @@ struct RunOptions { Path program; bool searchPath = true; - Strings args; - std::optional<uid_t> uid; - std::optional<uid_t> gid; - std::optional<Path> chdir; - std::optional<std::map<std::string, std::string>> environment; + Strings args = {}; + std::optional<uid_t> uid = {}; + std::optional<uid_t> gid = {}; + std::optional<Path> chdir = {}; + std::optional<std::map<std::string, std::string>> environment = {}; bool captureStdout = false; bool mergeStderrToStdout = false; bool isInteractive = false; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index f509fedff..2f5a11a28 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -83,7 +83,7 @@ void BufferedSink::flush() FdSink::~FdSink() { - try { flush(); } catch (...) { ignoreException(); } + try { flush(); } catch (...) { ignoreExceptionInDestructor(); } } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 3a9685e0e..08ea9a135 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -549,7 +549,7 @@ struct FramedSource : Source } } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -595,7 +595,7 @@ struct FramedSink : nix::BufferedSink to << 0; to.flush(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc index 04a697d01..dac2964ae 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/signals.cc @@ -12,13 +12,18 @@ std::atomic<bool> _isInterrupted = false; thread_local std::function<bool()> interruptCheck; +Interrupted makeInterrupted() +{ + return Interrupted("interrupted by the user"); +} + void _interrupted() { /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled kills the program! */ if (!std::uncaught_exceptions()) { - throw Interrupted("interrupted by the user"); + throw makeInterrupted(); } } @@ -78,7 +83,7 @@ void triggerInterrupt() try { callback(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } } diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh index 02f8d2ca3..538ff94b4 100644 --- a/src/libutil/signals.hh +++ b/src/libutil/signals.hh @@ -16,10 +16,13 @@ namespace nix { /* User interruption. */ +class Interrupted; + extern std::atomic<bool> _isInterrupted; extern thread_local std::function<bool()> interruptCheck; +Interrupted makeInterrupted(); void _interrupted(); void inline checkInterrupt() diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 0ff83e997..1c4488373 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -109,9 +109,21 @@ void ThreadPool::doWork(bool mainThread) try { std::rethrow_exception(exc); } catch (std::exception & e) { - if (!dynamic_cast<Interrupted*>(&e) && - !dynamic_cast<ThreadPoolShutDown*>(&e)) - ignoreException(); + if (!dynamic_cast<ThreadPoolShutDown*>(&e)) { + // Yes, this is not a destructor, but we cannot + // safely propagate an exception out of here. + // + // What happens is that if we do, shutdown() + // will have join() throw an exception if we + // are on a worker thread, preventing us from + // joining the rest of the threads. Although we + // could make the joining eat exceptions too, + // we could just as well not let Interrupted + // fall out to begin with, since the thread + // will immediately cleanly quit because of + // quit == true anyway. + ignoreExceptionInDestructor(); + } } catch (...) { } } diff --git a/src/meson.build b/src/meson.build index 66fbb13ba..8b63ef995 100644 --- a/src/meson.build +++ b/src/meson.build @@ -26,54 +26,9 @@ libasanoptions = declare_dependency( link_whole: asanoptions ) -build_remote_sources = files( - 'build-remote/build-remote.cc', -) -nix_build_sources = files( - 'nix-build/nix-build.cc', -) -nix_channel_sources = files( - 'nix-channel/nix-channel.cc', -) -unpack_channel_gen = gen_header.process('nix-channel/unpack-channel.nix') -nix_collect_garbage_sources = files( - 'nix-collect-garbage/nix-collect-garbage.cc', -) -nix_copy_closure_sources = files( - 'nix-copy-closure/nix-copy-closure.cc', -) -nix_env_buildenv_gen = gen_header.process('nix-env/buildenv.nix') -nix_env_sources = files( - 'nix-env/nix-env.cc', - 'nix-env/user-env.cc', -) -nix_instantiate_sources = files( - 'nix-instantiate/nix-instantiate.cc', -) -nix_store_sources = files( - 'nix-store/dotgraph.cc', - 'nix-store/graphml.cc', - 'nix-store/nix-store.cc', -) - -# Hurray for Meson list flattening! -nix2_commands_sources = [ - build_remote_sources, - nix_build_sources, - nix_channel_sources, - unpack_channel_gen, - nix_collect_garbage_sources, - nix_copy_closure_sources, - nix_env_buildenv_gen, - nix_env_sources, - nix_instantiate_sources, - nix_store_sources, -] +# Legacy commands. +subdir('legacy') # Finally, the nix command itself, which all of the other commands are implmented in terms of # as a multicall binary. subdir('nix') - -# Just copies nix-channel/unpack-channel.nix to the build directory. -# Done as a subdir to get Meson to respect the path hierarchy. -subdir('nix-channel') diff --git a/src/nix-channel/meson.build b/src/nix-channel/meson.build deleted file mode 100644 index 952dfdb78..000000000 --- a/src/nix-channel/meson.build +++ /dev/null @@ -1,5 +0,0 @@ -configure_file( - input : 'unpack-channel.nix', - output : 'unpack-channel.nix', - copy : true, -) diff --git a/src/nix/daemon-command.hh b/src/nix/daemon-command.hh new file mode 100644 index 000000000..454af88e2 --- /dev/null +++ b/src/nix/daemon-command.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixDaemon(); + +} diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index ca65c38e6..e1d183d7b 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -14,6 +14,7 @@ #include "signals.hh" #include "daemon.hh" #include "unix-domain-socket.hh" +#include "daemon-command.hh" #include <algorithm> #include <climits> @@ -36,7 +37,8 @@ #include <sys/ucred.h> #endif -using namespace nix; +namespace nix { + using namespace nix::daemon; /** @@ -496,7 +498,9 @@ static int main_nix_daemon(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); +void registerNixDaemon() { + LegacyCommands::add("nix-daemon", main_nix_daemon); +} struct CmdDaemon : StoreCommand { @@ -560,3 +564,5 @@ struct CmdDaemon : StoreCommand }; static auto rCmdDaemon = registerCommand2<CmdDaemon>({"daemon"}); + +} diff --git a/src/nix/develop.cc b/src/nix/develop.cc index fb144c904..d1615ecdc 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -639,7 +639,7 @@ struct CmdDevelop : Common, MixEnvironment throw Error("package 'nixpkgs#bashInteractive' does not provide a 'bin/bash'"); } catch (Error &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } // Override SHELL with the one chosen for this environment. diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 15c393c90..0c704a995 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -16,6 +16,7 @@ #include "eval-cache.hh" #include "markdown.hh" #include "terminal.hh" +#include "signals.hh" #include <limits> #include <nlohmann/json.hpp> @@ -367,9 +368,11 @@ struct CmdFlakeCheck : FlakeCommand auto reportError = [&](const Error & e) { try { throw e; + } catch (Interrupted & e) { + throw; } catch (Error & e) { if (settings.keepGoing) { - ignoreException(); + ignoreExceptionExceptInterrupt(); hasErrors = true; } else diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 059904150..f47f2204a 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -39,14 +39,8 @@ struct CmdFmt : SourceExprCommand { Strings programArgs{app.program}; // Propagate arguments from the CLI - if (args.empty()) { - // Format the current flake out of the box - programArgs.push_back("."); - } else { - // User wants more power, let them decide which paths to include/exclude - for (auto &i : args) { - programArgs.push_back(i); - } + for (auto &i : args) { + programArgs.push_back(i); } runProgramInStore(store, UseSearchPath::DontUse, app.program, programArgs); diff --git a/src/nix/hash-command.hh b/src/nix/hash-command.hh new file mode 100644 index 000000000..5383171a5 --- /dev/null +++ b/src/nix/hash-command.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixHash(); + +} diff --git a/src/nix/hash.cc b/src/nix/hash.cc index f6add527a..40b00c978 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -5,8 +5,9 @@ #include "shared.hh" #include "references.hh" #include "archive.hh" +#include "hash-command.hh" -using namespace nix; +namespace nix { struct CmdHashBase : Command { @@ -221,4 +222,8 @@ static int compatNixHash(int argc, char * * argv) return 0; } -static RegisterLegacyCommand r_nix_hash("nix-hash", compatNixHash); +void registerNixHash() { + LegacyCommands::add("nix-hash", compatNixHash); +} + +} diff --git a/src/nix/main.cc b/src/nix/main.cc index 4a3a7b4e7..fdd3ac2ae 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -15,6 +15,17 @@ #include "markdown.hh" #include "experimental-features-json.hh" #include "deprecated-features-json.hh" +#include "build-remote.hh" +#include "daemon-command.hh" +#include "hash-command.hh" +#include "nix-build.hh" +#include "nix-channel.hh" +#include "nix-collect-garbage.hh" +#include "nix-copy-closure.hh" +#include "nix-env.hh" +#include "nix-instantiate.hh" +#include "nix-store.hh" +#include "prefetch-command.hh" #include <sys/types.h> #include <sys/socket.h> @@ -30,6 +41,21 @@ void chrootHelper(int argc, char * * argv); namespace nix { +void registerLegacyCommands() +{ + registerNixEnv(); + registerNixBuildAndNixShell(); + registerNixInstantiate(); + registerNixCopyClosure(); + registerNixCollectGarbage(); + registerNixChannel(); + registerNixStore(); + registerBuildRemote(); + registerNixDaemon(); + registerNixPrefetchUrl(); + registerNixHash(); +} + static bool haveProxyEnvironmentVariables() { static const std::vector<std::string> proxyVariables = { @@ -356,8 +382,10 @@ void mainWrapped(int argc, char * * argv) // Clean up the progress bar if shown using --log-format in a legacy command too. // Otherwise, this is a harmless no-op. Finally f([] { logger->pause(); }); + { - auto legacy = (*RegisterLegacyCommand::commands)[programName]; + registerLegacyCommands(); + auto legacy = (*LegacyCommands::commands)[programName]; if (legacy) return legacy(argc, argv); } diff --git a/src/nix/meson.build b/src/nix/meson.build index 80223a390..cabdf0d2c 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -71,11 +71,21 @@ nix_sources = files( 'why-depends.cc', ) +nix_headers = files( + 'daemon-command.hh', + 'hash-command.hh', + 'prefetch-command.hh', +) + nix = executable( 'nix', nix_sources, + legacy_sources, nix_generated_headers, - nix2_commands_sources, + nix_headers, + legacy_headers, + legacy_generated_headers, + include_directories : legacy_include_directories, dependencies : [ libasanoptions, liblixcmd, diff --git a/src/nix/prefetch-command.hh b/src/nix/prefetch-command.hh new file mode 100644 index 000000000..078e83485 --- /dev/null +++ b/src/nix/prefetch-command.hh @@ -0,0 +1,8 @@ +#pragma once +/// @file + +namespace nix { + +void registerNixPrefetchUrl(); + +} diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b99cd5dd0..0183b0008 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -9,10 +9,11 @@ #include "eval-inline.hh" // IWYU pragma: keep #include "legacy.hh" #include "terminal.hh" +#include "prefetch-command.hh" #include <nlohmann/json.hpp> -using namespace nix; +namespace nix { /* If ‘url’ starts with ‘mirror://’, then resolve it using the list of mirrors defined in Nixpkgs. */ @@ -248,7 +249,9 @@ static int main_nix_prefetch_url(int argc, char * * argv) } } -static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); +void registerNixPrefetchUrl() { + LegacyCommands::add("nix-prefetch-url", main_nix_prefetch_url); +} struct CmdStorePrefetchFile : StoreCommand, MixJSON { @@ -328,3 +331,5 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON }; static auto rCmdStorePrefetchFile = registerCommand2<CmdStorePrefetchFile>({"store", "prefetch-file"}); + +} diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 401d5bd77..6739cb5c6 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -8,7 +8,7 @@ #include "archive.hh" #include "builtins/buildenv.hh" #include "flake/flakeref.hh" -#include "../nix-env/user-env.hh" +#include "user-env.hh" #include "profiles.hh" #include "names.hh" |