#include "attr-set.hh" #include "error.hh" #include "eval-settings.hh" #include "eval.hh" #include "finally.hh" #include "nixexpr.hh" #include "symbol-table.hh" #include "users.hh" #include "change_head.hh" #include "grammar.hh" #include "state.hh" #include #include // flip this define when doing parser development to enable some g checks. #if 0 #include #define ANALYZE_GRAMMAR \ ([] { \ const std::size_t issues = tao::pegtl::analyze(); \ assert(issues == 0); \ })() #else #define ANALYZE_GRAMMAR ((void) 0) #endif namespace p = tao::pegtl; namespace nix::parser { namespace { template inline constexpr const char * error_message = nullptr; #define error_message_for(...) \ template<> inline constexpr auto error_message<__VA_ARGS__> error_message_for(p::one<'{'>) = "expecting '{'"; error_message_for(p::one<'}'>) = "expecting '}'"; error_message_for(p::one<'"'>) = "expecting '\"'"; error_message_for(p::one<';'>) = "expecting ';'"; error_message_for(p::one<')'>) = "expecting ')'"; error_message_for(p::one<']'>) = "expecting ']'"; error_message_for(p::one<':'>) = "expecting ':'"; error_message_for(p::string<'\'', '\''>) = "expecting \"''\""; error_message_for(p::any) = "expecting any character"; error_message_for(grammar::eof) = "expecting end of file"; error_message_for(grammar::seps) = "expecting separators"; error_message_for(grammar::path::forbid_prefix_triple_slash) = "too many slashes in path"; error_message_for(grammar::path::forbid_prefix_double_slash_no_interp) = "path has a trailing slash"; error_message_for(grammar::expr) = "expecting expression"; error_message_for(grammar::expr::unary) = "expecting expression"; error_message_for(grammar::binding::equal) = "expecting '='"; error_message_for(grammar::expr::lambda::arg) = "expecting identifier"; error_message_for(grammar::formals) = "expecting formals"; error_message_for(grammar::attrpath) = "expecting attribute path"; error_message_for(grammar::expr::select) = "expecting selection expression"; error_message_for(grammar::t::kw_then) = "expecting 'then'"; error_message_for(grammar::t::kw_else) = "expecting 'else'"; error_message_for(grammar::t::kw_in) = "expecting 'in'"; struct SyntaxErrors { template static constexpr auto message = error_message; template static constexpr bool raise_on_failure = false; }; template struct Control : p::must_if::control { template [[noreturn]] static void raise(const ParseInput & in, States &&... st) { if (in.empty()) { std::string expected; if constexpr (constexpr auto msg = error_message) expected = fmt(", %s", msg); throw p::parse_error("unexpected end of file" + expected, in); } p::must_if::control::raise(in, st...); } }; struct ExprState : grammar:: operator_semantics>> { std::unique_ptr popExprOnly() { return std::move(popExpr().second); } template std::unique_ptr applyUnary(Args &&... args) { return std::make_unique(popExprOnly(), std::forward(args)...); } template std::unique_ptr applyBinary(PosIdx pos) { auto right = popExprOnly(), left = popExprOnly(); return std::make_unique(pos, std::move(left), std::move(right)); } std::unique_ptr call(PosIdx pos, Symbol fn, bool flip = false) { std::vector> args(2); args[flip ? 0 : 1] = popExprOnly(); args[flip ? 1 : 0] = popExprOnly(); return std::make_unique(pos, std::make_unique(fn), std::move(args)); } std::unique_ptr order(PosIdx pos, bool less, State & state) { return call(pos, state.s.lessThan, !less); } std::unique_ptr concatStrings(PosIdx pos) { std::vector>> args(2); args[1] = popExpr(); args[0] = popExpr(); return std::make_unique(pos, false, std::move(args)); } std::unique_ptr negate(PosIdx pos, State & state) { std::vector> args(2); args[0] = std::make_unique(0); args[1] = popExprOnly(); return std::make_unique(pos, std::make_unique(state.s.sub), std::move(args)); } std::pair> applyOp(PosIdx pos, auto & op, State & state) { using Op = grammar::op; auto not_ = [] (auto e) { return std::make_unique(std::move(e)); }; return { pos, (overloaded { [&] (Op::implies) { return applyBinary(pos); }, [&] (Op::or_) { return applyBinary(pos); }, [&] (Op::and_) { return applyBinary(pos); }, [&] (Op::equals) { return applyBinary(pos); }, [&] (Op::not_equals) { return applyBinary(pos); }, [&] (Op::less) { return order(pos, true, state); }, [&] (Op::greater_eq) { return not_(order(pos, true, state)); }, [&] (Op::greater) { return order(pos, false, state); }, [&] (Op::less_eq) { return not_(order(pos, false, state)); }, [&] (Op::update) { return applyBinary(pos); }, [&] (Op::not_) { return applyUnary(); }, [&] (Op::plus) { return concatStrings(pos); }, [&] (Op::minus) { return call(pos, state.s.sub); }, [&] (Op::mul) { return call(pos, state.s.mul); }, [&] (Op::div) { return call(pos, state.s.div); }, [&] (Op::concat) { return applyBinary(pos); }, [&] (has_attr & a) { return applyUnary(std::move(a.path)); }, [&] (Op::unary_minus) { return negate(pos, state); }, })(op) }; } // always_inline is needed, otherwise pushOp slows down considerably [[noreturn, gnu::always_inline]] static void badOperator(PosIdx pos, State & state) { throw ParseError({ .msg = HintFmt("syntax error, unexpected operator"), .pos = state.positions[pos] }); } template Expr & pushExpr(PosIdx pos, Args && ... args) { auto p = std::make_unique(std::forward(args)...); auto & result = *p; exprs.emplace_back(pos, std::move(p)); return result; } }; struct SubexprState { private: ExprState * up; public: explicit SubexprState(ExprState & up, auto &...) : up(&up) {} operator ExprState &() { return *up; } ExprState * operator->() { return up; } }; template struct BuildAST : grammar::nothing {}; struct LambdaState : SubexprState { using SubexprState::SubexprState; Symbol arg; std::unique_ptr formals; }; struct FormalsState : SubexprState { using SubexprState::SubexprState; Formals formals{}; Formal formal{}; }; template<> struct BuildAST { static void apply(const auto & in, FormalsState & s, State & ps) { s.formal = { .pos = ps.at(in), .name = ps.symbols.create(in.string_view()), }; } }; template<> struct BuildAST { static void apply0(FormalsState & s, State &) { s.formals.formals.emplace_back(std::move(s.formal)); } }; template<> struct BuildAST { static void apply0(FormalsState & s, State & ps) { s.formal.def = s->popExprOnly(); } }; template<> struct BuildAST { static void apply0(FormalsState & s, State &) { s.formals.ellipsis = true; } }; template<> struct BuildAST : change_head { static void success0(FormalsState & f, LambdaState & s, State &) { s.formals = std::make_unique(std::move(f.formals)); } }; struct AttrState : SubexprState { using SubexprState::SubexprState; std::vector attrs; template void pushAttr(T && attr, PosIdx) { attrs.emplace_back(std::forward(attr)); } }; template<> struct BuildAST { static void apply(const auto & in, auto & s, State & ps) { s.pushAttr(ps.symbols.create(in.string_view()), ps.at(in)); } }; template<> struct BuildAST { static void apply(const auto & in, auto & s, State & ps) { auto e = s->popExprOnly(); if (auto str = dynamic_cast(e.get())) s.pushAttr(ps.symbols.create(str->s), ps.at(in)); else s.pushAttr(std::move(e), ps.at(in)); } }; template<> struct BuildAST : BuildAST {}; struct BindingsState : SubexprState { using SubexprState::SubexprState; ExprAttrs attrs; AttrPath path; std::unique_ptr value; }; struct InheritState : SubexprState { using SubexprState::SubexprState; std::vector> attrs; std::unique_ptr from; PosIdx fromPos; template void pushAttr(T && attr, PosIdx pos) { attrs.emplace_back(std::forward(attr), pos); } }; template<> struct BuildAST { static void apply(const auto & in, InheritState & s, State & ps) { s.from = s->popExprOnly(); s.fromPos = ps.at(in); } }; template<> struct BuildAST : change_head { static void success0(InheritState & s, BindingsState & b, State & ps) { auto & attrs = b.attrs.attrs; // TODO this should not reuse generic attrpath rules. for (auto & [i, iPos] : s.attrs) { if (i.symbol) continue; if (auto str = dynamic_cast(i.expr.get())) i = AttrName(ps.symbols.create(str->s)); else { throw ParseError({ .msg = HintFmt("dynamic attributes not allowed in inherit"), .pos = ps.positions[iPos] }); } } if (s.from != nullptr) { if (!b.attrs.inheritFromExprs) b.attrs.inheritFromExprs = std::make_unique>>(); auto fromExpr = ref(std::move(s.from)); b.attrs.inheritFromExprs->push_back(fromExpr); for (auto & [i, iPos] : s.attrs) { if (attrs.find(i.symbol) != attrs.end()) ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); auto inheritFrom = std::make_unique( s.fromPos, b.attrs.inheritFromExprs->size() - 1, fromExpr ); attrs.emplace( i.symbol, ExprAttrs::AttrDef( std::make_unique(iPos, std::move(inheritFrom), i.symbol), iPos, ExprAttrs::AttrDef::Kind::InheritedFrom)); } } else { for (auto & [i, iPos] : s.attrs) { if (attrs.find(i.symbol) != attrs.end()) ps.dupAttr(i.symbol, iPos, attrs[i.symbol].pos); attrs.emplace( i.symbol, ExprAttrs::AttrDef( std::make_unique(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); } } } }; template<> struct BuildAST : change_head { static void success0(AttrState & a, BindingsState & s, State & ps) { s.path = std::move(a.attrs); } }; template<> struct BuildAST { static void apply0(BindingsState & s, State & ps) { s.value = s->popExprOnly(); } }; template<> struct BuildAST { static void apply(const auto & in, BindingsState & s, State & ps) { ps.addAttr(&s.attrs, std::move(s.path), std::move(s.value), ps.at(in)); } }; template<> struct BuildAST { static void apply(const auto & in, ExprState & s, State & ps) { if (in.string_view() == "__curPos") s.pushExpr(ps.at(in), ps.at(in)); else s.pushExpr(ps.at(in), ps.at(in), ps.symbols.create(in.string_view())); } }; template<> struct BuildAST { static void apply(const auto & in, ExprState & s, State & ps) { int64_t v; if (std::from_chars(in.begin(), in.end(), v).ec != std::errc{}) { throw ParseError({ .msg = HintFmt("invalid integer '%1%'", in.string_view()), .pos = ps.positions[ps.at(in)], }); } s.pushExpr(noPos, v); } }; template<> struct BuildAST { static void apply(const auto & in, ExprState & s, State & ps) { // copy the input into a temporary string so we can call stod. // can't use from_chars because libc++ (thus darwin) does not have it, // and floats are not performance-sensitive anyway. if they were you'd // be in much bigger trouble than this. // // we also get to do a locale-save dance because stod is locale-aware and // something (a plugin?) may have called setlocale or uselocale. static struct locale_hack { locale_t posix; locale_hack(): posix(newlocale(LC_ALL_MASK, "POSIX", 0)) { if (posix == 0) throw SysError("could not get POSIX locale"); } } locale; auto tmp = in.string(); double v = [&] { auto oldLocale = uselocale(locale.posix); Finally resetLocale([=] { uselocale(oldLocale); }); try { return std::stod(tmp); } catch (...) { throw ParseError({ .msg = HintFmt("invalid float '%1%'", in.string_view()), .pos = ps.positions[ps.at(in)], }); } }(); s.pushExpr(noPos, v); } }; struct StringState : SubexprState { using SubexprState::SubexprState; std::string currentLiteral; PosIdx currentPos; std::vector>> parts; void append(PosIdx pos, std::string_view s) { if (currentLiteral.empty()) currentPos = pos; currentLiteral += s; } // FIXME this truncates strings on NUL for compat with the old parser. ideally // we should use the decomposition the g gives us instead of iterating over // the entire string again. static void unescapeStr(std::string & str) { char * s = str.data(); char * t = s; char c; while ((c = *s++)) { if (c == '\\') { c = *s++; if (c == 'n') *t = '\n'; else if (c == 'r') *t = '\r'; else if (c == 't') *t = '\t'; else *t = c; } else if (c == '\r') { /* Normalise CR and CR/LF into LF. */ *t = '\n'; if (*s == '\n') s++; /* cr/lf */ } else *t = c; t++; } str.resize(t - str.data()); } void endLiteral() { if (!currentLiteral.empty()) { unescapeStr(currentLiteral); parts.emplace_back(currentPos, std::make_unique(std::move(currentLiteral))); } } std::unique_ptr finish() { if (parts.empty()) { unescapeStr(currentLiteral); return std::make_unique(std::move(currentLiteral)); } else { endLiteral(); auto pos = parts[0].first; return std::make_unique(pos, true, std::move(parts)); } } }; template struct BuildAST> { static void apply(const auto & in, StringState & s, State & ps) { s.append(ps.at(in), in.string_view()); } }; template<> struct BuildAST { static void apply(const auto & in, StringState & s, State & ps) { s.append(ps.at(in), in.string_view()); // FIXME compat with old parser } }; template<> struct BuildAST { static void apply(const auto & in, StringState & s, State & ps) { s.endLiteral(); s.parts.emplace_back(ps.at(in), s->popExprOnly()); } }; template<> struct BuildAST { static void apply(const auto & in, StringState & s, State & ps) { s.append(ps.at(in), "\\"); // FIXME compat with old parser s.append(ps.at(in), in.string_view()); } }; template<> struct BuildAST : change_head { static void success0(StringState & s, ExprState & e, State &) { e.exprs.emplace_back(noPos, s.finish()); } }; struct IndStringState : SubexprState { using SubexprState::SubexprState; std::vector, StringToken>>> parts; }; template struct BuildAST> { static void apply(const auto & in, IndStringState & s, State & ps) { s.parts.emplace_back(ps.at(in), StringToken{in.string_view(), Indented}); } }; template<> struct BuildAST { static void apply(const auto & in, IndStringState & s, State & ps) { s.parts.emplace_back(ps.at(in), s->popExprOnly()); } }; template<> struct BuildAST { static void apply(const auto & in, IndStringState & s, State & ps) { switch (*in.begin()) { case 'n': s.parts.emplace_back(ps.at(in), StringToken{"\n"}); break; case 'r': s.parts.emplace_back(ps.at(in), StringToken{"\r"}); break; case 't': s.parts.emplace_back(ps.at(in), StringToken{"\t"}); break; default: s.parts.emplace_back(ps.at(in), StringToken{in.string_view()}); break; } } }; template<> struct BuildAST : change_head { static void success(const auto & in, IndStringState & s, ExprState & e, State & ps) { e.exprs.emplace_back(noPos, ps.stripIndentation(ps.at(in), std::move(s.parts))); } }; template struct BuildAST> { static void apply(const auto & in, StringState & s, State & ps) { s.append(ps.at(in), in.string_view()); s.endLiteral(); } }; template<> struct BuildAST : BuildAST {}; template<> struct BuildAST { static void apply(const auto & in, StringState & s, State & ps) { Path path(absPath(in.string(), ps.basePath.path.abs())); /* add back in the trailing '/' to the first segment */ if (in.string_view().ends_with('/') && in.size() > 1) path += "/"; s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); } }; template<> struct BuildAST { static void apply(const auto & in, StringState & s, State & ps) { if (evalSettings.pureEval) throw Error("the path '%s' can not be resolved in pure mode", in.string_view()); Path path(getHome() + in.string_view().substr(1)); s.parts.emplace_back(ps.at(in), new ExprPath(std::move(path))); } }; template<> struct BuildAST { static void apply(const auto & in, StringState & s, State & ps) { std::vector> args{2}; args[0] = std::make_unique(ps.s.nixPath); args[1] = std::make_unique(in.string()); s.parts.emplace_back( ps.at(in), std::make_unique( ps.at(in), std::make_unique(ps.s.findFile), std::move(args))); } }; template<> struct BuildAST : change_head { template static void check_slash(PosIdx end, StringState & s, State & ps) { auto e = dynamic_cast(s.parts.back().second.get()); if (!e || !e->s.ends_with('/')) return; if (s.parts.size() > 1 || e->s != "/") throw ParseError({ .msg = HintFmt("path has a trailing slash"), .pos = ps.positions[end], }); } static void success(const auto & in, StringState & s, ExprState & e, State & ps) { s.endLiteral(); check_slash(ps.atEnd(in), s, ps); check_slash(ps.atEnd(in), s, ps); if (s.parts.size() == 1) { e.exprs.emplace_back(noPos, std::move(s.parts.back().second)); } else { e.pushExpr(ps.at(in), ps.at(in), false, std::move(s.parts)); } } }; // strings and paths sare handled fully by the grammar-level rule for now template<> struct BuildAST : p::maybe_nothing {}; template<> struct BuildAST : p::maybe_nothing {}; template<> struct BuildAST : p::maybe_nothing {}; template<> struct BuildAST { static void apply(const auto & in, ExprState & s, State & ps) { static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); if (noURLLiterals) throw ParseError({ .msg = HintFmt("URL literals are disabled"), .pos = ps.positions[ps.at(in)] }); s.pushExpr(ps.at(in), in.string()); } }; template<> struct BuildAST : change_head { static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { b.attrs.pos = ps.at(in); b.attrs.recursive = true; s.pushExpr(b.attrs.pos, b.attrs.pos, std::make_unique(std::move(b.attrs)), ps.s.body); } }; template<> struct BuildAST : change_head { static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { b.attrs.pos = ps.at(in); b.attrs.recursive = true; s.pushExpr(b.attrs.pos, std::move(b.attrs)); } }; template<> struct BuildAST : change_head { static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { b.attrs.pos = ps.at(in); s.pushExpr(b.attrs.pos, std::move(b.attrs)); } }; using ListState = std::vector>; template<> struct BuildAST : change_head { static void success(const auto & in, ListState & ls, ExprState & s, State & ps) { auto e = std::make_unique(); e->elems = std::move(ls); s.exprs.emplace_back(ps.at(in), std::move(e)); } }; template<> struct BuildAST : change_head { static void success0(ExprState & e, ListState & s, State & ps) { s.emplace_back(e.finish(ps).second); } }; struct SelectState : SubexprState { using SubexprState::SubexprState; PosIdx pos; ExprSelect * e = nullptr; }; template<> struct BuildAST { static void apply(const auto & in, SelectState & s, State & ps) { s.pos = ps.at(in); } }; template<> struct BuildAST : change_head { static void success0(AttrState & a, SelectState & s, State &) { s.e = &s->pushExpr(s.pos, s.pos, s->popExprOnly(), std::move(a.attrs), nullptr); } }; template<> struct BuildAST { static void apply0(SelectState & s, State &) { s.e->def = s->popExprOnly(); } }; template<> struct BuildAST { static void apply(const auto & in, SelectState & s, State & ps) { std::vector> args(1); args[0] = std::make_unique(ps.at(in), ps.s.or_); s->pushExpr(s.pos, s.pos, s->popExprOnly(), std::move(args)); } }; template<> struct BuildAST : change_head { static void success0(const auto &...) {} }; struct AppState : SubexprState { using SubexprState::SubexprState; PosIdx pos; ExprCall * e = nullptr; }; template<> struct BuildAST { static void apply(const auto & in, AppState & s, State & ps) { s.pos = ps.at(in); } }; template<> struct BuildAST { static void apply(auto & in, AppState & s, State & ps) { auto arg = s->popExprOnly(), fn = s->popExprOnly(); if ((s.e = dynamic_cast(fn.get()))) { // TODO remove. // AST compat with old parser, semantics are the same. // this can happen on occasions such as `

` or `a or b or`, // neither of which are super worth optimizing. s.e->args.push_back(std::move(arg)); s->exprs.emplace_back(noPos, std::move(fn)); } else { std::vector> args{1}; args[0] = std::move(arg); s.e = &s->pushExpr(s.pos, s.pos, std::move(fn), std::move(args)); } } }; template<> struct BuildAST { static void apply0(AppState & s, State & ps) { s.e->args.push_back(s->popExprOnly()); } }; template<> struct BuildAST : change_head { static void success0(const auto &...) {} }; template struct BuildAST> { static void apply(const auto & in, ExprState & s, State & ps) { s.pushOp(ps.at(in), Op{}, ps); } }; template<> struct BuildAST> : change_head { static void success(const auto & in, AttrState & a, ExprState & s, State & ps) { s.pushOp(ps.at(in), ExprState::has_attr{{}, std::move(a.attrs)}, ps); } }; template<> struct BuildAST { static void apply(const auto & in, LambdaState & s, State & ps) { s.arg = ps.symbols.create(in.string_view()); } }; template<> struct BuildAST : change_head { static void success(const auto & in, LambdaState & l, ExprState & s, State & ps) { if (l.formals) l.formals = ps.validateFormals(std::move(l.formals), ps.at(in), l.arg); s.pushExpr(ps.at(in), ps.at(in), l.arg, std::move(l.formals), l->popExprOnly()); } }; template<> struct BuildAST { static void apply(const auto & in, ExprState & s, State & ps) { auto body = s.popExprOnly(), cond = s.popExprOnly(); s.pushExpr(ps.at(in), ps.at(in), std::move(cond), std::move(body)); } }; template<> struct BuildAST { static void apply(const auto & in, ExprState & s, State & ps) { auto body = s.popExprOnly(), scope = s.popExprOnly(); s.pushExpr(ps.at(in), ps.at(in), std::move(scope), std::move(body)); } }; template<> struct BuildAST : change_head { static void success(const auto & in, BindingsState & b, ExprState & s, State & ps) { if (!b.attrs.dynamicAttrs.empty()) throw ParseError({ .msg = HintFmt("dynamic attributes not allowed in let"), .pos = ps.positions[ps.at(in)] }); s.pushExpr(ps.at(in), std::make_unique(std::move(b.attrs)), b->popExprOnly()); } }; template<> struct BuildAST { static void apply(const auto & in, ExprState & s, State & ps) { auto else_ = s.popExprOnly(), then = s.popExprOnly(), cond = s.popExprOnly(); s.pushExpr(ps.at(in), ps.at(in), std::move(cond), std::move(then), std::move(else_)); } }; template<> struct BuildAST : change_head { static void success0(ExprState & inner, ExprState & outer, State & ps) { outer.exprs.push_back(inner.finish(ps)); } }; } } namespace nix { Expr * EvalState::parse( char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, std::shared_ptr & staticEnv) { parser::State s = { symbols, positions, basePath, positions.addOrigin(origin, length), exprSymbols, }; parser::ExprState x; assert(length >= 2); assert(text[length - 1] == 0); assert(text[length - 2] == 0); length -= 2; p::string_input inp{std::string_view{text, length}, "input"}; try { p::parse(inp, x, s); } catch (p::parse_error & e) { auto pos = e.positions().back(); throw ParseError({ .msg = HintFmt("syntax error, %s", e.message()), .pos = positions[s.positions.add(s.origin, pos.byte)] }); } auto [_pos, result] = x.finish(s); result->bindVars(*this, staticEnv); return result.release(); } }