diff options
Diffstat (limited to 'src/libexpr')
35 files changed, 2293 insertions, 1561 deletions
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index c50c6d92b..32deecfae 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -9,7 +9,7 @@ namespace nix { static Strings parseAttrPath(std::string_view s) { Strings res; - string cur; + std::string cur; auto i = s.begin(); while (i != s.end()) { if (*i == '.') { @@ -41,7 +41,7 @@ std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s) } -std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath, +std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const std::string & attrPath, Bindings & autoArgs, Value & vIn) { Strings tokens = parseAttrPath(attrPath); @@ -58,7 +58,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr Value * vNew = state.allocValue(); state.autoCallFunction(autoArgs, *v, *vNew); v = vNew; - state.forceValue(*v); + state.forceValue(*v, noPos); /* It should evaluate to either a set or an expression, according to what is specified in the attrPath. */ @@ -74,8 +74,14 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr throw Error("empty attribute name in selection path '%1%'", attrPath); Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); - if (a == v->attrs->end()) - throw AttrPathNotFound("attribute '%1%' in selection path '%2%' not found", attr, attrPath); + if (a == v->attrs->end()) { + std::set<std::string> attrNames; + for (auto & attr : *v->attrs) + attrNames.insert(attr.name); + + auto suggestions = Suggestions::bestMatches(attrNames, attr); + throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath); + } v = &*a->value; pos = *a->pos; } @@ -121,7 +127,7 @@ Pos findPackageFilename(EvalState & state, Value & v, std::string what) std::string filename(pos, 0, colon); unsigned int lineno; try { - lineno = std::stoi(std::string(pos, colon + 1)); + lineno = std::stoi(std::string(pos, colon + 1, std::string::npos)); } catch (std::invalid_argument & e) { throw ParseError("cannot parse line number '%s'", pos); } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index 2ee3ea089..ff1135a06 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -10,8 +10,11 @@ namespace nix { MakeError(AttrPathNotFound, Error); MakeError(NoPositionInfo, Error); -std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attrPath, - Bindings & autoArgs, Value & vIn); +std::pair<Value *, Pos> findAlongAttrPath( + EvalState & state, + const std::string & attrPath, + Bindings & autoArgs, + Value & vIn); /* Heuristic to find the filename and lineno or a nix value. */ Pos findPackageFilename(EvalState & state, Value & v, std::string what); diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index b6091c955..52ac47e9b 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -7,26 +7,19 @@ namespace nix { + /* Allocate a new array of attributes for an attribute set with a specific capacity. The space is implicitly reserved after the Bindings structure. */ Bindings * EvalState::allocBindings(size_t capacity) { + if (capacity == 0) + return &emptyBindings; if (capacity > std::numeric_limits<Bindings::size_t>::max()) throw Error("attribute set of size %d is too big", capacity); - return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); -} - - -void EvalState::mkAttrs(Value & v, size_t capacity) -{ - if (capacity == 0) { - v = vEmptySet; - return; - } - v.mkAttrs(allocBindings(capacity)); nrAttrsets++; nrAttrsInAttrsets += capacity; + return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); } @@ -41,15 +34,36 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) } -Value * EvalState::allocAttr(Value & vAttrs, const std::string & name) +Value * EvalState::allocAttr(Value & vAttrs, std::string_view name) { return allocAttr(vAttrs, symbols.create(name)); } +Value & BindingsBuilder::alloc(const Symbol & name, ptr<Pos> pos) +{ + auto value = state.allocValue(); + bindings->push_back(Attr(name, value, pos)); + return *value; +} + + +Value & BindingsBuilder::alloc(std::string_view name, ptr<Pos> pos) +{ + return alloc(state.symbols.create(name), pos); +} + + void Bindings::sort() { - std::sort(begin(), end()); + if (size_) std::sort(begin(), end()); +} + + +Value & Value::mkAttrs(BindingsBuilder & bindings) +{ + mkAttrs(bindings.finish()); + return *this; } diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 7d6ffc9f3..cad9743ea 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -105,7 +105,7 @@ public: for (size_t n = 0; n < size_; n++) res.emplace_back(&attrs[n]); std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) { - return (const string &) a->name < (const string &) b->name; + return (const std::string &) a->name < (const std::string &) b->name; }); return res; } @@ -113,5 +113,52 @@ public: friend class EvalState; }; +/* A wrapper around Bindings that ensures that its always in sorted + order at the end. The only way to consume a BindingsBuilder is to + call finish(), which sorts the bindings. */ +class BindingsBuilder +{ + Bindings * bindings; + +public: + // needed by std::back_inserter + using value_type = Attr; + + EvalState & state; + + BindingsBuilder(EvalState & state, Bindings * bindings) + : bindings(bindings), state(state) + { } + + void insert(Symbol name, Value * value, ptr<Pos> pos = ptr(&noPos)) + { + insert(Attr(name, value, pos)); + } + + void insert(const Attr & attr) + { + push_back(attr); + } + + void push_back(const Attr & attr) + { + bindings->push_back(attr); + } + + Value & alloc(const Symbol & name, ptr<Pos> pos = ptr(&noPos)); + + Value & alloc(std::string_view name, ptr<Pos> pos = ptr(&noPos)); + + Bindings * finish() + { + bindings->sort(); + return bindings; + } + + Bindings * alreadySorted() + { + return bindings; + } +}; } diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index fb0932c00..e50ff244c 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -73,30 +73,29 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { - Bindings * res = state.allocBindings(autoArgs.size()); + auto res = state.buildBindings(autoArgs.size()); for (auto & i : autoArgs) { - Value * v = state.allocValue(); + auto v = state.allocValue(); if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath("."))); + state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), absPath("."))); else - mkString(*v, string(i.second, 1)); - res->push_back(Attr(state.symbols.create(i.first), v)); + v->mkString(((std::string_view) i.second).substr(1)); + res.insert(state.symbols.create(i.first), v); } - res->sort(); - return res; + return res.finish(); } -Path lookupFileArg(EvalState & state, string s) +Path lookupFileArg(EvalState & state, std::string_view s) { if (isUri(s)) { return state.store->toRealPath( fetchers::downloadTarball( state.store, resolveUri(s), "source", false).first.storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { - Path p = s.substr(1, s.size() - 2); + Path p(s.substr(1, s.size() - 2)); return state.findFile(p); } else - return absPath(s); + return absPath(std::string(s)); } } diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh index 0e113fff1..03fa226aa 100644 --- a/src/libexpr/common-eval-args.hh +++ b/src/libexpr/common-eval-args.hh @@ -22,6 +22,6 @@ private: std::map<std::string, std::string> autoArgs; }; -Path lookupFileArg(EvalState & state, string s); +Path lookupFileArg(EvalState & state, std::string_view s); } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index d7e21783d..188223957 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -336,7 +336,7 @@ Value & AttrCursor::getValue() if (!_value) { if (parent) { auto & vParent = parent->first->getValue(); - root->state.forceAttrs(vParent); + root->state.forceAttrs(vParent, noPos); auto attr = vParent.attrs->get(parent->second); if (!attr) throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); @@ -381,7 +381,7 @@ Value & AttrCursor::forceValue() auto & v = getValue(); try { - root->state.forceValue(v); + root->state.forceValue(v, noPos); } catch (EvalError &) { debug("setting '%s' to failed", getAttrPathStr()); if (root->db) @@ -406,6 +406,16 @@ Value & AttrCursor::forceValue() return v; } +Suggestions AttrCursor::getSuggestionsForAttr(Symbol name) +{ + auto attrNames = getAttrs(); + std::set<std::string> strAttrNames; + for (auto & name : attrNames) + strAttrNames.insert(std::string(name)); + + return Suggestions::bestMatches(strAttrNames, name); +} + std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) { if (root->db) { @@ -446,6 +456,11 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + for (auto & attr : *v.attrs) { + if (root->db) + root->db->setPlaceholder({cachedValue->first, attr.name}); + } + auto attr = v.attrs->get(name); if (!attr) { @@ -464,7 +479,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()}; } - return std::make_shared<AttrCursor>( + return make_ref<AttrCursor>( root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2)); } @@ -473,27 +488,31 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name) return maybeGetAttr(root->state.symbols.create(name)); } -std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors) +ref<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors) { auto p = maybeGetAttr(name, forceErrors); if (!p) throw Error("attribute '%s' does not exist", getAttrPathStr(name)); - return p; + return ref(p); } -std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name) +ref<AttrCursor> AttrCursor::getAttr(std::string_view name) { return getAttr(root->state.symbols.create(name)); } -std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force) +OrSuggestions<ref<AttrCursor>> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force) { auto res = shared_from_this(); for (auto & attr : attrPath) { - res = res->maybeGetAttr(attr, force); - if (!res) return {}; + auto child = res->maybeGetAttr(attr, force); + if (!child) { + auto suggestions = res->getSuggestionsForAttr(attr); + return OrSuggestions<ref<AttrCursor>>::failed(suggestions); + } + res = child; } - return res; + return ref(res); } std::string AttrCursor::getString() @@ -596,7 +615,7 @@ std::vector<Symbol> AttrCursor::getAttrs() for (auto & attr : *getValue().attrs) attrs.push_back(attr.name); std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) { - return (const string &) a < (const string &) b; + return (const std::string &) a < (const std::string &) b; }); if (root->db) diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 43b34ebcb..40f1d4ffc 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -94,15 +94,17 @@ public: std::string getAttrPathStr(Symbol name) const; + Suggestions getSuggestionsForAttr(Symbol name); + std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false); std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name); - std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false); + ref<AttrCursor> getAttr(Symbol name, bool forceErrors = false); - std::shared_ptr<AttrCursor> getAttr(std::string_view name); + ref<AttrCursor> getAttr(std::string_view name); - std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); + OrSuggestions<ref<AttrCursor>> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); std::string getString(); diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 655408cd3..aef1f6351 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -15,12 +15,6 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s)) }); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) -{ - throw TypeError(s, showType(v)); -} - - LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v)) { throw TypeError({ @@ -32,6 +26,13 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const void EvalState::forceValue(Value & v, const Pos & pos) { + forceValue(v, [&]() { return pos; }); +} + + +template<typename Callable> +void EvalState::forceValue(Value & v, Callable getPos) +{ if (v.isThunk()) { Env * env = v.thunk.env; Expr * expr = v.thunk.expr; @@ -47,31 +48,22 @@ void EvalState::forceValue(Value & v, const Pos & pos) else if (v.isApp()) callFunction(*v.app.left, *v.app.right, v, noPos); else if (v.isBlackhole()) - throwEvalError(pos, "infinite recursion encountered"); -} - - -inline void EvalState::forceAttrs(Value & v) -{ - forceValue(v); - if (v.type() != nAttrs) - throwTypeError("value is %1% while a set was expected", v); + throwEvalError(getPos(), "infinite recursion encountered"); } inline void EvalState::forceAttrs(Value & v, const Pos & pos) { - forceValue(v, pos); - if (v.type() != nAttrs) - throwTypeError(pos, "value is %1% while a set was expected", v); + forceAttrs(v, [&]() { return pos; }); } -inline void EvalState::forceList(Value & v) +template <typename Callable> +inline void EvalState::forceAttrs(Value & v, Callable getPos) { - forceValue(v); - if (!v.isList()) - throwTypeError("value is %1% while a list was expected", v); + forceValue(v, getPos); + if (v.type() != nAttrs) + throwTypeError(getPos(), "value is %1% while a set was expected", v); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 800839a8d..5bf161cc0 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,5 +1,6 @@ #include "eval.hh" #include "hash.hh" +#include "types.hh" #include "util.hh" #include "store-api.hh" #include "derivations.hh" @@ -36,6 +37,19 @@ namespace nix { +static char * allocString(size_t size) +{ + char * t; +#if HAVE_BOEHMGC + t = (char *) GC_MALLOC_ATOMIC(size); +#else + t = malloc(size); +#endif + if (!t) throw std::bad_alloc(); + return t; +} + + static char * dupString(const char * s) { char * t; @@ -49,9 +63,15 @@ static char * dupString(const char * s) } -static char * dupStringWithLen(const char * s, size_t size) +// When there's no need to write to the string, we can optimize away empty +// string allocations. +// This function handles makeImmutableStringWithLen(null, 0) by returning the +// empty string. +static const char * makeImmutableStringWithLen(const char * s, size_t size) { char * t; + if (size == 0) + return ""; #if HAVE_BOEHMGC t = GC_STRNDUP(s, size); #else @@ -61,6 +81,10 @@ static char * dupStringWithLen(const char * s, size_t size) return t; } +static inline const char * makeImmutableString(std::string_view s) { + return makeImmutableStringWithLen(s.data(), s.size()); +} + RootValue allocRootValue(Value * v) { @@ -72,15 +96,10 @@ RootValue allocRootValue(Value * v) } -void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) +void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v) { checkInterrupt(); - if (!active.insert(&v).second) { - str << "<CYCLE>"; - return; - } - switch (v.internalType) { case tInt: str << v.integer; @@ -106,24 +125,32 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu str << "null"; break; case tAttrs: { - str << "{ "; - for (auto & i : v.attrs->lexicographicOrder()) { - str << i->name << " = "; - printValue(str, active, *i->value); - str << "; "; + if (!v.attrs->empty() && !seen.insert(v.attrs).second) + str << "<REPEAT>"; + else { + str << "{ "; + for (auto & i : v.attrs->lexicographicOrder()) { + str << i->name << " = "; + printValue(str, seen, *i->value); + str << "; "; + } + str << "}"; } - str << "}"; break; } case tList1: case tList2: case tListN: - str << "[ "; - for (unsigned int n = 0; n < v.listSize(); ++n) { - printValue(str, active, *v.listElems()[n]); - str << " "; + if (v.listSize() && !seen.insert(v.listElems()).second) + str << "<REPEAT>"; + else { + str << "[ "; + for (auto v2 : v.listItems()) { + printValue(str, seen, *v2); + str << " "; + } + str << "]"; } - str << "]"; break; case tThunk: case tApp: @@ -145,22 +172,20 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu str << v.fpoint; break; default: - throw Error("invalid value"); + abort(); } - - active.erase(&v); } std::ostream & operator << (std::ostream & str, const Value & v) { - std::set<const Value *> active; - printValue(str, active, v); + std::set<const void *> seen; + printValue(str, seen, v); return str; } -const Value *getPrimOp(const Value &v) { +const Value * getPrimOp(const Value &v) { const Value * primOp = &v; while (primOp->isPrimOpApp()) { primOp = primOp->primOpApp.left; @@ -169,7 +194,7 @@ const Value *getPrimOp(const Value &v) { return primOp; } -string showType(ValueType type) +std::string_view showType(ValueType type) { switch (type) { case nInt: return "an integer"; @@ -188,24 +213,24 @@ string showType(ValueType type) } -string showType(const Value & v) +std::string showType(const Value & v) { switch (v.internalType) { case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: - return fmt("the built-in function '%s'", string(v.primOp->name)); + return fmt("the built-in function '%s'", std::string(v.primOp->name)); case tPrimOpApp: - return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); + return fmt("the partially applied built-in function '%s'", std::string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); case tThunk: return "a thunk"; case tApp: return "a function application"; case tBlackhole: return "a black hole"; default: - return showType(v.type()); + return std::string(showType(v.type())); } } -Pos Value::determinePos(const Pos &pos) const +Pos Value::determinePos(const Pos & pos) const { switch (internalType) { case tAttrs: return *attrs->pos; @@ -342,7 +367,7 @@ void initGC() /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const string & s) +static Strings parseNixPath(const std::string & s) { Strings res; @@ -412,10 +437,21 @@ EvalState::EvalState( , sDescription(symbols.create("description")) , sSelf(symbols.create("self")) , sEpsilon(symbols.create("")) + , sStartSet(symbols.create("startSet")) + , sOperator(symbols.create("operator")) + , sKey(symbols.create("key")) + , sPath(symbols.create("path")) + , sPrefix(symbols.create("prefix")) , repair(NoRepair) + , emptyBindings(0) , store(store) , buildStore(buildStore ? buildStore : store) , regexCache(makeRegexCache()) +#if HAVE_BOEHMGC + , valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr)) +#else + , valueAllocCache(std::make_shared<void *>(nullptr)) +#endif , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { @@ -445,17 +481,15 @@ EvalState::EvalState( StorePathSet closure; store->computeFSClosure(store->toStorePath(r.second).first, closure); for (auto & path : closure) - allowedPaths->insert(store->printStorePath(path)); + allowPath(path); } catch (InvalidPath &) { - allowedPaths->insert(r.second); + allowPath(r.second); } } else - allowedPaths->insert(r.second); + allowPath(r.second); } } - vEmptySet.mkAttrs(allocBindings(0)); - createBaseEnv(); } @@ -466,7 +500,7 @@ EvalState::~EvalState() void EvalState::requireExperimentalFeatureOnEvaluation( - const std::string & feature, + const ExperimentalFeature & feature, const std::string_view fName, const Pos & pos) { @@ -482,6 +516,26 @@ void EvalState::requireExperimentalFeatureOnEvaluation( } } +void EvalState::allowPath(const Path & path) +{ + if (allowedPaths) + allowedPaths->insert(path); +} + +void EvalState::allowPath(const StorePath & storePath) +{ + if (allowedPaths) + allowedPaths->insert(store->toRealPath(storePath)); +} + +void EvalState::allowAndSetStorePathString(const StorePath &storePath, Value & v) +{ + allowPath(storePath); + + auto path = store->printStorePath(storePath); + v.mkString(path, PathSet({path})); +} + Path EvalState::checkSourcePath(const Path & path_) { if (!allowedPaths) return path_; @@ -507,8 +561,12 @@ Path EvalState::checkSourcePath(const Path & path_) } } - if (!found) - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath); + if (!found) { + auto modeInformation = evalSettings.pureEval + ? "in pure eval mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation); + } /* Resolve symlinks. */ debug(format("checking access to '%s'") % abspath); @@ -521,7 +579,7 @@ Path EvalState::checkSourcePath(const Path & path_) } } - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); + throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path); } @@ -567,22 +625,28 @@ Path EvalState::toRealPath(const Path & path, const PathSet & context) } -Value * EvalState::addConstant(const string & name, Value & v) +Value * EvalState::addConstant(const std::string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v2; - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + addConstant(name, v2); return v2; } -Value * EvalState::addPrimOp(const string & name, +void EvalState::addConstant(const std::string & name, Value * v) +{ + staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); + baseEnv.values[baseEnvDispl++] = v; + auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); +} + + +Value * EvalState::addPrimOp(const std::string & name, size_t arity, PrimOpFun primOp) { - auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + auto name2 = name.substr(0, 2) == "__" ? name.substr(2) : name; Symbol sym = symbols.create(name2); /* Hack to make constants lazy: turn them into a application of @@ -591,13 +655,13 @@ Value * EvalState::addPrimOp(const string & name, auto vPrimOp = allocValue(); vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym }); Value v; - mkApp(v, *vPrimOp, *vPrimOp); + v.mkApp(vPrimOp, vPrimOp); return addConstant(name, v); } Value * v = allocValue(); v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym }); - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); return v; @@ -613,7 +677,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) auto vPrimOp = allocValue(); vPrimOp->mkPrimOp(new PrimOp(std::move(primOp))); Value v; - mkApp(v, *vPrimOp, *vPrimOp); + v.mkApp(vPrimOp, vPrimOp); return addConstant(primOp.name, v); } @@ -623,14 +687,14 @@ Value * EvalState::addPrimOp(PrimOp && primOp) Value * v = allocValue(); v->mkPrimOp(new PrimOp(std::move(primOp))); - staticBaseEnv.vars[envName] = baseEnvDispl; + staticBaseEnv.vars.emplace_back(envName, baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); return v; } -Value & EvalState::getBuiltin(const string & name) +Value & EvalState::getBuiltin(const std::string & name) { return *baseEnv.values[0]->attrs->find(symbols.create(name))->value; } @@ -658,12 +722,12 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v) evaluator. So here are some helper functions for throwing exceptions. */ -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) +LocalNoInlineNoReturn(void throwEvalError(const char * s, const std::string & s2)) { throw EvalError(s, s2); } -LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const std::string & s2)) { throw EvalError({ .msg = hintfmt(s, s2), @@ -671,12 +735,12 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const }); } -LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) +LocalNoInlineNoReturn(void throwEvalError(const char * s, const std::string & s2, const std::string & s3)) { throw EvalError(s, s2, s3); } -LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3)) +LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const std::string & s2, const std::string & s3)) { throw EvalError({ .msg = hintfmt(s, s2, s3), @@ -709,7 +773,12 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const }); } -LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1)) +LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) +{ + throw TypeError(s, showType(v)); +} + +LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const std::string & s1)) { throw AssertionError({ .msg = hintfmt(s, s1), @@ -717,7 +786,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, }); } -LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1)) +LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const std::string & s1)) { throw UndefinedVarError({ .msg = hintfmt(s, s1), @@ -725,7 +794,7 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * }); } -LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1)) +LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const std::string & s1)) { throw MissingArgumentError({ .msg = hintfmt(s, s1), @@ -733,26 +802,25 @@ LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char }); } -LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2)) +LocalNoInline(void addErrorTrace(Error & e, const char * s, const std::string & s2)) { e.addTrace(std::nullopt, s, s2); } -LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2)) +LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const std::string & s2)) { e.addTrace(pos, s, s2); } -void mkString(Value & v, const char * s) +void Value::mkString(std::string_view s) { - v.mkString(dupString(s)); + mkString(makeImmutableString(s)); } -Value & mkString(Value & v, std::string_view s, const PathSet & context) +static void copyContextToValue(Value & v, const PathSet & context) { - v.mkString(dupStringWithLen(s.data(), s.size())); if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) @@ -761,19 +829,30 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context) v.string.context[n++] = dupString(i.c_str()); v.string.context[n] = 0; } - return v; } +void Value::mkString(std::string_view s, const PathSet & context) +{ + mkString(s); + copyContextToValue(*this, context); +} + +void Value::mkStringMove(const char * s, const PathSet & context) +{ + mkString(s); + copyContextToValue(*this, context); +} -void mkPath(Value & v, const char * s) + +void Value::mkPath(std::string_view s) { - v.mkPath(dupString(s)); + mkPath(makeImmutableString(s)); } inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) { - for (size_t l = var.level; l; --l, env = env->up) ; + for (auto l = var.level; l; --l, env = env->up) ; if (!var.fromWith) return env->values[var.displ]; @@ -799,8 +878,23 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) Value * EvalState::allocValue() { + /* We use the boehm batch allocator to speed up allocations of Values (of which there are many). + GC_malloc_many returns a linked list of objects of the given size, where the first word + of each object is also the pointer to the next object in the list. This also means that we + have to explicitly clear the first word of every object we take. */ + if (!*valueAllocCache) { + *valueAllocCache = GC_malloc_many(sizeof(Value)); + if (!*valueAllocCache) throw std::bad_alloc(); + } + + /* GC_NEXT is a convenience macro for accessing the first word of an object. + Take the first list item, advance the list to the next item, and clear the next pointer. */ + void * p = *valueAllocCache; + GC_PTR_STORE_AND_DIRTY(&*valueAllocCache, GC_NEXT(p)); + GC_NEXT(p) = nullptr; + nrValues++; - auto v = (Value *) allocBytes(sizeof(Value)); + auto v = (Value *) p; return v; } @@ -845,13 +939,13 @@ void EvalState::mkThunk_(Value & v, Expr * expr) void EvalState::mkPos(Value & v, ptr<Pos> pos) { if (pos->file.set()) { - mkAttrs(v, 3); - mkString(*allocAttr(v, sFile), pos->file); - mkInt(*allocAttr(v, sLine), pos->line); - mkInt(*allocAttr(v, sColumn), pos->column); - v.attrs->sort(); + auto attrs = buildBindings(3); + attrs.alloc(sFile).mkString(pos->file); + attrs.alloc(sLine).mkInt(pos->line); + attrs.alloc(sColumn).mkInt(pos->column); + v.mkAttrs(attrs); } else - mkNull(v); + v.mkNull(); } @@ -1030,8 +1124,8 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v) void ExprAttrs::eval(EvalState & state, Env & env, Value & v) { - state.mkAttrs(v, attrs.size() + dynamicAttrs.size()); - Env *dynamicEnv = &env; + v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish()); + auto dynamicEnv = &env; if (recursive) { /* Create a new environment that contains the attributes in @@ -1046,7 +1140,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - size_t displ = 0; + Displacement displ = 0; for (auto & i : attrs) { Value * vAttr; if (hasOverrides && !i.second.inherited) { @@ -1068,7 +1162,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Hence we need __overrides.) */ if (hasOverrides) { Value * vOverrides = (*v.attrs)[overrides->second.displ].value; - state.forceAttrs(*vOverrides); + state.forceAttrs(*vOverrides, [&]() { return vOverrides->determinePos(noPos); }); Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size()); for (auto & i : *v.attrs) newBnds->push_back(i); @@ -1122,7 +1216,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - size_t displ = 0; + Displacement displ = 0; for (auto & i : attrs->attrs) env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); @@ -1133,8 +1227,8 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) void ExprList::eval(EvalState & state, Env & env, Value & v) { state.mkList(v, elems.size()); - for (size_t n = 0; n < elems.size(); ++n) - v.listElems()[n] = elems[n]->maybeThunk(state, env); + for (auto [n, v2] : enumerate(v.listItems())) + const_cast<Value * &>(v2) = elems[n]->maybeThunk(state, env); } @@ -1146,7 +1240,7 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v) } -static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath) +static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath) { std::ostringstream out; bool first = true; @@ -1216,20 +1310,20 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) e->eval(state, env, vTmp); for (auto & i : attrPath) { - state.forceValue(*vAttrs); + state.forceValue(*vAttrs, noPos); Bindings::iterator j; Symbol name = getName(i, state, env); if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { - mkBool(v, false); + v.mkBool(false); return; } else { vAttrs = j->value; } } - mkBool(v, true); + v.mkBool(true); } @@ -1239,144 +1333,184 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) } -void ExprApp::eval(EvalState & state, Env & env, Value & v) +void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos) { - /* FIXME: vFun prevents GCC from doing tail call optimisation. */ - Value vFun; - e1->eval(state, env, vFun); - state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos); -} + auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr; + forceValue(fun, pos); -void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) -{ - /* Figure out the number of arguments still needed. */ - size_t argsDone = 0; - Value * primOp = &fun; - while (primOp->isPrimOpApp()) { - argsDone++; - primOp = primOp->primOpApp.left; - } - assert(primOp->isPrimOp()); - auto arity = primOp->primOp->arity; - auto argsLeft = arity - argsDone; - - if (argsLeft == 1) { - /* We have all the arguments, so call the primop. */ - - /* Put all the arguments in an array. */ - Value * vArgs[arity]; - auto n = arity - 1; - vArgs[n--] = &arg; - for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left) - vArgs[n--] = arg->primOpApp.right; - - /* And call the primop. */ - nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, pos, vArgs, v); - } else { - Value * fun2 = allocValue(); - *fun2 = fun; - v.mkPrimOpApp(fun2, &arg); - } -} + Value vCur(fun); -void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) -{ - auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr; + auto makeAppChain = [&]() + { + vRes = vCur; + for (size_t i = 0; i < nrArgs; ++i) { + auto fun2 = allocValue(); + *fun2 = vRes; + vRes.mkPrimOpApp(fun2, args[i]); + } + }; - forceValue(fun, pos); + Attr * functor; - if (fun.isPrimOp() || fun.isPrimOpApp()) { - callPrimOp(fun, arg, v, pos); - return; - } + while (nrArgs > 0) { - if (fun.type() == nAttrs) { - auto found = fun.attrs->find(sFunctor); - if (found != fun.attrs->end()) { - /* fun may be allocated on the stack of the calling function, - * but for functors we may keep a reference, so heap-allocate - * a copy and use that instead. - */ - auto & fun2 = *allocValue(); - fun2 = fun; - /* !!! Should we use the attr pos here? */ - Value v2; - callFunction(*found->value, fun2, v2, pos); - return callFunction(v2, arg, v, pos); - } - } + if (vCur.isLambda()) { - if (!fun.isLambda()) - throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); + ExprLambda & lambda(*vCur.lambda.fun); - ExprLambda & lambda(*fun.lambda.fun); + auto size = + (lambda.arg.empty() ? 0 : 1) + + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); + Env & env2(allocEnv(size)); + env2.up = vCur.lambda.env; - auto size = - (lambda.arg.empty() ? 0 : 1) + - (lambda.matchAttrs ? lambda.formals->formals.size() : 0); - Env & env2(allocEnv(size)); - env2.up = fun.lambda.env; + Displacement displ = 0; - size_t displ = 0; + if (!lambda.hasFormals()) + env2.values[displ++] = args[0]; - if (!lambda.matchAttrs) - env2.values[displ++] = &arg; + else { + forceAttrs(*args[0], pos); - else { - forceAttrs(arg, pos); - - if (!lambda.arg.empty()) - env2.values[displ++] = &arg; - - /* For each formal argument, get the actual argument. If - there is no matching actual argument but the formal - argument has a default, use the default. */ - size_t attrsUsed = 0; - for (auto & i : lambda.formals->formals) { - Bindings::iterator j = arg.attrs->find(i.name); - if (j == arg.attrs->end()) { - if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", - lambda, i.name); - env2.values[displ++] = i.def->maybeThunk(*this, env2); + if (!lambda.arg.empty()) + env2.values[displ++] = args[0]; + + /* For each formal argument, get the actual argument. If + there is no matching actual argument but the formal + argument has a default, use the default. */ + size_t attrsUsed = 0; + for (auto & i : lambda.formals->formals) { + auto j = args[0]->attrs->get(i.name); + if (!j) { + if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", + lambda, i.name); + env2.values[displ++] = i.def->maybeThunk(*this, env2); + } else { + attrsUsed++; + env2.values[displ++] = j->value; + } + } + + /* Check that each actual argument is listed as a formal + argument (unless the attribute match specifies a `...'). */ + if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) { + /* Nope, so show the first unexpected argument to the + user. */ + for (auto & i : *args[0]->attrs) + if (!lambda.formals->has(i.name)) + throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); + abort(); // can't happen + } + } + + nrFunctionCalls++; + if (countCalls) incrFunctionCall(&lambda); + + /* Evaluate the body. */ + try { + lambda.body->eval(*this, env2, vCur); + } catch (Error & e) { + if (loggerSettings.showTrace.get()) { + addErrorTrace(e, lambda.pos, "while evaluating %s", + (lambda.name.set() + ? "'" + (const std::string &) lambda.name + "'" + : "anonymous lambda")); + addErrorTrace(e, pos, "from call site%s", ""); + } + throw; + } + + nrArgs--; + args += 1; + } + + else if (vCur.isPrimOp()) { + + size_t argsLeft = vCur.primOp->arity; + + if (nrArgs < argsLeft) { + /* We don't have enough arguments, so create a tPrimOpApp chain. */ + makeAppChain(); + return; + } else { + /* We have all the arguments, so call the primop. */ + nrPrimOpCalls++; + if (countCalls) primOpCalls[vCur.primOp->name]++; + vCur.primOp->fun(*this, pos, args, vCur); + + nrArgs -= argsLeft; + args += argsLeft; + } + } + + else if (vCur.isPrimOpApp()) { + /* Figure out the number of arguments still needed. */ + size_t argsDone = 0; + Value * primOp = &vCur; + while (primOp->isPrimOpApp()) { + argsDone++; + primOp = primOp->primOpApp.left; + } + assert(primOp->isPrimOp()); + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; + + if (nrArgs < argsLeft) { + /* We still don't have enough arguments, so extend the tPrimOpApp chain. */ + makeAppChain(); + return; } else { - attrsUsed++; - env2.values[displ++] = j->value; + /* We have all the arguments, so call the primop with + the previous and new arguments. */ + + Value * vArgs[arity]; + auto n = argsDone; + for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) + vArgs[--n] = arg->primOpApp.right; + + for (size_t i = 0; i < argsLeft; ++i) + vArgs[argsDone + i] = args[i]; + + nrPrimOpCalls++; + if (countCalls) primOpCalls[primOp->primOp->name]++; + primOp->primOp->fun(*this, pos, vArgs, vCur); + + nrArgs -= argsLeft; + args += argsLeft; } } - /* Check that each actual argument is listed as a formal - argument (unless the attribute match specifies a `...'). */ - if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { - /* Nope, so show the first unexpected argument to the - user. */ - for (auto & i : *arg.attrs) - if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) - throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); - abort(); // can't happen + else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) { + /* 'vCur' may be allocated on the stack of the calling + function, but for functors we may keep a reference, so + heap-allocate a copy and use that instead. */ + Value * args2[] = {allocValue(), args[0]}; + *args2[0] = vCur; + /* !!! Should we use the attr pos here? */ + callFunction(*functor->value, 2, args2, vCur, pos); + nrArgs--; + args++; } + + else + throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur); } - nrFunctionCalls++; - if (countCalls) incrFunctionCall(&lambda); + vRes = vCur; +} - /* Evaluate the body. This is conditional on showTrace, because - catching exceptions makes this function not tail-recursive. */ - if (loggerSettings.showTrace.get()) - try { - lambda.body->eval(*this, env2, v); - } catch (Error & e) { - addErrorTrace(e, lambda.pos, "while evaluating %s", - (lambda.name.set() - ? "'" + (string) lambda.name + "'" - : "anonymous lambda")); - addErrorTrace(e, pos, "from call site%s", ""); - throw; - } - else - fun.lambda.fun->body->eval(*this, env2, v); + +void ExprCall::eval(EvalState & state, Env & env, Value & v) +{ + Value vFun; + fun->eval(state, env, vFun); + + Value * vArgs[args.size()]; + for (size_t i = 0; i < args.size(); ++i) + vArgs[i] = args[i]->maybeThunk(state, env); + + state.callFunction(vFun, args.size(), vArgs, v, pos); } @@ -1390,39 +1524,39 @@ void EvalState::incrFunctionCall(ExprLambda * fun) void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) { - forceValue(fun); + auto pos = fun.determinePos(noPos); + + forceValue(fun, pos); if (fun.type() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { Value * v = allocValue(); - callFunction(*found->value, fun, *v, noPos); - forceValue(*v); + callFunction(*found->value, fun, *v, pos); + forceValue(*v, pos); return autoCallFunction(args, *v, res); } } - if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) { + if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) { res = fun; return; } - Value * actualArgs = allocValue(); - mkAttrs(*actualArgs, std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size())); + auto attrs = buildBindings(std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size())); if (fun.lambda.fun->formals->ellipsis) { // If the formals have an ellipsis (eg the function accepts extra args) pass // all available automatic arguments (which includes arguments specified on // the command line via --arg/--argstr) - for (auto& v : args) { - actualArgs->attrs->push_back(v); - } + for (auto & v : args) + attrs.insert(v); } else { // Otherwise, only pass the arguments that the function accepts for (auto & i : fun.lambda.fun->formals->formals) { Bindings::iterator j = args.find(i.name); if (j != args.end()) { - actualArgs->attrs->push_back(*j); + attrs.insert(*j); } else if (!i.def) { throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') @@ -1435,9 +1569,7 @@ https://nixos.org/manual/nix/stable/#ss-functions.)", i.name); } } - actualArgs->attrs->sort(); - - callFunction(fun, *actualArgs, res, noPos); + callFunction(fun, allocValue()->mkAttrs(attrs), res, noPos); } @@ -1472,7 +1604,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) void ExprOpNot::eval(EvalState & state, Env & env, Value & v) { - mkBool(v, !state.evalBool(env, e)); + v.mkBool(!state.evalBool(env, e)); } @@ -1480,7 +1612,7 @@ void ExprOpEq::eval(EvalState & state, Env & env, Value & v) { Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); - mkBool(v, state.eqValues(v1, v2)); + v.mkBool(state.eqValues(v1, v2)); } @@ -1488,25 +1620,25 @@ void ExprOpNEq::eval(EvalState & state, Env & env, Value & v) { Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); - mkBool(v, !state.eqValues(v1, v2)); + v.mkBool(!state.eqValues(v1, v2)); } void ExprOpAnd::eval(EvalState & state, Env & env, Value & v) { - mkBool(v, state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); + v.mkBool(state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); } void ExprOpOr::eval(EvalState & state, Env & env, Value & v) { - mkBool(v, state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); + v.mkBool(state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); } void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) { - mkBool(v, !state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); + v.mkBool(!state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); } @@ -1521,7 +1653,7 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) if (v1.attrs->size() == 0) { v = v2; return; } if (v2.attrs->size() == 0) { v = v1; return; } - state.mkAttrs(v, v1.attrs->size() + v2.attrs->size()); + auto attrs = state.buildBindings(v1.attrs->size() + v2.attrs->size()); /* Merge the sets, preferring values from the second set. Make sure to keep the resulting vector in sorted order. */ @@ -1530,17 +1662,19 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) while (i != v1.attrs->end() && j != v2.attrs->end()) { if (i->name == j->name) { - v.attrs->push_back(*j); + attrs.insert(*j); ++i; ++j; } else if (i->name < j->name) - v.attrs->push_back(*i++); + attrs.insert(*i++); else - v.attrs->push_back(*j++); + attrs.insert(*j++); } - while (i != v1.attrs->end()) v.attrs->push_back(*i++); - while (j != v2.attrs->end()) v.attrs->push_back(*j++); + while (i != v1.attrs->end()) attrs.insert(*i++); + while (j != v2.attrs->end()) attrs.insert(*j++); + + v.mkAttrs(attrs.alreadySorted()); state.nrOpUpdateValuesCopied += v.attrs->size(); } @@ -1587,15 +1721,39 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) { PathSet context; - std::ostringstream s; + std::vector<BackedStringView> s; + size_t sSize = 0; NixInt n = 0; NixFloat nf = 0; bool first = !forceString; ValueType firstType = nString; - for (auto & i : *es) { - Value vTmp; + const auto str = [&] { + std::string result; + result.reserve(sSize); + for (const auto & part : s) result += *part; + return result; + }; + /* c_str() is not str().c_str() because we want to create a string + Value. allocating a GC'd string directly and moving it into a + Value lets us avoid an allocation and copy. */ + const auto c_str = [&] { + char * result = allocString(sSize + 1); + char * tmp = result; + for (const auto & part : s) { + memcpy(tmp, part->data(), part->size()); + tmp += part->size(); + } + *tmp = 0; + return result; + }; + + Value values[es->size()]; + Value * vTmpP = values; + + for (auto & [i_pos, i] : *es) { + Value & vTmp = *vTmpP++; i->eval(state, env, vTmp); /* If the first element is a path, then the result will also @@ -1615,34 +1773,37 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) nf = n; nf += vTmp.fpoint; } else - throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); + throwEvalError(i_pos, "cannot add %1% to an integer", showType(vTmp)); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { nf += vTmp.integer; } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint; } else - throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); - } else + throwEvalError(i_pos, "cannot add %1% to a float", showType(vTmp)); + } else { + if (s.empty()) s.reserve(es->size()); /* skip canonization of first path, which would only be not canonized in the first place if it's coming from a ./${foo} type path */ - s << state.coerceToString(pos, vTmp, context, false, firstType == nString, !first); + auto part = state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first); + sSize += part->size(); + s.emplace_back(std::move(part)); + } first = false; } if (firstType == nInt) - mkInt(v, n); + v.mkInt(n); else if (firstType == nFloat) - mkFloat(v, nf); + v.mkFloat(nf); else if (firstType == nPath) { if (!context.empty()) throwEvalError(pos, "a string that refers to a store path cannot be appended to a path"); - auto path = canonPath(s.str()); - mkPath(v, path.c_str()); + v.mkPath(canonPath(str())); } else - mkString(v, s.str(), context); + v.mkStringMove(c_str(), context); } @@ -1661,7 +1822,7 @@ void EvalState::forceValueDeep(Value & v) recurse = [&](Value & v) { if (!seen.insert(&v).second) return; - forceValue(v); + forceValue(v, [&]() { return v.determinePos(noPos); }); if (v.type() == nAttrs) { for (auto & i : *v.attrs) @@ -1674,8 +1835,8 @@ void EvalState::forceValueDeep(Value & v) } else if (v.isList()) { - for (size_t n = 0; n < v.listSize(); ++n) - recurse(*v.listElems()[n]); + for (auto v2 : v.listItems()) + recurse(*v2); } }; @@ -1726,7 +1887,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos) } -string EvalState::forceString(Value & v, const Pos & pos) +std::string_view EvalState::forceString(Value & v, const Pos & pos) { forceValue(v, pos); if (v.type() != nString) { @@ -1735,13 +1896,13 @@ string EvalState::forceString(Value & v, const Pos & pos) else throwTypeError("value is %1% while a string was expected", v); } - return string(v.string.s); + return v.string.s; } /* Decode a context string ‘!<name>!<path>’ into a pair <path, name>. */ -std::pair<string, string> decodeContext(std::string_view s) +std::pair<std::string, std::string> decodeContext(std::string_view s) { if (s.at(0) == '!') { size_t index = s.find("!", 1); @@ -1770,17 +1931,17 @@ std::vector<std::pair<Path, std::string>> Value::getContext() } -string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) +std::string_view EvalState::forceString(Value & v, PathSet & context, const Pos & pos) { - string s = forceString(v, pos); + auto s = forceString(v, pos); copyContext(v, context); return s; } -string EvalState::forceStringNoCtx(Value & v, const Pos & pos) +std::string_view EvalState::forceStringNoCtx(Value & v, const Pos & pos) { - string s = forceString(v, pos); + auto s = forceString(v, pos); if (v.string.context) { if (pos) throwEvalError(pos, "the string '%1%' is not allowed to refer to a store path (such as '%2%')", @@ -1798,47 +1959,48 @@ bool EvalState::isDerivation(Value & v) if (v.type() != nAttrs) return false; Bindings::iterator i = v.attrs->find(sType); if (i == v.attrs->end()) return false; - forceValue(*i->value); + forceValue(*i->value, *i->pos); if (i->value->type() != nString) return false; return strcmp(i->value->string.s, "derivation") == 0; } -std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v, +std::optional<std::string> EvalState::tryAttrsToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore, bool copyToStore) { auto i = v.attrs->find(sToString); if (i != v.attrs->end()) { Value v1; callFunction(*i->value, v, v1, pos); - return coerceToString(pos, v1, context, coerceMore, copyToStore); + return coerceToString(pos, v1, context, coerceMore, copyToStore).toOwned(); } return {}; } -string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, +BackedStringView EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore, bool copyToStore, bool canonicalizePath) { forceValue(v, pos); - string s; - if (v.type() == nString) { copyContext(v, context); - return v.string.s; + return std::string_view(v.string.s); } if (v.type() == nPath) { - Path path(canonicalizePath ? canonPath(v.path) : v.path); - return copyToStore ? copyPathToStore(context, path) : path; + BackedStringView path(PathView(v.path)); + if (canonicalizePath) + path = canonPath(*path); + if (copyToStore) + path = copyPathToStore(context, std::move(path).toOwned()); + return path; } if (v.type() == nAttrs) { auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); - if (maybeString) { - return *maybeString; - } + if (maybeString) + return std::move(*maybeString); auto i = v.attrs->find(sOutPath); if (i == v.attrs->end()) throwTypeError(pos, "cannot coerce a set to a string"); return coerceToString(pos, *i->value, context, coerceMore, copyToStore); @@ -1858,16 +2020,15 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, if (v.type() == nNull) return ""; if (v.isList()) { - string result; - for (size_t n = 0; n < v.listSize(); ++n) { - result += coerceToString(pos, *v.listElems()[n], - context, coerceMore, copyToStore); + std::string result; + for (auto [n, v2] : enumerate(v.listItems())) { + result += *coerceToString(pos, *v2, context, coerceMore, copyToStore); if (n < v.listSize() - 1 /* !!! not quite correct */ - && (!v.listElems()[n]->isList() || v.listElems()[n]->listSize() != 0)) + && (!v2->isList() || v2->listSize() != 0)) result += " "; } - return result; + return std::move(result); } } @@ -1875,7 +2036,7 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, } -string EvalState::copyPathToStore(PathSet & context, const Path & path) +std::string EvalState::copyPathToStore(PathSet & context, const Path & path) { if (nix::isDerivation(path)) throwEvalError("file names are not allowed to end in '%1%'", drvExtension); @@ -1889,6 +2050,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); dstPath = store->printStorePath(p); + allowPath(p); srcToStore.insert_or_assign(path, std::move(p)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); } @@ -1900,17 +2062,29 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context) { - string path = coerceToString(pos, v, context, false, false); + auto path = coerceToString(pos, v, context, false, false).toOwned(); if (path == "" || path[0] != '/') throwEvalError(pos, "string '%1%' doesn't represent an absolute path", path); return path; } +StorePath EvalState::coerceToStorePath(const Pos & pos, Value & v, PathSet & context) +{ + auto path = coerceToString(pos, v, context, false, false).toOwned(); + if (auto storePath = store->maybeParseStorePath(path)) + return *storePath; + throw EvalError({ + .msg = hintfmt("path '%1%' is not in the Nix store", path), + .errPos = pos + }); +} + + bool EvalState::eqValues(Value & v1, Value & v2) { - forceValue(v1); - forceValue(v2); + forceValue(v1, noPos); + forceValue(v2, noPos); /* !!! Hack to support some old broken code that relies on pointer equality tests between sets. (Specifically, builderDefs calls @@ -2070,11 +2244,11 @@ void EvalState::printStats() for (auto & i : functionCalls) { auto obj = list.object(); if (i.first->name.set()) - obj.attr("name", (const string &) i.first->name); + obj.attr("name", (const std::string &) i.first->name); else obj.attr("name", nullptr); if (i.first->pos) { - obj.attr("file", (const string &) i.first->pos.file); + obj.attr("file", (const std::string &) i.first->pos.file); obj.attr("line", i.first->pos.line); obj.attr("column", i.first->pos.column); } @@ -2086,7 +2260,7 @@ void EvalState::printStats() for (auto & i : attrSelects) { auto obj = list.object(); if (i.first) { - obj.attr("file", (const string &) i.first.file); + obj.attr("file", (const std::string &) i.first.file); obj.attr("line", i.first.line); obj.attr("column", i.first.column); } @@ -2103,7 +2277,7 @@ void EvalState::printStats() } -string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const +std::string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { throw TypeError({ .msg = hintfmt("cannot coerce %1% to a string", showType()), diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9df6150c6..800b00eef 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -1,10 +1,12 @@ #pragma once #include "attr-set.hh" +#include "types.hh" #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" #include "config.hh" +#include "experimental-features.hh" #include <map> #include <optional> @@ -43,8 +45,6 @@ struct Env }; -Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet()); - void copyContext(const Value & v, PathSet & context); @@ -81,7 +81,8 @@ public: sContentAddressed, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, - sDescription, sSelf, sEpsilon; + sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath, + sPrefix; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -92,7 +93,7 @@ public: mode. */ std::optional<PathSet> allowedPaths; - Value vEmptySet; + Bindings emptyBindings; /* Store used to materialise .drv files. */ const ref<Store> store; @@ -132,6 +133,9 @@ private: /* Cache used by prim_match(). */ std::shared_ptr<RegexCache> regexCache; + /* Allocation cache for GC'd Value objects. */ + std::shared_ptr<void *> valueAllocCache; + public: EvalState( @@ -141,15 +145,27 @@ public: ~EvalState(); void requireExperimentalFeatureOnEvaluation( - const std::string & feature, + const ExperimentalFeature &, const std::string_view fName, const Pos & pos ); - void addToSearchPath(const string & s); + void addToSearchPath(const std::string & s); SearchPath getSearchPath() { return searchPath; } + /* Allow access to a path. */ + void allowPath(const Path & path); + + /* Allow access to a store path. Note that this gets remapped to + the real store path if `store` is a chroot store. */ + void allowPath(const StorePath & storePath); + + /* Allow access to a store path and return it as a string. */ + void allowAndSetStorePathString(const StorePath & storePath, Value & v); + + /* Check whether access to a path is allowed and throw an error if + not. Otherwise return the canonicalised path. */ Path checkSourcePath(const Path & path); void checkURI(const std::string & uri); @@ -168,8 +184,8 @@ public: Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv); /* Parse a Nix expression from the specified string. */ - Expr * parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv); - Expr * parseExprFromString(std::string_view s, const Path & basePath); + Expr * parseExprFromString(std::string s, const Path & basePath, StaticEnv & staticEnv); + Expr * parseExprFromString(std::string s, const Path & basePath); Expr * parseStdin(); @@ -189,8 +205,8 @@ public: void resetFileCache(); /* Look up a file in the search path. */ - Path findFile(const string & path); - Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos); + Path findFile(const std::string_view path); + Path findFile(SearchPath & searchPath, const std::string_view path, const Pos & pos = noPos); /* If the specified search path element is a URI, download it. */ std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem); @@ -209,7 +225,10 @@ public: of the evaluation of the thunk. If `v' is a delayed function application, call the function and overwrite `v' with the result. Otherwise, this is a no-op. */ - inline void forceValue(Value & v, const Pos & pos = noPos); + inline void forceValue(Value & v, const Pos & pos); + + template <typename Callable> + inline void forceValue(Value & v, Callable getPos); /* Force a value, then recursively force list elements and attributes. */ @@ -219,37 +238,43 @@ public: NixInt forceInt(Value & v, const Pos & pos); NixFloat forceFloat(Value & v, const Pos & pos); bool forceBool(Value & v, const Pos & pos); - inline void forceAttrs(Value & v); - inline void forceAttrs(Value & v, const Pos & pos); - inline void forceList(Value & v); + + void forceAttrs(Value & v, const Pos & pos); + + template <typename Callable> + inline void forceAttrs(Value & v, Callable getPos); + inline void forceList(Value & v, const Pos & pos); void forceFunction(Value & v, const Pos & pos); // either lambda or primop - string forceString(Value & v, const Pos & pos = noPos); - string forceString(Value & v, PathSet & context, const Pos & pos = noPos); - string forceStringNoCtx(Value & v, const Pos & pos = noPos); + std::string_view forceString(Value & v, const Pos & pos = noPos); + std::string_view forceString(Value & v, PathSet & context, const Pos & pos = noPos); + std::string_view forceStringNoCtx(Value & v, const Pos & pos = noPos); /* Return true iff the value `v' denotes a derivation (i.e. a set with attribute `type = "derivation"'). */ bool isDerivation(Value & v); - std::optional<string> tryAttrsToString(const Pos & pos, Value & v, + std::optional<std::string> tryAttrsToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore = false, bool copyToStore = true); /* String coercion. Converts strings, paths and derivations to a string. If `coerceMore' is set, also converts nulls, integers, booleans and lists to a string. If `copyToStore' is set, referenced paths are copied to the Nix store as a side effect. */ - string coerceToString(const Pos & pos, Value & v, PathSet & context, + BackedStringView coerceToString(const Pos & pos, Value & v, PathSet & context, bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - string copyPathToStore(PathSet & context, const Path & path); + std::string copyPathToStore(PathSet & context, const Path & path); /* Path coercion. Converts strings, paths and derivations to a path. The result is guaranteed to be a canonicalised, absolute path. Nothing is copied to the store. */ Path coerceToPath(const Pos & pos, Value & v, PathSet & context); + /* Like coerceToPath, but the result must be a store path. */ + StorePath coerceToStorePath(const Pos & pos, Value & v, PathSet & context); + public: /* The base environment, containing the builtin functions and @@ -265,16 +290,18 @@ private: void createBaseEnv(); - Value * addConstant(const string & name, Value & v); + Value * addConstant(const std::string & name, Value & v); - Value * addPrimOp(const string & name, + void addConstant(const std::string & name, Value * v); + + Value * addPrimOp(const std::string & name, size_t arity, PrimOpFun primOp); Value * addPrimOp(PrimOp && primOp); public: - Value & getBuiltin(const string & name); + Value & getBuiltin(const std::string & name); struct Doc { @@ -295,8 +322,8 @@ private: friend struct ExprAttrs; friend struct ExprLet; - Expr * parse(const char * text, FileOrigin origin, const Path & path, - const Path & basePath, StaticEnv & staticEnv); + Expr * parse(char * text, size_t length, FileOrigin origin, const PathView path, + const PathView basePath, StaticEnv & staticEnv); public: @@ -306,8 +333,14 @@ public: bool isFunctor(Value & fun); - void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos); - void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos); + // FIXME: use std::span + void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos); + + void callFunction(Value & fun, Value & arg, Value & vRes, const Pos & pos) + { + Value * args[] = {&arg}; + callFunction(fun, 1, args, vRes, pos); + } /* Automatically call a function for which each argument has a default value or has a binding in the `args' map. */ @@ -318,12 +351,16 @@ public: Env & allocEnv(size_t size); Value * allocAttr(Value & vAttrs, const Symbol & name); - Value * allocAttr(Value & vAttrs, const std::string & name); + Value * allocAttr(Value & vAttrs, std::string_view name); Bindings * allocBindings(size_t capacity); + BindingsBuilder buildBindings(size_t capacity) + { + return BindingsBuilder(*this, allocBindings(capacity)); + } + void mkList(Value & v, size_t length); - void mkAttrs(Value & v, size_t capacity); void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, ptr<Pos> pos); @@ -332,7 +369,10 @@ public: /* Print statistics. */ void printStats(); - void realiseContext(const PathSet & context); + /* Realise the given context, and return a mapping from the placeholders + * used to construct the associated value to their final store path + */ + [[nodiscard]] StringMap realiseContext(const PathSet & context); private: @@ -373,16 +413,19 @@ private: friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v); + friend void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v); + + friend struct Value; }; /* Return a string representing the type of the value `v'. */ -string showType(ValueType type); -string showType(const Value & v); +std::string_view showType(ValueType type); +std::string showType(const Value & v); /* Decode a context string ‘!<name>!<path>’ into a pair <path, name>. */ -std::pair<string, string> decodeContext(std::string_view s); +std::pair<std::string, std::string> decodeContext(std::string_view s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 41b6f78ed..a811e59a1 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,4 +1,6 @@ #include "flake.hh" +#include "globals.hh" +#include "fetch-settings.hh" #include <nlohmann/json.hpp> @@ -37,11 +39,11 @@ void ConfigFile::apply() // FIXME: Move into libutil/config.cc. std::string valueS; - if (auto s = std::get_if<std::string>(&value)) + if (auto* s = std::get_if<std::string>(&value)) valueS = *s; - else if (auto n = std::get_if<int64_t>(&value)) - valueS = fmt("%d", n); - else if (auto b = std::get_if<Explicit<bool>>(&value)) + else if (auto* n = std::get_if<int64_t>(&value)) + valueS = fmt("%d", *n); + else if (auto* b = std::get_if<Explicit<bool>>(&value)) valueS = b->t ? "true" : "false"; else if (auto ss = std::get_if<std::vector<std::string>>(&value)) valueS = concatStringsSep(" ", *ss); // FIXME: evil @@ -52,21 +54,19 @@ void ConfigFile::apply() auto trustedList = readTrustedList(); bool trusted = false; - - if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) { + if (nix::fetchSettings.acceptFlakeConfig){ + trusted = true; + } else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) { trusted = *saved; + warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS); } else { // FIXME: filter ANSI escapes, newlines, \r, etc. - if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { - if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = false; - writeTrustedList(trustedList); - } - } else { - if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = trusted = true; - writeTrustedList(trustedList); - } + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') { + trusted = true; + } + if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?", trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') { + trustedList[name][valueS] = trusted; + writeTrustedList(trustedList); } } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 1a1fa6938..6a1aca40d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -6,6 +6,7 @@ #include "store-api.hh" #include "fetchers.hh" #include "finally.hh" +#include "fetch-settings.hh" namespace nix { @@ -64,8 +65,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( debug("got tree '%s' from '%s'", state.store->printStorePath(tree.storePath), lockedRef); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); @@ -90,11 +90,11 @@ static void expectType(EvalState & state, ValueType type, static std::map<FlakeId, FlakeInput> parseFlakeInputs( EvalState & state, Value * value, const Pos & pos, - const std::optional<Path> & baseDir); + const std::optional<Path> & baseDir, InputPath lockRootPath); static FlakeInput parseFlakeInput(EvalState & state, const std::string & inputName, Value * value, const Pos & pos, - const std::optional<Path> & baseDir) + const std::optional<Path> & baseDir, InputPath lockRootPath) { expectType(state, nAttrs, *value, pos); @@ -118,10 +118,12 @@ static FlakeInput parseFlakeInput(EvalState & state, expectType(state, nBool, *attr.value, *attr.pos); input.isFlake = attr.value->boolean; } else if (attr.name == sInputs) { - input.overrides = parseFlakeInputs(state, attr.value, *attr.pos, baseDir); + input.overrides = parseFlakeInputs(state, attr.value, *attr.pos, baseDir, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, *attr.pos); - input.follows = parseInputPath(attr.value->string.s); + auto follows(parseInputPath(attr.value->string.s)); + follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); + input.follows = follows; } else { switch (attr.value->type()) { case nString: @@ -156,7 +158,7 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos); if (url) - input.ref = parseFlakeRef(*url, baseDir, true); + input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); } if (!input.follows && !input.ref) @@ -167,7 +169,7 @@ static FlakeInput parseFlakeInput(EvalState & state, static std::map<FlakeId, FlakeInput> parseFlakeInputs( EvalState & state, Value * value, const Pos & pos, - const std::optional<Path> & baseDir) + const std::optional<Path> & baseDir, InputPath lockRootPath) { std::map<FlakeId, FlakeInput> inputs; @@ -179,7 +181,8 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs( inputAttr.name, inputAttr.value, *inputAttr.pos, - baseDir)); + baseDir, + lockRootPath)); } return inputs; @@ -189,14 +192,15 @@ static Flake getFlake( EvalState & state, const FlakeRef & originalRef, bool allowLookup, - FlakeCache & flakeCache) + FlakeCache & flakeCache, + InputPath lockRootPath) { auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, allowLookup, flakeCache); // Guard against symlink attacks. - auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir); - auto flakeFile = canonPath(flakeDir + "/flake.nix"); + auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true); + auto flakeFile = canonPath(flakeDir + "/flake.nix", true); if (!isInDir(flakeFile, sourceInfo.actualPath)) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", lockedRef, state.store->printStorePath(sourceInfo.storePath)); @@ -224,14 +228,14 @@ static Flake getFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos, flakeDir); + flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos, flakeDir, lockRootPath); auto sOutputs = state.symbols.create("outputs"); if (auto outputs = vInfo.attrs->get(sOutputs)) { expectType(state, nFunction, *outputs->value, *outputs->pos); - if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) { + if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) { for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { @@ -251,19 +255,24 @@ static Flake getFlake( for (auto & setting : *nixConfig->value->attrs) { forceTrivialValue(state, *setting.value, *setting.pos); if (setting.value->type() == nString) - flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)}); + flake.config.settings.insert({setting.name, std::string(state.forceStringNoCtx(*setting.value, *setting.pos))}); + else if (setting.value->type() == nPath) { + PathSet emptyContext = {}; + flake.config.settings.emplace( + setting.name, + state.coerceToString(*setting.pos, *setting.value, emptyContext, false, true, true) .toOwned()); + } else if (setting.value->type() == nInt) flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); else if (setting.value->type() == nBool) - flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); + flake.config.settings.insert({setting.name, Explicit<bool> { state.forceBool(*setting.value, *setting.pos) }}); else if (setting.value->type() == nList) { std::vector<std::string> ss; - for (unsigned int n = 0; n < setting.value->listSize(); ++n) { - auto elem = setting.value->listElems()[n]; + for (auto elem : setting.value->listItems()) { if (elem->type() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", setting.name, showType(*setting.value)); - ss.push_back(state.forceStringNoCtx(*elem, *setting.pos)); + ss.emplace_back(state.forceStringNoCtx(*elem, *setting.pos)); } flake.config.settings.insert({setting.name, ss}); } @@ -285,6 +294,11 @@ static Flake getFlake( return flake; } +Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup, FlakeCache & flakeCache) +{ + return getFlake(state, originalRef, allowLookup, flakeCache, {}); +} + Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) { FlakeCache flakeCache; @@ -298,17 +312,17 @@ LockedFlake lockFlake( const FlakeRef & topRef, const LockFlags & lockFlags) { - settings.requireExperimentalFeature("flakes"); + settings.requireExperimentalFeature(Xp::Flakes); FlakeCache flakeCache; - auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + auto useRegistries = lockFlags.useRegistries.value_or(fetchSettings.useRegistries); auto flake = getFlake(state, topRef, useRegistries, flakeCache); if (lockFlags.applyNixConfig) { flake.config.apply(); - // FIXME: send new config to the daemon. + state.store->setOptions(); } try { @@ -330,23 +344,14 @@ LockedFlake lockFlake( std::vector<FlakeRef> parents; - struct LockParent { - /* The path to this parent. */ - InputPath path; - - /* Whether we are currently inside a top-level lockfile - (inputs absolute) or subordinate lockfile (inputs - relative). */ - bool absolute; - }; - std::function<void( const FlakeInputs & flakeInputs, std::shared_ptr<Node> node, const InputPath & inputPathPrefix, std::shared_ptr<const Node> oldNode, - const LockParent & parent, - const Path & parentPath)> + const InputPath & lockRootPath, + const Path & parentPath, + bool trustLock)> computeLocks; computeLocks = [&]( @@ -354,8 +359,9 @@ LockedFlake lockFlake( std::shared_ptr<Node> node, const InputPath & inputPathPrefix, std::shared_ptr<const Node> oldNode, - const LockParent & parent, - const Path & parentPath) + const InputPath & lockRootPath, + const Path & parentPath, + bool trustLock) { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); @@ -398,17 +404,7 @@ LockedFlake lockFlake( if (input.follows) { InputPath target; - if (parent.absolute && !hasOverride) { - target = *input.follows; - } else { - if (hasOverride) { - target = inputPathPrefix; - target.pop_back(); - } else - target = parent.path; - - for (auto & i : *input.follows) target.push_back(i); - } + target.insert(target.end(), input.follows->begin(), input.follows->end()); debug("input '%s' follows '%s'", inputPathS, printInputPath(target)); node->inputs.insert_or_assign(id, target); @@ -447,22 +443,18 @@ LockedFlake lockFlake( update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); - auto hasChildUpdate = + auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->size() > inputPath.size() && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); - if (hasChildUpdate) { - auto inputFlake = getFlake( - state, oldLock->lockedRef, false, flakeCache); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, parent, parentPath); - } else { + FlakeInputs fakeInputs; + + if (!mustRefetch) { /* No need to fetch this flake, we can be lazy. However there may be new overrides on the inputs of this flake, so we need to check those. */ - FlakeInputs fakeInputs; - for (auto & i : oldLock->inputs) { if (auto lockedNode = std::get_if<0>(&i.second)) { fakeInputs.emplace(i.first, FlakeInput { @@ -470,21 +462,47 @@ LockedFlake lockFlake( .isFlake = (*lockedNode)->isFlake, }); } else if (auto follows = std::get_if<1>(&i.second)) { + if (! trustLock) { + // It is possible that the flake has changed, + // so we must confirm all the follows that are in the lockfile are also in the flake. + auto overridePath(inputPath); + overridePath.push_back(i.first); + auto o = overrides.find(overridePath); + // If the override disappeared, we have to refetch the flake, + // since some of the inputs may not be present in the lockfile. + if (o == overrides.end()) { + mustRefetch = true; + // There's no point populating the rest of the fake inputs, + // since we'll refetch the flake anyways. + break; + } + } + auto absoluteFollows(lockRootPath); + absoluteFollows.insert(absoluteFollows.end(), follows->begin(), follows->end()); fakeInputs.emplace(i.first, FlakeInput { - .follows = *follows, + .follows = absoluteFollows, }); } } - - computeLocks(fakeInputs, childNode, inputPath, oldLock, parent, parentPath); } + auto localPath(parentPath); + // If this input is a path, recurse it down. + // This allows us to resolve path inputs relative to the current flake. + if ((*input.ref).input.getType() == "path") + localPath = absPath(*input.ref->input.getSourcePath(), parentPath); + computeLocks( + mustRefetch + ? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs + : fakeInputs, + childNode, inputPath, oldLock, lockRootPath, parentPath, !mustRefetch); + } else { /* We need to create a new lock file entry. So fetch this input. */ debug("creating new input '%s'", inputPathS); - if (!lockFlags.allowMutable && !input.ref->input.isImmutable()) + if (!lockFlags.allowMutable && !input.ref->input.isLocked()) throw Error("cannot update flake input '%s' in pure mode", inputPathS); if (input.isFlake) { @@ -496,7 +514,7 @@ LockedFlake lockFlake( if (localRef.input.getType() == "path") localPath = absPath(*input.ref->input.getSourcePath(), parentPath); - auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache); + auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache, inputPath); /* Note: in case of an --override-input, we use the *original* ref (input2.ref) for the @@ -517,13 +535,6 @@ LockedFlake lockFlake( parents.push_back(*input.ref); Finally cleanup([&]() { parents.pop_back(); }); - // Follows paths from existing inputs in the top-level lockfile are absolute, - // whereas paths in subordinate lockfiles are relative to those lockfiles. - LockParent newParent { - .path = inputPath, - .absolute = oldLock ? true : false - }; - /* Recursively process the inputs of this flake. Also, unless we already have this flake in the top-level lock file, use this flake's @@ -534,7 +545,7 @@ LockedFlake lockFlake( ? std::dynamic_pointer_cast<const Node>(oldLock) : LockFile::read( inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root, - newParent, localPath); + oldLock ? lockRootPath : inputPath, localPath, false); } else { @@ -552,17 +563,12 @@ LockedFlake lockFlake( } }; - LockParent parent { - .path = {}, - .absolute = true - }; - // Bring in the current ref for relative path resolution if we have it - auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir); + auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); computeLocks( flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath); + lockFlags.recreateLockFile ? nullptr : oldLockFile.root, {}, parentPath, false); for (auto & i : lockFlags.inputOverrides) if (!overridesUsed.count(i.first)) @@ -586,7 +592,7 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (auto sourcePath = topRef.input.getSourcePath()) { if (!newLockFile.isImmutable()) { - if (settings.warnDirty) + if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has a mutable input", topRef); } else { if (!lockFlags.updateLockFile) @@ -609,12 +615,24 @@ LockedFlake lockFlake( newLockFile.write(path); + std::optional<std::string> commitMessage = std::nullopt; + if (lockFlags.commitLockFile) { + std::string cm; + + cm = fetchSettings.commitLockFileSummary.get(); + + if (cm == "") { + cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + } + + cm += "\n\nFlake lock file updates:\n\n"; + cm += filterANSIEscapes(diff, true); + commitMessage = cm; + } + topRef.input.markChangedFile( (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", - lockFlags.commitLockFile - ? std::optional<std::string>(fmt("%s: %s\n\nFlake lock file changes:\n\n%s", - relPath, lockFileExists ? "Update" : "Add", filterANSIEscapes(diff, true))) - : std::nullopt); + commitMessage); /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could @@ -633,7 +651,7 @@ LockedFlake lockFlake( now. Corner case: we could have reverted from a dirty to a clean tree! */ if (flake.lockedRef.input == prevLockedRef.input - && !flake.lockedRef.input.isImmutable()) + && !flake.lockedRef.input.isLocked()) throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); } } else @@ -662,7 +680,7 @@ void callFlake(EvalState & state, auto vTmp1 = state.allocValue(); auto vTmp2 = state.allocValue(); - mkString(*vLocks, lockedFlake.lockFile.to_string()); + vLocks->mkString(lockedFlake.lockFile.to_string()); emitTreeAttrs( state, @@ -672,7 +690,7 @@ void callFlake(EvalState & state, false, lockedFlake.flake.forceDirty); - mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); + vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); if (!state.vCallFlake) { state.vCallFlake = allocRootValue(state.allocValue()); @@ -688,18 +706,18 @@ void callFlake(EvalState & state, static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.requireExperimentalFeatureOnEvaluation("flakes", "builtins.getFlake", pos); + state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos); - auto flakeRefS = state.forceStringNoCtx(*args[0], pos); + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos)); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (evalSettings.pureEval && !flakeRef.input.isImmutable()) - throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); + if (evalSettings.pureEval && !flakeRef.input.isLocked()) + throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); callFlake(state, lockFlake(state, flakeRef, LockFlags { .updateLockFile = false, - .useRegistries = !evalSettings.pureEval && settings.useRegistries, + .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, .allowMutable = !evalSettings.pureEval, }), v); diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 29128d789..c1eae413f 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -48,9 +48,12 @@ FlakeRef FlakeRef::resolve(ref<Store> store) const } FlakeRef parseFlakeRef( - const std::string & url, const std::optional<Path> & baseDir, bool allowMissing) + const std::string & url, + const std::optional<Path> & baseDir, + bool allowMissing, + bool isFlake) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; @@ -67,7 +70,10 @@ std::optional<FlakeRef> maybeParseFlakeRef( } std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( - const std::string & url, const std::optional<Path> & baseDir, bool allowMissing) + const std::string & url, + const std::optional<Path> & baseDir, + bool allowMissing, + bool isFlake) { using namespace fetchers; @@ -92,7 +98,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ .url = url, - .base = "flake:" + std::string(match[1]), + .base = "flake:" + match.str(1), .scheme = "flake", .authority = "", .path = match[1], @@ -100,58 +106,83 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( return std::make_pair( FlakeRef(Input::fromURL(parsedURL), ""), - percentDecode(std::string(match[6]))); + percentDecode(match.str(6))); } else if (std::regex_match(url, match, pathUrlRegex)) { std::string path = match[1]; - std::string fragment = percentDecode(std::string(match[3])); + std::string fragment = percentDecode(match.str(3)); if (baseDir) { /* Check if 'url' is a path (either absolute or relative to 'baseDir'). If so, search upward to the root of the repo (i.e. the directory containing .git). */ - path = absPath(path, baseDir, true); + path = absPath(path, baseDir); + + if (isFlake) { + + if (!allowMissing && !pathExists(path + "/flake.nix")){ + notice("path '%s' does not contain a 'flake.nix', searching up",path); + + // Save device to detect filesystem boundary + dev_t device = lstat(path).st_dev; + bool found = false; + while (path != "/") { + if (pathExists(path + "/flake.nix")) { + found = true; + break; + } else if (pathExists(path + "/.git")) + throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); + else { + if (lstat(path).st_dev != device) + throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + } + path = dirOf(path); + } + if (!found) + throw BadURL("could not find a flake.nix file"); + } - if (!S_ISDIR(lstat(path).st_mode)) - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); - if (!allowMissing && !pathExists(path + "/flake.nix")) - throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); - auto flakeRoot = path; - std::string subdir; + auto flakeRoot = path; + std::string subdir; - while (flakeRoot != "/") { - if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; - auto parsedURL = ParsedURL{ - .url = base, // FIXME - .base = base, - .scheme = "git+file", - .authority = "", - .path = flakeRoot, - .query = decodeQuery(match[2]), - }; + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = decodeQuery(match[2]), + }; - if (subdir != "") { - if (parsedURL.query.count("dir")) - throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); - parsedURL.query.insert_or_assign("dir", subdir); - } + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } - if (pathExists(flakeRoot + "/.git/shallow")) - parsedURL.query.insert_or_assign("shallow", "1"); + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), - fragment); - } + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + fragment); + } - subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); - flakeRoot = dirOf(flakeRoot); + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + } } } else { diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 0292eb210..1fddfd9a0 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -62,13 +62,19 @@ struct FlakeRef std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); FlakeRef parseFlakeRef( - const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false); + const std::string & url, + const std::optional<Path> & baseDir = {}, + bool allowMissing = false, + bool isFlake = true); std::optional<FlakeRef> maybeParseFlake( const std::string & url, const std::optional<Path> & baseDir = {}); std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( - const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false); + const std::string & url, + const std::optional<Path> & baseDir = {}, + bool allowMissing = false, + bool isFlake = true); std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( const std::string & url, const std::optional<Path> & baseDir = {}); diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index fda340789..60b52d578 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -35,7 +35,7 @@ LockedNode::LockedNode(const nlohmann::json & json) , originalRef(getFlakeRef(json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { - if (!lockedRef.input.isImmutable()) + if (!lockedRef.input.isLocked()) throw Error("lockfile contains mutable lock '%s'", fetchers::attrsToJSON(lockedRef.input.toAttrs())); } @@ -220,7 +220,7 @@ bool LockFile::isImmutable() const for (auto & i : nodes) { if (i == root) continue; auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i); - if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false; + if (lockedNode && !lockedNode->lockedRef.input.isLocked()) return false; } return true; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index f774e6493..7630c5ff4 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -11,8 +11,8 @@ namespace nix { -DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) - : state(&state), attrs(attrs), attrPath(attrPath) +DrvInfo::DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs) + : state(&state), attrs(attrs), attrPath(std::move(attrPath)) { } @@ -22,7 +22,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat { auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs); - this->drvPath = store->printStorePath(drvPath); + this->drvPath = drvPath; auto drv = store->derivationFromPath(drvPath); @@ -41,13 +41,11 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); auto & [outputName, output] = *i; - auto optStorePath = output.path(*store, drv.name, outputName); - if (optStorePath) - outPath = store->printStorePath(*optStorePath); + outPath = {output.path(*store, drv.name, outputName)}; } -string DrvInfo::queryName() const +std::string DrvInfo::queryName() const { if (name == "" && attrs) { auto i = attrs->find(state->sName); @@ -58,7 +56,7 @@ string DrvInfo::queryName() const } -string DrvInfo::querySystem() const +std::string DrvInfo::querySystem() const { if (system == "" && attrs) { auto i = attrs->find(state->sSystem); @@ -68,24 +66,35 @@ string DrvInfo::querySystem() const } -string DrvInfo::queryDrvPath() const +std::optional<StorePath> DrvInfo::queryDrvPath() const { - if (drvPath == "" && attrs) { + if (!drvPath && attrs) { Bindings::iterator i = attrs->find(state->sDrvPath); PathSet context; - drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; + if (i == attrs->end()) + drvPath = {std::nullopt}; + else + drvPath = {state->coerceToStorePath(*i->pos, *i->value, context)}; } - return drvPath; + return drvPath.value_or(std::nullopt); } -string DrvInfo::queryOutPath() const +StorePath DrvInfo::requireDrvPath() const +{ + if (auto drvPath = queryDrvPath()) + return *drvPath; + throw Error("derivation does not contain a 'drvPath' attribute"); +} + + +StorePath DrvInfo::queryOutPath() const { if (!outPath && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); PathSet context; if (i != attrs->end()) - outPath = state->coerceToPath(*i->pos, *i->value, context); + outPath = state->coerceToStorePath(*i->pos, *i->value, context); } if (!outPath) throw UnimplementedError("CA derivations are not yet supported"); @@ -102,21 +111,21 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) state->forceList(*i->value, *i->pos); /* For each output... */ - for (unsigned int j = 0; j < i->value->listSize(); ++j) { + for (auto elem : i->value->listItems()) { /* Evaluate the corresponding set. */ - string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos); + std::string name(state->forceStringNoCtx(*elem, *i->pos)); Bindings::iterator out = attrs->find(state->symbols.create(name)); if (out == attrs->end()) continue; // FIXME: throw error? - state->forceAttrs(*out->value); + state->forceAttrs(*out->value, *i->pos); /* And evaluate its ‘outPath’ attribute. */ Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? PathSet context; - outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context); + outputs.emplace(name, state->coerceToStorePath(*outPath->pos, *outPath->value, context)); } } else - outputs["out"] = queryOutPath(); + outputs.emplace("out", queryOutPath()); } if (!onlyOutputsToInstall || !attrs) return outputs; @@ -128,9 +137,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) throw errMsg; Outputs result; - for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->type() != nString) throw errMsg; - auto out = outputs.find((*i)->string.s); + for (auto elem : outTI->listItems()) { + if (elem->type() != nString) throw errMsg; + auto out = outputs.find(elem->string.s); if (out == outputs.end()) throw errMsg; result.insert(*out); } @@ -138,7 +147,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) } -string DrvInfo::queryOutputName() const +std::string DrvInfo::queryOutputName() const { if (outputName == "" && attrs) { Bindings::iterator i = attrs->find(state->sOutputName); @@ -172,10 +181,10 @@ StringSet DrvInfo::queryMetaNames() bool DrvInfo::checkMeta(Value & v) { - state->forceValue(v); + state->forceValue(v, [&]() { return v.determinePos(noPos); }); if (v.type() == nList) { - for (unsigned int n = 0; n < v.listSize(); ++n) - if (!checkMeta(*v.listElems()[n])) return false; + for (auto elem : v.listItems()) + if (!checkMeta(*elem)) return false; return true; } else if (v.type() == nAttrs) { @@ -190,7 +199,7 @@ bool DrvInfo::checkMeta(Value & v) } -Value * DrvInfo::queryMeta(const string & name) +Value * DrvInfo::queryMeta(const std::string & name) { if (!getMeta()) return 0; Bindings::iterator a = meta->find(state->symbols.create(name)); @@ -199,7 +208,7 @@ Value * DrvInfo::queryMeta(const string & name) } -string DrvInfo::queryMetaString(const string & name) +std::string DrvInfo::queryMetaString(const std::string & name) { Value * v = queryMeta(name); if (!v || v->type() != nString) return ""; @@ -207,7 +216,7 @@ string DrvInfo::queryMetaString(const string & name) } -NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) +NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; @@ -221,7 +230,7 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) return def; } -NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) +NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def) { Value * v = queryMeta(name); if (!v) return def; @@ -236,7 +245,7 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) } -bool DrvInfo::queryMetaBool(const string & name, bool def) +bool DrvInfo::queryMetaBool(const std::string & name, bool def) { Value * v = queryMeta(name); if (!v) return def; @@ -251,23 +260,22 @@ bool DrvInfo::queryMetaBool(const string & name, bool def) } -void DrvInfo::setMeta(const string & name, Value * v) +void DrvInfo::setMeta(const std::string & name, Value * v) { getMeta(); - Bindings * old = meta; - meta = state->allocBindings(1 + (old ? old->size() : 0)); + auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0)); Symbol sym = state->symbols.create(name); - if (old) - for (auto i : *old) + if (meta) + for (auto i : *meta) if (i.name != sym) - meta->push_back(i); - if (v) meta->push_back(Attr(sym, v)); - meta->sort(); + attrs.insert(i); + if (v) attrs.insert(sym, v); + meta = attrs.finish(); } /* Cache for already considered attrsets. */ -typedef set<Bindings *> Done; +typedef std::set<Bindings *> Done; /* Evaluate value `v'. If it evaluates to a set of type `derivation', @@ -275,11 +283,11 @@ typedef set<Bindings *> Done; The result boolean indicates whether it makes sense for the caller to recursively search for derivations in `v'. */ static bool getDerivation(EvalState & state, Value & v, - const string & attrPath, DrvInfos & drvs, Done & done, + const std::string & attrPath, DrvInfos & drvs, Done & done, bool ignoreAssertionFailures) { try { - state.forceValue(v); + state.forceValue(v, [&]() { return v.determinePos(noPos); }); if (!state.isDerivation(v)) return true; /* Remove spurious duplicates (e.g., a set like `rec { x = @@ -312,7 +320,7 @@ std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, } -static string addToPath(const string & s1, const string & s2) +static std::string addToPath(const std::string & s1, const std::string & s2) { return s1.empty() ? s2 : s1 + "." + s2; } @@ -322,7 +330,7 @@ static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); static void getDerivations(EvalState & state, Value & vIn, - const string & pathPrefix, Bindings & autoArgs, + const std::string & pathPrefix, Bindings & autoArgs, DrvInfos & drvs, Done & done, bool ignoreAssertionFailures) { @@ -347,7 +355,7 @@ static void getDerivations(EvalState & state, Value & vIn, debug("evaluating attribute '%1%'", i->name); if (!std::regex_match(std::string(i->name), attrRegex)) continue; - string pathPrefix2 = addToPath(pathPrefix, i->name); + std::string pathPrefix2 = addToPath(pathPrefix, i->name); if (combineChannels) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { @@ -364,10 +372,10 @@ static void getDerivations(EvalState & state, Value & vIn, } else if (v.type() == nList) { - for (unsigned int n = 0; n < v.listSize(); ++n) { - string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); - if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) - getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + for (auto [n, elem] : enumerate(v.listItems())) { + std::string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n)); + if (getDerivation(state, *elem, pathPrefix2, drvs, done, ignoreAssertionFailures)) + getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } } @@ -375,7 +383,7 @@ static void getDerivations(EvalState & state, Value & vIn, } -void getDerivations(EvalState & state, Value & v, const string & pathPrefix, +void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix, Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures) { Done done; diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 29bb6a660..3ca6f1fca 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -1,6 +1,7 @@ #pragma once #include "eval.hh" +#include "path.hh" #include <string> #include <map> @@ -12,16 +13,16 @@ namespace nix { struct DrvInfo { public: - typedef std::map<string, Path> Outputs; + typedef std::map<std::string, StorePath> Outputs; private: EvalState * state; - mutable string name; - mutable string system; - mutable string drvPath; - mutable std::optional<string> outPath; - mutable string outputName; + mutable std::string name; + mutable std::string system; + mutable std::optional<std::optional<StorePath>> drvPath; + mutable std::optional<StorePath> outPath; + mutable std::string outputName; Outputs outputs; bool failed = false; // set if we get an AssertionError @@ -33,36 +34,37 @@ private: bool checkMeta(Value & v); public: - string attrPath; /* path towards the derivation */ + std::string attrPath; /* path towards the derivation */ DrvInfo(EvalState & state) : state(&state) { }; - DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs); + DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs); DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs); - string queryName() const; - string querySystem() const; - string queryDrvPath() const; - string queryOutPath() const; - string queryOutputName() const; + std::string queryName() const; + std::string querySystem() const; + std::optional<StorePath> queryDrvPath() const; + StorePath requireDrvPath() const; + StorePath queryOutPath() const; + std::string queryOutputName() const; /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ Outputs queryOutputs(bool onlyOutputsToInstall = false); StringSet queryMetaNames(); - Value * queryMeta(const string & name); - string queryMetaString(const string & name); - NixInt queryMetaInt(const string & name, NixInt def); - NixFloat queryMetaFloat(const string & name, NixFloat def); - bool queryMetaBool(const string & name, bool def); - void setMeta(const string & name, Value * v); + Value * queryMeta(const std::string & name); + std::string queryMetaString(const std::string & name); + NixInt queryMetaInt(const std::string & name, NixInt def); + NixFloat queryMetaFloat(const std::string & name, NixFloat def); + bool queryMetaBool(const std::string & name, bool def); + void setMeta(const std::string & name, Value * v); /* MetaInfo queryMetaInfo(EvalState & state) const; MetaValue queryMetaInfo(EvalState & state, const string & name) const; */ - void setName(const string & s) { name = s; } - void setDrvPath(const string & s) { drvPath = s; } - void setOutPath(const string & s) { outPath = s; } + void setName(const std::string & s) { name = s; } + void setDrvPath(StorePath path) { drvPath = {{std::move(path)}}; } + void setOutPath(StorePath path) { outPath = {{std::move(path)}}; } void setFailed() { failed = true; }; bool hasFailed() { return failed; }; @@ -70,9 +72,9 @@ public: #if HAVE_BOEHMGC -typedef list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos; +typedef std::list<DrvInfo, traceable_allocator<DrvInfo> > DrvInfos; #else -typedef list<DrvInfo> DrvInfos; +typedef std::list<DrvInfo> DrvInfos; #endif @@ -81,7 +83,7 @@ typedef list<DrvInfo> DrvInfos; std::optional<DrvInfo> getDerivation(EvalState & state, Value & v, bool ignoreAssertionFailures); -void getDerivations(EvalState & state, Value & v, const string & pathPrefix, +void getDerivations(EvalState & state, Value & v, const std::string & pathPrefix, Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures); diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 9ca5ac86d..99a475ff9 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -37,10 +37,10 @@ class JSONSax : nlohmann::json_sax<json> { ValueMap attrs; std::unique_ptr<JSONState> resolve(EvalState & state) override { - Value & v = parent->value(state); - state.mkAttrs(v, attrs.size()); + auto attrs2 = state.buildBindings(attrs.size()); for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second)); + attrs2.insert(i.first, i.second); + parent->value(state).mkAttrs(attrs2.alreadySorted()); return std::move(parent); } void add() override { v = nullptr; } @@ -76,45 +76,51 @@ class JSONSax : nlohmann::json_sax<json> { EvalState & state; std::unique_ptr<JSONState> rs; - template<typename T, typename... Args> inline bool handle_value(T f, Args... args) - { - f(rs->value(state), args...); - rs->add(); - return true; - } - public: JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; bool null() { - return handle_value(mkNull); + rs->value(state).mkNull(); + rs->add(); + return true; } bool boolean(bool val) { - return handle_value(mkBool, val); + rs->value(state).mkBool(val); + rs->add(); + return true; } bool number_integer(number_integer_t val) { - return handle_value(mkInt, val); + rs->value(state).mkInt(val); + rs->add(); + return true; } bool number_unsigned(number_unsigned_t val) { - return handle_value(mkInt, val); + rs->value(state).mkInt(val); + rs->add(); + return true; } bool number_float(number_float_t val, const string_t & s) { - return handle_value(mkFloat, val); + rs->value(state).mkFloat(val); + rs->add(); + return true; } bool string(string_t & val) { - return handle_value<void(Value&, const char*)>(mkString, val.c_str()); + rs->value(state).mkString(val); + rs->add(); + return true; } + #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 bool binary(binary_t&) { @@ -157,7 +163,7 @@ public: } }; -void parseJSON(EvalState & state, const string & s_, Value & v) +void parseJSON(EvalState & state, const std::string_view & s_, Value & v) { JSONSax parser(state, v); bool res = json::sax_parse(s_, &parser); diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh index 3b0fdae11..84bec4eba 100644 --- a/src/libexpr/json-to-value.hh +++ b/src/libexpr/json-to-value.hh @@ -8,6 +8,6 @@ namespace nix { MakeError(JSONParseError, EvalError); -void parseJSON(EvalState & state, const string & s, Value & v); +void parseJSON(EvalState & state, const std::string_view & s, Value & v); } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 51593eccd..e276b0467 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -64,28 +64,32 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) } -static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) +// we make use of the fact that the parser receives a private copy of the input +// string and can munge around in it. +static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) { - string t; - t.reserve(length); + char * result = s; + char * t = s; char c; + // the input string is terminated with *two* NULs, so we can safely take + // *one* character after the one being checked against. while ((c = *s++)) { if (c == '\\') { - assert(*s); c = *s++; - if (c == 'n') t += '\n'; - else if (c == 'r') t += '\r'; - else if (c == 't') t += '\t'; - else t += c; + if (c == 'n') *t = '\n'; + else if (c == 'r') *t = '\r'; + else if (c == 't') *t = '\t'; + else *t = c; } else if (c == '\r') { /* Normalise CR and CR/LF into LF. */ - t += '\n'; + *t = '\n'; if (*s == '\n') s++; /* cr/lf */ } - else t += c; + else *t = c; + t++; } - return new ExprString(symbols.create(t)); + return {result, size_t(t - result)}; } @@ -138,7 +142,7 @@ or { return OR_KW; } \/\/ { return UPDATE; } \+\+ { return CONCAT; } -{ID} { yylval->id = strdup(yytext); return ID; } +{ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; } {INT} { errno = 0; try { yylval->n = boost::lexical_cast<int64_t>(yytext); @@ -172,7 +176,7 @@ or { return OR_KW; } /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ - yylval->e = unescapeStr(data->symbols, yytext, yyleng); + yylval->str = unescapeStr(data->symbols, yytext, yyleng); return STR; } <STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -187,26 +191,26 @@ or { return OR_KW; } \'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } <IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { - yylval->e = new ExprIndStr(yytext); + yylval->str = {yytext, (size_t) yyleng, true}; return IND_STR; } <IND_STRING>\'\'\$ | <IND_STRING>\$ { - yylval->e = new ExprIndStr("$"); + yylval->str = {"$", 1}; return IND_STR; } <IND_STRING>\'\'\' { - yylval->e = new ExprIndStr("''"); + yylval->str = {"''", 2}; return IND_STR; } <IND_STRING>\'\'\\{ANY} { - yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); + yylval->str = unescapeStr(data->symbols, yytext + 2, yyleng - 2); return IND_STR; } <IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } <IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; } <IND_STRING>\' { - yylval->e = new ExprIndStr("'"); + yylval->str = {"'", 1}; return IND_STR; } @@ -220,14 +224,14 @@ or { return OR_KW; } <PATH_START>{PATH_SEG} { POP_STATE(); PUSH_STATE(INPATH_SLASH); - yylval->path = strdup(yytext); + yylval->path = {yytext, (size_t) yyleng}; return PATH; } <PATH_START>{HPATH_START} { POP_STATE(); PUSH_STATE(INPATH_SLASH); - yylval->path = strdup(yytext); + yylval->path = {yytext, (size_t) yyleng}; return HPATH; } @@ -236,7 +240,7 @@ or { return OR_KW; } PUSH_STATE(INPATH_SLASH); else PUSH_STATE(INPATH); - yylval->path = strdup(yytext); + yylval->path = {yytext, (size_t) yyleng}; return PATH; } {HPATH} { @@ -244,7 +248,7 @@ or { return OR_KW; } PUSH_STATE(INPATH_SLASH); else PUSH_STATE(INPATH); - yylval->path = strdup(yytext); + yylval->path = {yytext, (size_t) yyleng}; return HPATH; } @@ -260,7 +264,7 @@ or { return OR_KW; } PUSH_STATE(INPATH_SLASH); else PUSH_STATE(INPATH); - yylval->e = new ExprString(data->symbols.create(string(yytext))); + yylval->str = {yytext, (size_t) yyleng}; return STR; } <INPATH>{ANY} | @@ -279,8 +283,8 @@ or { return OR_KW; } throw ParseError("path has a trailing slash"); } -{SPATH} { yylval->path = strdup(yytext); return SPATH; } -{URI} { yylval->uri = strdup(yytext); return URI; } +{SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; } +{URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; } [ \t\r\n]+ /* eat up whitespace */ \#[^\r\n]* /* single-line comments */ diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 492b819e7..a2def65a6 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -16,10 +16,10 @@ std::ostream & operator << (std::ostream & str, const Expr & e) return str; } -static void showString(std::ostream & str, const string & s) +static void showString(std::ostream & str, std::string_view s) { str << '"'; - for (auto c : (string) s) + for (auto c : s) if (c == '"' || c == '\\' || c == '$') str << "\\" << c; else if (c == '\n') str << "\\n"; else if (c == '\r') str << "\\r"; @@ -28,7 +28,7 @@ static void showString(std::ostream & str, const string & s) str << '"'; } -static void showId(std::ostream & str, const string & s) +static void showId(std::ostream & str, std::string_view s) { if (s.empty()) str << "\"\""; @@ -103,11 +103,18 @@ void ExprAttrs::show(std::ostream & str) const { if (recursive) str << "rec "; str << "{ "; - for (auto & i : attrs) - if (i.second.inherited) - str << "inherit " << i.first << " " << "; "; + typedef const decltype(attrs)::value_type * Attr; + std::vector<Attr> sorted; + for (auto & i : attrs) sorted.push_back(&i); + std::sort(sorted.begin(), sorted.end(), [](Attr a, Attr b) { + return (const std::string &) a->first < (const std::string &) b->first; + }); + for (auto & i : sorted) { + if (i->second.inherited) + str << "inherit " << i->first << " " << "; "; else - str << i.first << " = " << *i.second.e << "; "; + str << i->first << " = " << *i->second.e << "; "; + } for (auto & i : dynamicAttrs) str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; "; str << "}"; @@ -124,7 +131,7 @@ void ExprList::show(std::ostream & str) const void ExprLambda::show(std::ostream & str) const { str << "("; - if (matchAttrs) { + if (hasFormals()) { str << "{ "; bool first = true; for (auto & i : formals->formals) { @@ -143,6 +150,16 @@ void ExprLambda::show(std::ostream & str) const str << ": " << *body << ")"; } +void ExprCall::show(std::ostream & str) const +{ + str << '(' << *fun; + for (auto e : args) { + str << ' '; + str << *e; + } + str << ')'; +} + void ExprLet::show(std::ostream & str) const { str << "(let "; @@ -181,7 +198,7 @@ void ExprConcatStrings::show(std::ostream & str) const str << "("; for (auto & i : *es) { if (first) first = false; else str << " + "; - str << *i; + str << *i.second; } str << ")"; } @@ -201,7 +218,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos) auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%"); switch (pos.origin) { case foFile: - f % (string) pos.file; + f % (const std::string &) pos.file; break; case foStdin: case foString: @@ -217,7 +234,7 @@ std::ostream & operator << (std::ostream & str, const Pos & pos) } -string showAttrPath(const AttrPath & attrPath) +std::string showAttrPath(const AttrPath & attrPath) { std::ostringstream out; bool first = true; @@ -263,13 +280,13 @@ void ExprVar::bindVars(const StaticEnv & env) /* Check whether the variable appears in the environment. If so, set its level and displacement. */ const StaticEnv * curEnv; - unsigned int level; + Level level; int withLevel = -1; for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { if (curEnv->isWith) { if (withLevel == -1) withLevel = level; } else { - StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); + auto i = curEnv->find(name); if (i != curEnv->vars.end()) { fromWith = false; this->level = level; @@ -311,14 +328,16 @@ void ExprOpHasAttr::bindVars(const StaticEnv & env) void ExprAttrs::bindVars(const StaticEnv & env) { const StaticEnv * dynamicEnv = &env; - StaticEnv newEnv(false, &env); + StaticEnv newEnv(false, &env, recursive ? attrs.size() : 0); if (recursive) { dynamicEnv = &newEnv; - unsigned int displ = 0; + Displacement displ = 0; for (auto & i : attrs) - newEnv.vars[i.first] = i.second.displ = displ++; + newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + + // No need to sort newEnv since attrs is in sorted order. for (auto & i : attrs) i.second.e->bindVars(i.second.inherited ? env : newEnv); @@ -342,15 +361,20 @@ void ExprList::bindVars(const StaticEnv & env) void ExprLambda::bindVars(const StaticEnv & env) { - StaticEnv newEnv(false, &env); + StaticEnv newEnv( + false, &env, + (hasFormals() ? formals->formals.size() : 0) + + (arg.empty() ? 0 : 1)); - unsigned int displ = 0; + Displacement displ = 0; - if (!arg.empty()) newEnv.vars[arg] = displ++; + if (!arg.empty()) newEnv.vars.emplace_back(arg, displ++); - if (matchAttrs) { + if (hasFormals()) { for (auto & i : formals->formals) - newEnv.vars[i.name] = displ++; + newEnv.vars.emplace_back(i.name, displ++); + + newEnv.sort(); for (auto & i : formals->formals) if (i.def) i.def->bindVars(newEnv); @@ -359,13 +383,22 @@ void ExprLambda::bindVars(const StaticEnv & env) body->bindVars(newEnv); } +void ExprCall::bindVars(const StaticEnv & env) +{ + fun->bindVars(env); + for (auto e : args) + e->bindVars(env); +} + void ExprLet::bindVars(const StaticEnv & env) { - StaticEnv newEnv(false, &env); + StaticEnv newEnv(false, &env, attrs->attrs.size()); - unsigned int displ = 0; + Displacement displ = 0; for (auto & i : attrs->attrs) - newEnv.vars[i.first] = i.second.displ = displ++; + newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + + // No need to sort newEnv since attrs->attrs is in sorted order. for (auto & i : attrs->attrs) i.second.e->bindVars(i.second.inherited ? env : newEnv); @@ -379,7 +412,7 @@ void ExprWith::bindVars(const StaticEnv & env) level so that `lookupVar' can look up variables in the previous `with' if this one doesn't contain the desired attribute. */ const StaticEnv * curEnv; - unsigned int level; + Level level; prevWith = 0; for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++) if (curEnv->isWith) { @@ -413,7 +446,7 @@ void ExprOpNot::bindVars(const StaticEnv & env) void ExprConcatStrings::bindVars(const StaticEnv & env) { for (auto & i : *es) - i->bindVars(env); + i.second->bindVars(env); } void ExprPos::bindVars(const StaticEnv & env) @@ -435,9 +468,9 @@ void ExprLambda::setName(Symbol & name) } -string ExprLambda::showNamePos() const +std::string ExprLambda::showNamePos() const { - return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str(); + return fmt("%1% at %2%", name.set() ? "'" + (std::string) name + "'" : "anonymous function", pos); } @@ -447,10 +480,9 @@ string ExprLambda::showNamePos() const size_t SymbolTable::totalSize() const { size_t n = 0; - for (auto & i : symbols) + for (auto & i : store) n += i.size(); return n; } - } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 51a14cd59..12b54b8eb 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -4,8 +4,6 @@ #include "symbol-table.hh" #include "error.hh" -#include <map> - namespace nix { @@ -28,18 +26,21 @@ struct Pos FileOrigin origin; Symbol file; unsigned int line, column; - Pos() : origin(foString), line(0), column(0) { }; + + Pos() : origin(foString), line(0), column(0) { } Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column) - : origin(origin), file(file), line(line), column(column) { }; + : origin(origin), file(file), line(line), column(column) { } + operator bool() const { return line != 0; } + bool operator < (const Pos & p2) const { if (!line) return p2.line; if (!p2.line) return false; - int d = ((string) file).compare((string) p2.file); + int d = ((const std::string &) file).compare((const std::string &) p2.file); if (d < 0) return true; if (d > 0) return false; if (line < p2.line) return true; @@ -70,7 +71,7 @@ struct AttrName typedef std::vector<AttrName> AttrPath; -string showAttrPath(const AttrPath & attrPath); +std::string showAttrPath(const AttrPath & attrPath); /* Abstract syntax of Nix expressions. */ @@ -96,7 +97,7 @@ struct ExprInt : Expr { NixInt n; Value v; - ExprInt(NixInt n) : n(n) { mkInt(v, n); }; + ExprInt(NixInt n) : n(n) { v.mkInt(n); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; @@ -105,36 +106,32 @@ struct ExprFloat : Expr { NixFloat nf; Value v; - ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); }; + ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; struct ExprString : Expr { - Symbol s; + std::string s; Value v; - ExprString(const Symbol & s) : s(s) { mkString(v, s); }; + ExprString(std::string s) : s(std::move(s)) { v.mkString(this->s.data()); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; -/* Temporary class used during parsing of indented strings. */ -struct ExprIndStr : Expr -{ - string s; - ExprIndStr(const string & s) : s(s) { }; -}; - struct ExprPath : Expr { - string s; + std::string s; Value v; - ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); }; + ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; +typedef uint32_t Level; +typedef uint32_t Displacement; + struct ExprVar : Expr { Pos pos; @@ -150,8 +147,8 @@ struct ExprVar : Expr value is obtained by getting the attribute named `name' from the set stored in the environment that is `level' levels up from the current one.*/ - unsigned int level; - unsigned int displ; + Level level; + Displacement displ; ExprVar(const Symbol & name) : name(name) { }; ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; @@ -185,7 +182,7 @@ struct ExprAttrs : Expr bool inherited; Expr * e; Pos pos; - unsigned int displ; // displacement + Displacement displ; // displacement AttrDef(Expr * e, const Pos & pos, bool inherited=false) : inherited(inherited), e(e), pos(pos) { }; AttrDef() { }; @@ -222,10 +219,25 @@ struct Formal struct Formals { - typedef std::list<Formal> Formals_; + typedef std::vector<Formal> Formals_; Formals_ formals; - std::set<Symbol> argNames; // used during parsing bool ellipsis; + + bool has(Symbol arg) const { + auto it = std::lower_bound(formals.begin(), formals.end(), arg, + [] (const Formal & f, const Symbol & sym) { return f.name < sym; }); + return it != formals.end() && it->name == arg; + } + + std::vector<Formal> lexicographicOrder() const + { + std::vector<Formal> result(formals.begin(), formals.end()); + std::sort(result.begin(), result.end(), + [] (const Formal & a, const Formal & b) { + return std::string_view(a.name) < std::string_view(b.name); + }); + return result; + } }; struct ExprLambda : Expr @@ -233,20 +245,26 @@ struct ExprLambda : Expr Pos pos; Symbol name; Symbol arg; - bool matchAttrs; Formals * formals; Expr * body; - ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body) - : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) + ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body) + : pos(pos), arg(arg), formals(formals), body(body) { - if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) - throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", arg), - .errPos = pos - }); }; void setName(Symbol & name); - string showNamePos() const; + std::string showNamePos() const; + inline bool hasFormals() const { return formals != nullptr; } + COMMON_METHODS +}; + +struct ExprCall : Expr +{ + Expr * fun; + std::vector<Expr *> args; + Pos pos; + ExprCall(const Pos & pos, Expr * fun, std::vector<Expr *> && args) + : fun(fun), args(args), pos(pos) + { } COMMON_METHODS }; @@ -308,7 +326,6 @@ struct ExprOpNot : Expr void eval(EvalState & state, Env & env, Value & v); \ }; -MakeBinOp(ExprApp, "") MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=") MakeBinOp(ExprOpAnd, "&&") @@ -321,8 +338,8 @@ struct ExprConcatStrings : Expr { Pos pos; bool forceString; - vector<Expr *> * es; - ExprConcatStrings(const Pos & pos, bool forceString, vector<Expr *> * es) + std::vector<std::pair<Pos, Expr *> > * es; + ExprConcatStrings(const Pos & pos, bool forceString, std::vector<std::pair<Pos, Expr *> > * es) : pos(pos), forceString(forceString), es(es) { }; COMMON_METHODS }; @@ -342,9 +359,39 @@ struct StaticEnv { bool isWith; const StaticEnv * up; - typedef std::map<Symbol, unsigned int> Vars; + + // Note: these must be in sorted order. + typedef std::vector<std::pair<Symbol, Displacement>> Vars; Vars vars; - StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { }; + + StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { + vars.reserve(expectedSize); + }; + + void sort() + { + std::stable_sort(vars.begin(), vars.end(), + [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; }); + } + + void deduplicate() + { + auto it = vars.begin(), jt = it, end = vars.end(); + while (jt != end) { + *it = *jt++; + while (jt != end && it->first == jt->first) *it = *jt++; + it++; + } + vars.erase(it, end); + } + + Vars::const_iterator find(const Symbol & name) const + { + Vars::value_type key(name, 0); + auto i = std::lower_bound(vars.begin(), vars.end(), key); + if (i != vars.end() && i->first == name) return i; + return vars.end(); + } }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e3749783a..919b9cfae 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -16,6 +16,8 @@ #ifndef BISON_HEADER #define BISON_HEADER +#include <variant> + #include "util.hh" #include "nixexpr.hh" @@ -33,16 +35,28 @@ namespace nix { Symbol file; FileOrigin origin; std::optional<ErrorInfo> error; - Symbol sLetBody; ParseData(EvalState & state) : state(state) , symbols(state.symbols) - , sLetBody(symbols.create("<let-body>")) { }; }; + struct ParserFormals { + std::vector<Formal> formals; + bool ellipsis = false; + }; + } +// using C a struct allows us to avoid having to define the special +// members that using string_view here would implicitly delete. +struct StringToken { + const char * p; + size_t l; + bool hasIndentation; + operator std::string_view() const { return {p, l}; } +}; + #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data) @@ -126,14 +140,14 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, auto j2 = jAttrs->attrs.find(ad.first); if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. dupAttr(ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs[ad.first] = ad.second; + jAttrs->attrs.emplace(ad.first, ad.second); } } else { dupAttr(attrPath, pos, j->second.pos); } } else { // This attr path is not defined. Let's create it. - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); + attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); e->setName(i->symbol); } } else { @@ -142,21 +156,46 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, } -static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) +static Formals * toFormals(ParseData & data, ParserFormals * formals, + Pos pos = noPos, Symbol arg = {}) { - if (!formals->argNames.insert(formal.name).second) + std::sort(formals->formals.begin(), formals->formals.end(), + [] (const auto & a, const auto & b) { + return std::tie(a.name, a.pos) < std::tie(b.name, b.pos); + }); + + std::optional<std::pair<Symbol, Pos>> duplicate; + for (size_t i = 0; i + 1 < formals->formals.size(); i++) { + if (formals->formals[i].name != formals->formals[i + 1].name) + continue; + std::pair thisDup{formals->formals[i].name, formals->formals[i + 1].pos}; + duplicate = std::min(thisDup, duplicate.value_or(thisDup)); + } + if (duplicate) + throw ParseError({ + .msg = hintfmt("duplicate formal function argument '%1%'", duplicate->first), + .errPos = duplicate->second + }); + + Formals result; + result.ellipsis = formals->ellipsis; + result.formals = std::move(formals->formals); + + if (arg.set() && result.has(arg)) throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", - formal.name), + .msg = hintfmt("duplicate formal function argument '%1%'", arg), .errPos = pos }); - formals->formals.push_front(formal); + + delete formals; + return new Formals(std::move(result)); } -static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Expr *> & es) +static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, + std::vector<std::pair<Pos, std::variant<Expr *, StringToken> > > & es) { - if (es.empty()) return new ExprString(symbols.create("")); + if (es.empty()) return new ExprString(""); /* Figure out the minimum indentation. Note that by design whitespace-only final lines are not taken into account. (So @@ -164,21 +203,21 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex bool atStartOfLine = true; /* = seen only whitespace in the current line */ size_t minIndent = 1000000; size_t curIndent = 0; - for (auto & i : es) { - ExprIndStr * e = dynamic_cast<ExprIndStr *>(i); - if (!e) { - /* Anti-quotations end the current start-of-line whitespace. */ + for (auto & [i_pos, i] : es) { + auto * str = std::get_if<StringToken>(&i); + if (!str || !str->hasIndentation) { + /* Anti-quotations and escaped characters end the current start-of-line whitespace. */ if (atStartOfLine) { atStartOfLine = false; if (curIndent < minIndent) minIndent = curIndent; } continue; } - for (size_t j = 0; j < e->s.size(); ++j) { + for (size_t j = 0; j < str->l; ++j) { if (atStartOfLine) { - if (e->s[j] == ' ') + if (str->p[j] == ' ') curIndent++; - else if (e->s[j] == '\n') { + else if (str->p[j] == '\n') { /* Empty line, doesn't influence minimum indentation. */ curIndent = 0; @@ -186,7 +225,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex atStartOfLine = false; if (curIndent < minIndent) minIndent = curIndent; } - } else if (e->s[j] == '\n') { + } else if (str->p[j] == '\n') { atStartOfLine = true; curIndent = 0; } @@ -194,53 +233,54 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex } /* Strip spaces from each line. */ - vector<Expr *> * es2 = new vector<Expr *>; + std::vector<std::pair<Pos, Expr *> > * es2 = new std::vector<std::pair<Pos, Expr *> >; atStartOfLine = true; size_t curDropped = 0; size_t n = es.size(); - for (vector<Expr *>::iterator i = es.begin(); i != es.end(); ++i, --n) { - ExprIndStr * e = dynamic_cast<ExprIndStr *>(*i); - if (!e) { - atStartOfLine = false; - curDropped = 0; - es2->push_back(*i); - continue; - } - - string s2; - for (size_t j = 0; j < e->s.size(); ++j) { + auto i = es.begin(); + const auto trimExpr = [&] (Expr * e) { + atStartOfLine = false; + curDropped = 0; + es2->emplace_back(i->first, e); + }; + const auto trimString = [&] (const StringToken & t) { + std::string s2; + for (size_t j = 0; j < t.l; ++j) { if (atStartOfLine) { - if (e->s[j] == ' ') { + if (t.p[j] == ' ') { if (curDropped++ >= minIndent) - s2 += e->s[j]; + s2 += t.p[j]; } - else if (e->s[j] == '\n') { + else if (t.p[j] == '\n') { curDropped = 0; - s2 += e->s[j]; + s2 += t.p[j]; } else { atStartOfLine = false; curDropped = 0; - s2 += e->s[j]; + s2 += t.p[j]; } } else { - s2 += e->s[j]; - if (e->s[j] == '\n') atStartOfLine = true; + s2 += t.p[j]; + if (t.p[j] == '\n') atStartOfLine = true; } } /* Remove the last line if it is empty and consists only of spaces. */ if (n == 1) { - string::size_type p = s2.find_last_of('\n'); - if (p != string::npos && s2.find_first_not_of(' ', p + 1) == string::npos) - s2 = string(s2, 0, p + 1); + std::string::size_type p = s2.find_last_of('\n'); + if (p != std::string::npos && s2.find_first_not_of(' ', p + 1) == std::string::npos) + s2 = std::string(s2, 0, p + 1); } - es2->push_back(new ExprString(symbols.create(s2))); + es2->emplace_back(i->first, new ExprString(s2)); + }; + for (; i != es.end(); ++i, --n) { + std::visit(overloaded { trimExpr, trimString }, i->second); } /* If this is a single string, then don't do a concatenation. */ - return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(pos, true, es2); + return es2->size() == 1 && dynamic_cast<ExprString *>((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2); } @@ -271,29 +311,32 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err nix::Expr * e; nix::ExprList * list; nix::ExprAttrs * attrs; - nix::Formals * formals; + nix::ParserFormals * formals; nix::Formal * formal; nix::NixInt n; nix::NixFloat nf; - const char * id; // !!! -> Symbol - char * path; - char * uri; + StringToken id; // !!! -> Symbol + StringToken path; + StringToken uri; + StringToken str; std::vector<nix::AttrName> * attrNames; - std::vector<nix::Expr *> * string_parts; + std::vector<std::pair<nix::Pos, nix::Expr *> > * string_parts; + std::vector<std::pair<nix::Pos, std::variant<nix::Expr *, StringToken> > > * ind_string_parts; } %type <e> start expr expr_function expr_if expr_op -%type <e> expr_app expr_select expr_simple +%type <e> expr_select expr_simple expr_app %type <list> expr_list %type <attrs> binds %type <formals> formals %type <formal> formal %type <attrNames> attrs attrpath -%type <string_parts> string_parts_interpolated ind_string_parts +%type <string_parts> string_parts_interpolated +%type <ind_string_parts> ind_string_parts %type <e> path_start string_parts string_attr %type <id> attr %token <id> ID ATTRPATH -%token <e> STR IND_STR +%token <str> STR IND_STR %token <n> INT %token <nf> FLOAT %token <path> PATH HPATH SPATH PATH_END @@ -324,13 +367,19 @@ expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), toFormals(*data, $2), $5); } | '{' formals '}' '@' ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); } + { + Symbol arg = data->symbols.create($5); + $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $2, CUR_POS, arg), $7); + } | ID '@' '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); } + { + Symbol arg = data->symbols.create($1); + $$ = new ExprLambda(CUR_POS, arg, toFormals(*data, $4, CUR_POS, arg), $7); + } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } | WITH expr ';' expr_function @@ -353,31 +402,36 @@ expr_if expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } - | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); } + | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); } - | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); } + | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); } | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } | expr_op '+' expr_op - { $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); } - | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); } - | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); } - | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); } + { $$ = new ExprConcatStrings(CUR_POS, false, new std::vector<std::pair<Pos, Expr *> >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } + | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); } | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } | expr_app ; expr_app - : expr_app expr_select - { $$ = new ExprApp(CUR_POS, $1, $2); } - | expr_select { $$ = $1; } + : expr_app expr_select { + if (auto e2 = dynamic_cast<ExprCall *>($1)) { + e2->args.push_back($2); + $$ = $1; + } else + $$ = new ExprCall(CUR_POS, $1, {$2}); + } + | expr_select ; expr_select @@ -388,13 +442,14 @@ expr_select | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW - { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.create("or"))); } + { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } | expr_simple { $$ = $1; } ; expr_simple : ID { - if (strcmp($1, "__curPos") == 0) + std::string_view s = "__curPos"; + if ($1.l == s.size() && strncmp($1.p, s.data(), s.size()) == 0) $$ = new ExprPos(CUR_POS); else $$ = new ExprVar(CUR_POS, data->symbols.create($1)); @@ -407,24 +462,24 @@ expr_simple } | path_start PATH_END { $$ = $1; } | path_start string_parts_interpolated PATH_END { - $2->insert($2->begin(), $1); + $2->insert($2->begin(), {makeCurPos(@1, data), $1}); $$ = new ExprConcatStrings(CUR_POS, false, $2); } | SPATH { - string path($1 + 1, strlen($1) - 2); - $$ = new ExprApp(CUR_POS, - new ExprApp(new ExprVar(data->symbols.create("__findFile")), - new ExprVar(data->symbols.create("__nixPath"))), - new ExprString(data->symbols.create(path))); + std::string path($1.p + 1, $1.l - 2); + $$ = new ExprCall(CUR_POS, + new ExprVar(data->symbols.create("__findFile")), + {new ExprVar(data->symbols.create("__nixPath")), + new ExprString(path)}); } | URI { - static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); + static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); if (noURLLiterals) throw ParseError({ .msg = hintfmt("URL literals are disabled"), .errPos = CUR_POS }); - $$ = new ExprString(data->symbols.create($1)); + $$ = new ExprString(std::string($1)); } | '(' expr ')' { $$ = $2; } /* Let expressions `let {..., body = ...}' are just desugared @@ -439,40 +494,41 @@ expr_simple ; string_parts - : STR + : STR { $$ = new ExprString(std::string($1)); } | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); } - | { $$ = new ExprString(data->symbols.create("")); } + | { $$ = new ExprString(""); } ; string_parts_interpolated - : string_parts_interpolated STR { $$ = $1; $1->push_back($2); } - | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } - | DOLLAR_CURLY expr '}' { $$ = new vector<Expr *>; $$->push_back($2); } + : string_parts_interpolated STR + { $$ = $1; $1->emplace_back(makeCurPos(@2, data), new ExprString(std::string($2))); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } + | DOLLAR_CURLY expr '}' { $$ = new std::vector<std::pair<Pos, Expr *> >; $$->emplace_back(makeCurPos(@1, data), $2); } | STR DOLLAR_CURLY expr '}' { - $$ = new vector<Expr *>; - $$->push_back($1); - $$->push_back($3); + $$ = new std::vector<std::pair<Pos, Expr *> >; + $$->emplace_back(makeCurPos(@1, data), new ExprString(std::string($1))); + $$->emplace_back(makeCurPos(@2, data), $3); } ; path_start : PATH { - Path path(absPath($1, data->basePath)); + Path path(absPath({$1.p, $1.l}, data->basePath)); /* add back in the trailing '/' to the first segment */ - if ($1[strlen($1)-1] == '/' && strlen($1) > 1) + if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; $$ = new ExprPath(path); } | HPATH { - Path path(getHome() + string($1 + 1)); + Path path(getHome() + std::string($1.p + 1, $1.l - 1)); $$ = new ExprPath(path); } ; ind_string_parts - : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); } - | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } - | { $$ = new vector<Expr *>; } + : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } + | { $$ = new std::vector<std::pair<Pos, std::variant<Expr *, StringToken> > >; } ; binds @@ -483,7 +539,7 @@ binds if ($$->attrs.find(i.symbol) != $$->attrs.end()) dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); Pos pos = makeCurPos(@3, data); - $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } } | binds INHERIT '(' expr ')' attrs ';' @@ -492,7 +548,7 @@ binds for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); - $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); } } | { $$ = new ExprAttrs(makeCurPos(@0, data)); } @@ -504,7 +560,7 @@ attrs { $$ = $1; ExprString * str = dynamic_cast<ExprString *>($2); if (str) { - $$->push_back(AttrName(str->s)); + $$->push_back(AttrName(data->symbols.create(str->s))); delete str; } else throw ParseError({ @@ -521,17 +577,17 @@ attrpath { $$ = $1; ExprString * str = dynamic_cast<ExprString *>($3); if (str) { - $$->push_back(AttrName(str->s)); + $$->push_back(AttrName(data->symbols.create(str->s))); delete str; } else $$->push_back(AttrName($3)); } - | attr { $$ = new vector<AttrName>; $$->push_back(AttrName(data->symbols.create($1))); } + | attr { $$ = new std::vector<AttrName>; $$->push_back(AttrName(data->symbols.create($1))); } | string_attr - { $$ = new vector<AttrName>; + { $$ = new std::vector<AttrName>; ExprString *str = dynamic_cast<ExprString *>($1); if (str) { - $$->push_back(AttrName(str->s)); + $$->push_back(AttrName(data->symbols.create(str->s))); delete str; } else $$->push_back(AttrName($1)); @@ -540,7 +596,7 @@ attrpath attr : ID { $$ = $1; } - | OR_KW { $$ = "or"; } + | OR_KW { $$ = {"or", 2}; } ; string_attr @@ -555,13 +611,13 @@ expr_list formals : formal ',' formals - { $$ = $3; addFormal(CUR_POS, $$, *$1); } + { $$ = $3; $$->formals.push_back(*$1); } | formal - { $$ = new Formals; addFormal(CUR_POS, $$, *$1); $$->ellipsis = false; } + { $$ = new ParserFormals; $$->formals.push_back(*$1); $$->ellipsis = false; } | - { $$ = new Formals; $$->ellipsis = false; } + { $$ = new ParserFormals; $$->ellipsis = false; } | ELLIPSIS - { $$ = new Formals; $$->ellipsis = true; } + { $$ = new ParserFormals; $$->ellipsis = true; } ; formal @@ -586,8 +642,8 @@ formal namespace nix { -Expr * EvalState::parse(const char * text, FileOrigin origin, - const Path & path, const Path & basePath, StaticEnv & staticEnv) +Expr * EvalState::parse(char * text, size_t length, FileOrigin origin, + const PathView path, const PathView basePath, StaticEnv & staticEnv) { yyscan_t scanner; ParseData data(*this); @@ -606,7 +662,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin, data.basePath = basePath; yylex_init(&scanner); - yy_scan_string(text, scanner); + yy_scan_buffer(text, length, scanner); int res = yyparse(scanner, &data); yylex_destroy(scanner); @@ -652,63 +708,70 @@ Expr * EvalState::parseExprFromFile(const Path & path) Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv) { - return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv); + auto buffer = readFile(path); + // readFile should have left some extra space for terminators + buffer.append("\0\0", 2); + return parse(buffer.data(), buffer.size(), foFile, path, dirOf(path), staticEnv); } -Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv) +Expr * EvalState::parseExprFromString(std::string s, const Path & basePath, StaticEnv & staticEnv) { - return parse(s.data(), foString, "", basePath, staticEnv); + s.append("\0\0", 2); + return parse(s.data(), s.size(), foString, "", basePath, staticEnv); } -Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath) +Expr * EvalState::parseExprFromString(std::string s, const Path & basePath) { - return parseExprFromString(s, basePath, staticBaseEnv); + return parseExprFromString(std::move(s), basePath, staticBaseEnv); } Expr * EvalState::parseStdin() { //Activity act(*logger, lvlTalkative, format("parsing standard input")); - return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv); + auto buffer = drainFD(0); + // drainFD should have left some extra space for terminators + buffer.append("\0\0", 2); + return parse(buffer.data(), buffer.size(), foStdin, "", absPath("."), staticBaseEnv); } -void EvalState::addToSearchPath(const string & s) +void EvalState::addToSearchPath(const std::string & s) { size_t pos = s.find('='); - string prefix; + std::string prefix; Path path; - if (pos == string::npos) { + if (pos == std::string::npos) { path = s; } else { - prefix = string(s, 0, pos); - path = string(s, pos + 1); + prefix = std::string(s, 0, pos); + path = std::string(s, pos + 1); } searchPath.emplace_back(prefix, path); } -Path EvalState::findFile(const string & path) +Path EvalState::findFile(const std::string_view path) { return findFile(searchPath, path); } -Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos) +Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, const Pos & pos) { for (auto & i : searchPath) { std::string suffix; if (i.first.empty()) - suffix = "/" + path; + suffix = concatStrings("/", path); else { auto s = i.first.size(); if (path.compare(0, s, i.first) != 0 || (path.size() > s && path[s] != '/')) continue; - suffix = path.size() == s ? "" : "/" + string(path, s); + suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); } auto r = resolveSearchPathElem(i); if (!r.first) continue; @@ -717,7 +780,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos } if (hasPrefix(path, "nix/")) - return corepkgsPrefix + path.substr(4); + return concatStrings(corepkgsPrefix, path.substr(4)); throw ThrownError({ .msg = hintfmt(evalSettings.pureEval @@ -752,7 +815,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl res = { true, path }; else { logWarning({ - .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.second) }); res = { false, "" }; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0422102a9..3124025aa 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -12,6 +12,8 @@ #include "value-to-xml.hh" #include "primops.hh" +#include <boost/container/small_vector.hpp> + #include <sys/types.h> #include <sys/stat.h> #include <unistd.h> @@ -35,9 +37,10 @@ namespace nix { InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} -void EvalState::realiseContext(const PathSet & context) +StringMap EvalState::realiseContext(const PathSet & context) { std::vector<DerivedPath::Built> drvs; + StringMap res; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -46,38 +49,79 @@ void EvalState::realiseContext(const PathSet & context) throw InvalidPathError(store->printStorePath(ctx)); if (!outputName.empty() && ctx.isDerivation()) { drvs.push_back({ctx, {outputName}}); + } else { + res.insert_or_assign(ctxS, ctxS); } } - if (drvs.empty()) return; + if (drvs.empty()) return {}; if (!evalSettings.enableImportFromDerivation) throw Error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", store->printStorePath(drvs.begin()->drvPath)); - /* For performance, prefetch all substitute info. */ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; + /* Build/substitute the context. */ std::vector<DerivedPath> buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); - store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(buildReqs); + /* Get all the output paths corresponding to the placeholders we had */ + for (auto & [drvPath, outputs] : drvs) { + auto outputPaths = store->queryDerivationOutputMap(drvPath); + for (auto & outputName : outputs) { + if (outputPaths.count(outputName) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store->printStorePath(drvPath), outputName); + res.insert_or_assign( + downstreamPlaceholder(*store, drvPath, outputName), + store->printStorePath(outputPaths.at(outputName)) + ); + } + } + /* Add the output of this derivations to the allowed paths. */ if (allowedPaths) { - for (auto & [drvPath, outputs] : drvs) { - auto outputPaths = store->queryDerivationOutputMap(drvPath); - for (auto & outputName : outputs) { - if (outputPaths.count(outputName) == 0) - throw Error("derivation '%s' does not have an output named '%s'", - store->printStorePath(drvPath), outputName); - allowedPaths->insert(store->printStorePath(outputPaths.at(outputName))); - } + for (auto & [_placeholder, outputPath] : res) { + allowPath(store->toRealPath(outputPath)); } } + + return res; +} + +struct RealisePathFlags { + // Whether to check that the path is allowed in pure eval mode + bool checkForPureEval = true; +}; + +static Path realisePath(EvalState & state, const Pos & pos, Value & v, const RealisePathFlags flags = {}) +{ + PathSet context; + + auto path = [&]() + { + try { + return state.coerceToPath(pos, v, context); + } catch (Error & e) { + e.addTrace(pos, "while realising the context of a path"); + throw; + } + }(); + + try { + StringMap rewrites = state.realiseContext(context); + + auto realPath = state.toRealPath(rewriteStrings(path, rewrites), context); + + return flags.checkForPureEval + ? state.checkSourcePath(realPath) + : realPath; + } catch (Error & e) { + e.addTrace(pos, "while realising the context of path '%s'", path); + throw; + } } /* Add and attribute to the given attribute map from the output name to @@ -92,13 +136,15 @@ void EvalState::realiseContext(const PathSet & context) the actual path. The 'drv' and 'drvPath' outputs must correspond. */ -static void mkOutputString(EvalState & state, Value & v, - const StorePath & drvPath, const BasicDerivation & drv, - std::pair<string, DerivationOutput> o) +static void mkOutputString( + EvalState & state, + BindingsBuilder & attrs, + const StorePath & drvPath, + const BasicDerivation & drv, + const std::pair<std::string, DerivationOutput> & o) { auto optOutputPath = o.second.path(*state.store, drv.name, o.first); - mkString( - *state.allocAttr(v, state.symbols.create(o.first)), + attrs.alloc(o.first).mkString( optOutputPath ? state.store->printStorePath(*optOutputPath) /* Downstream we would substitute this for an actual path once @@ -113,22 +159,7 @@ static void mkOutputString(EvalState & state, Value & v, argument. */ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vScope, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, vPath, context); - - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError({ - .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), - .errPos = pos - }); - } catch (Error & e) { - e.addTrace(pos, "while importing '%s'", path); - throw; - } - - Path realPath = state.checkSourcePath(state.toRealPath(path, context)); + auto path = realisePath(state, pos, vPath); // FIXME auto isValidDerivationInStore = [&]() -> std::optional<StorePath> { @@ -143,23 +174,19 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS if (auto optStorePath = isValidDerivationInStore()) { auto storePath = *optStorePath; Derivation drv = state.store->readDerivation(storePath); - Value & w = *state.allocValue(); - state.mkAttrs(w, 3 + drv.outputs.size()); - Value * v2 = state.allocAttr(w, state.sDrvPath); - mkString(*v2, path, {"=" + path}); - v2 = state.allocAttr(w, state.sName); - mkString(*v2, drv.env["name"]); - Value * outputsVal = - state.allocAttr(w, state.symbols.create("outputs")); - state.mkList(*outputsVal, drv.outputs.size()); - unsigned int outputs_index = 0; - - for (const auto & o : drv.outputs) { - mkOutputString(state, w, storePath, drv, o); - outputsVal->listElems()[outputs_index] = state.allocValue(); - mkString(*(outputsVal->listElems()[outputs_index++]), o.first); + auto attrs = state.buildBindings(3 + drv.outputs.size()); + attrs.alloc(state.sDrvPath).mkString(path, {"=" + path}); + attrs.alloc(state.sName).mkString(drv.env["name"]); + auto & outputsVal = attrs.alloc(state.sOutputs); + state.mkList(outputsVal, drv.outputs.size()); + + for (const auto & [i, o] : enumerate(drv.outputs)) { + mkOutputString(state, attrs, storePath, drv, o); + (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first); } - w.attrs->sort(); + + auto w = state.allocValue(); + w->mkAttrs(attrs); if (!state.vImportedDrvToDerivation) { state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); @@ -169,7 +196,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS } state.forceFunction(**state.vImportedDrvToDerivation, pos); - mkApp(v, **state.vImportedDrvToDerivation, w); + v.mkApp(*state.vImportedDrvToDerivation, w); state.forceAttrs(v, pos); } @@ -181,23 +208,26 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS else { if (!vScope) - state.evalFile(realPath, v); + state.evalFile(path, v); else { - state.forceAttrs(*vScope); + state.forceAttrs(*vScope, pos); Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; - StaticEnv staticEnv(false, &state.staticBaseEnv); + StaticEnv staticEnv(false, &state.staticBaseEnv, vScope->attrs->size()); unsigned int displ = 0; for (auto & attr : *vScope->attrs) { - staticEnv.vars[attr.name] = displ; + staticEnv.vars.emplace_back(attr.name, displ); env->values[displ++] = attr.value; } - printTalkative("evaluating file '%1%'", realPath); - Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); + // No need to call staticEnv.sort(), because + // args[0]->attrs is already sorted. + + printTalkative("evaluating file '%1%'", path); + Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); e->eval(state, *env, v); } @@ -282,23 +312,9 @@ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); /* Load a ValueInitializer from a DSO and return whatever it initializes */ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError({ - .msg = hintfmt( - "cannot import '%1%', since path '%2%' is not valid", - path, e.path), - .errPos = pos - }); - } - - path = state.checkSourcePath(path); + auto path = realisePath(state, pos, *args[0]); - string sym = state.forceStringNoCtx(*args[1], pos); + std::string sym(state.forceStringNoCtx(*args[1], pos)); void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) @@ -334,13 +350,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) }); } PathSet context; - auto program = state.coerceToString(pos, *elems[0], context, false, false); + auto program = state.coerceToString(pos, *elems[0], context, false, false).toOwned(); Strings commandArgs; for (unsigned int i = 1; i < args[0]->listSize(); ++i) { - commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false)); + commandArgs.push_back(state.coerceToString(pos, *elems[i], context, false, false).toOwned()); } try { - state.realiseContext(context); + auto _ = state.realiseContext(context); // FIXME: Handle CA derivations } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid", @@ -352,7 +368,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) auto output = runProgram(program, true, commandArgs); Expr * parsed; try { - parsed = state.parseExprFromString(output, pos.file); + parsed = state.parseExprFromString(std::move(output), pos.file); } catch (Error & e) { e.addTrace(pos, "While parsing the output from '%1%'", program); throw; @@ -370,7 +386,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - string t; + std::string t; switch (args[0]->type()) { case nInt: t = "int"; break; case nBool: t = "bool"; break; @@ -386,7 +402,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu case nFloat: t = "float"; break; case nThunk: abort(); } - mkString(v, state.symbols.create(t)); + v.mkString(state.symbols.create(t)); } static RegisterPrimOp primop_typeOf({ @@ -404,7 +420,7 @@ static RegisterPrimOp primop_typeOf({ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nNull); + v.mkBool(args[0]->type() == nNull); } static RegisterPrimOp primop_isNull({ @@ -414,7 +430,7 @@ static RegisterPrimOp primop_isNull({ Return `true` if *e* evaluates to `null`, and `false` otherwise. > **Warning** - > + > > This function is *deprecated*; just write `e == null` instead. )", .fun = prim_isNull, @@ -424,7 +440,7 @@ static RegisterPrimOp primop_isNull({ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nFunction); + v.mkBool(args[0]->type() == nFunction); } static RegisterPrimOp primop_isFunction({ @@ -440,7 +456,7 @@ static RegisterPrimOp primop_isFunction({ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nInt); + v.mkBool(args[0]->type() == nInt); } static RegisterPrimOp primop_isInt({ @@ -456,7 +472,7 @@ static RegisterPrimOp primop_isInt({ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nFloat); + v.mkBool(args[0]->type() == nFloat); } static RegisterPrimOp primop_isFloat({ @@ -472,7 +488,7 @@ static RegisterPrimOp primop_isFloat({ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nString); + v.mkBool(args[0]->type() == nString); } static RegisterPrimOp primop_isString({ @@ -488,7 +504,7 @@ static RegisterPrimOp primop_isString({ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nBool); + v.mkBool(args[0]->type() == nBool); } static RegisterPrimOp primop_isBool({ @@ -504,7 +520,7 @@ static RegisterPrimOp primop_isBool({ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nPath); + v.mkBool(args[0]->type() == nPath); } static RegisterPrimOp primop_isPath({ @@ -518,7 +534,11 @@ static RegisterPrimOp primop_isPath({ struct CompareValues { - bool operator () (const Value * v1, const Value * v2) const + EvalState & state; + + CompareValues(EvalState & state) : state(state) { }; + + bool operator () (Value * v1, Value * v2) const { if (v1->type() == nFloat && v2->type() == nInt) return v1->fpoint < v2->integer; @@ -535,6 +555,17 @@ struct CompareValues return strcmp(v1->string.s, v2->string.s) < 0; case nPath: return strcmp(v1->path, v2->path) < 0; + case nList: + // Lexicographic comparison + for (size_t i = 0;; i++) { + if (i == v2->listSize()) { + return false; + } else if (i == v1->listSize()) { + return true; + } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i])) { + return (*this)(v1->listElems()[i], v2->listElems()[i]); + } + } default: throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); } @@ -543,24 +574,24 @@ struct CompareValues #if HAVE_BOEHMGC -typedef list<Value *, gc_allocator<Value *> > ValueList; +typedef std::list<Value *, gc_allocator<Value *> > ValueList; #else -typedef list<Value *> ValueList; +typedef std::list<Value *> ValueList; #endif static Bindings::iterator getAttr( EvalState & state, - string funcName, - string attrName, + std::string_view funcName, + Symbol attrSym, Bindings * attrSet, const Pos & pos) { - Bindings::iterator value = attrSet->find(state.symbols.create(attrName)); + Bindings::iterator value = attrSet->find(attrSym); if (value == attrSet->end()) { hintformat errorMsg = hintfmt( "attribute '%s' missing for call to '%s'", - attrName, + attrSym, funcName ); @@ -579,7 +610,7 @@ static Bindings::iterator getAttr( // Adding another trace for the function name to make it clear // which call received wrong arguments. e.addTrace(pos, hintfmt("while invoking '%s'", funcName)); - throw; + throw e; } } @@ -594,7 +625,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar Bindings::iterator startSet = getAttr( state, "genericClosure", - "startSet", + state.sStartSet, args[0]->attrs, pos ); @@ -602,14 +633,14 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar state.forceList(*startSet->value, pos); ValueList workSet; - for (unsigned int n = 0; n < startSet->value->listSize(); ++n) - workSet.push_back(startSet->value->listElems()[n]); + for (auto elem : startSet->value->listItems()) + workSet.push_back(elem); /* Get the operator. */ Bindings::iterator op = getAttr( state, "genericClosure", - "operator", + state.sOperator, args[0]->attrs, pos ); @@ -622,7 +653,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar ValueList res; // `doneKeys' doesn't need to be a GC root, because its values are // reachable from res. - set<Value *, CompareValues> doneKeys; + auto cmp = CompareValues(state); + std::set<Value *, decltype(cmp)> doneKeys(cmp); while (!workSet.empty()) { Value * e = *(workSet.begin()); workSet.pop_front(); @@ -630,7 +662,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar state.forceAttrs(*e, pos); Bindings::iterator key = - e->attrs->find(state.symbols.create("key")); + e->attrs->find(state.sKey); if (key == e->attrs->end()) throw EvalError({ .msg = hintfmt("attribute 'key' required"), @@ -643,13 +675,13 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar /* Call the `operator' function with `e' as argument. */ Value call; - mkApp(call, *op->value, *e); + call.mkApp(op->value, e); state.forceList(call, pos); /* Add the values returned by the operator to the work set. */ - for (unsigned int n = 0; n < call.listSize(); ++n) { - state.forceValue(*call.listElems()[n], pos); - workSet.push_back(call.listElems()[n]); + for (auto elem : call.listItems()) { + state.forceValue(*elem, pos); + workSet.push_back(elem); } } @@ -675,7 +707,7 @@ static RegisterPrimOp primop_abort({ .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - string s = state.coerceToString(pos, *args[0], context); + auto s = state.coerceToString(pos, *args[0], context).toOwned(); throw Abort("evaluation aborted with the following error message: '%1%'", s); } }); @@ -693,7 +725,7 @@ static RegisterPrimOp primop_throw({ .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - string s = state.coerceToString(pos, *args[0], context); + auto s = state.coerceToString(pos, *args[0], context).toOwned(); throw ThrownError(s); } }); @@ -705,7 +737,7 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a v = *args[1]; } catch (Error & e) { PathSet context; - e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context)); + e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context).toOwned()); throw; } } @@ -719,7 +751,7 @@ static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { static void prim_ceil(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); - mkInt(v, ceil(value)); + v.mkInt(ceil(value)); } static RegisterPrimOp primop_ceil({ @@ -738,7 +770,7 @@ static RegisterPrimOp primop_ceil({ static void prim_floor(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); - mkInt(v, floor(value)); + v.mkInt(floor(value)); } static RegisterPrimOp primop_floor({ @@ -758,16 +790,16 @@ static RegisterPrimOp primop_floor({ * else => {success=false; value=false;} */ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.mkAttrs(v, 2); + auto attrs = state.buildBindings(2); try { state.forceValue(*args[0], pos); - v.attrs->push_back(Attr(state.sValue, args[0])); - mkBool(*state.allocAttr(v, state.symbols.create("success")), true); + attrs.insert(state.sValue, args[0]); + attrs.alloc("success").mkBool(true); } catch (AssertionError & e) { - mkBool(*state.allocAttr(v, state.sValue), false); - mkBool(*state.allocAttr(v, state.symbols.create("success")), false); + attrs.alloc(state.sValue).mkBool(false); + attrs.alloc("success").mkBool(false); } - v.attrs->sort(); + v.mkAttrs(attrs); } static RegisterPrimOp primop_tryEval({ @@ -794,8 +826,8 @@ static RegisterPrimOp primop_tryEval({ /* Return an environment variable. Use with care. */ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) { - string name = state.forceStringNoCtx(*args[0], pos); - mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); + std::string name(state.forceStringNoCtx(*args[0], pos)); + v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); } static RegisterPrimOp primop_getEnv({ @@ -903,7 +935,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * pos ); - string drvName; + std::string drvName; Pos & posDrvName(*attr->pos); try { drvName = state.forceStringNoCtx(*attr->value, pos); @@ -941,10 +973,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & i : args[0]->attrs->lexicographicOrder()) { if (i->name == state.sIgnoreNulls) continue; - const string & key = i->name; + const std::string & key = i->name; vomit("processing attribute '%1%'", key); - auto handleHashMode = [&](const std::string & s) { + auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; else @@ -989,16 +1021,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } if (i->name == state.sContentAddressed) { - settings.requireExperimentalFeature("ca-derivations"); contentAddressed = state.forceBool(*i->value, pos); + if (contentAddressed) + settings.requireExperimentalFeature(Xp::CaDerivations); } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { state.forceList(*i->value, pos); - for (unsigned int n = 0; n < i->value->listSize(); ++n) { - string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); + for (auto elem : i->value->listItems()) { + auto s = state.coerceToString(posDrvName, *elem, context, true).toOwned(); drv.args.push_back(s); } } @@ -1012,7 +1045,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (i->name == state.sStructuredAttrs) continue; auto placeholder(jsonObject->placeholder(key)); - printValueAsJSON(state, true, *i->value, placeholder, context); + printValueAsJSON(state, true, *i->value, pos, placeholder, context); if (i->name == state.sBuilder) drv.builder = state.forceString(*i->value, context, posDrvName); @@ -1028,18 +1061,18 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Require ‘outputs’ to be a list of strings. */ state.forceList(*i->value, posDrvName); Strings ss; - for (unsigned int n = 0; n < i->value->listSize(); ++n) - ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName)); + for (auto elem : i->value->listItems()) + ss.emplace_back(state.forceStringNoCtx(*elem, posDrvName)); handleOutputs(ss); } } else { - auto s = state.coerceToString(*i->pos, *i->value, context, true); + auto s = state.coerceToString(*i->pos, *i->value, context, true).toOwned(); drv.env.emplace(key, s); - if (i->name == state.sBuilder) drv.builder = s; - else if (i->name == state.sSystem) drv.platform = s; - else if (i->name == state.sOutputHash) outputHash = s; - else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s; + if (i->name == state.sBuilder) drv.builder = std::move(s); + else if (i->name == state.sSystem) drv.platform = std::move(s); + else if (i->name == state.sOutputHash) outputHash = std::move(s); + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = std::move(s); else if (i->name == state.sOutputHashMode) handleHashMode(s); else if (i->name == state.sOutputs) handleOutputs(tokenizeString<Strings>(s)); @@ -1085,7 +1118,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Handle derivation outputs of the form ‘!<name>!<path>’. */ else if (path.at(0) == '!') { - std::pair<string, string> ctx = decodeContext(path); + auto ctx = decodeContext(path); drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second); } @@ -1147,7 +1180,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drv.outputs.insert_or_assign(i, DerivationOutput { .output = DerivationOutputCAFloating { .method = ingestionMethod, - .hashType = std::move(ht), + .hashType = ht, }, }); } @@ -1174,7 +1207,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * // hash per output. auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); std::visit(overloaded { - [&](Hash h) { + [&](Hash & h) { for (auto & i : outputs) { auto outPath = state.store->makeOutputPath(i, h, drvName); drv.env[i] = state.store->printStorePath(outPath); @@ -1186,11 +1219,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * }); } }, - [&](CaOutputHashes) { + [&](CaOutputHashes &) { // Shouldn't happen as the toplevel derivation is not CA. assert(false); }, - [&](DeferredHash _) { + [&](DeferredHash &) { for (auto & i : outputs) { drv.outputs.insert_or_assign(i, DerivationOutput { @@ -1220,11 +1253,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drvHashes.lock()->insert_or_assign(drvPath, h); } - state.mkAttrs(v, 1 + drv.outputs.size()); - mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); + auto attrs = state.buildBindings(1 + drv.outputs.size()); + attrs.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS}); for (auto & i : drv.outputs) - mkOutputString(state, v, drvPath, drv, i); - v.attrs->sort(); + mkOutputString(state, attrs, drvPath, drv, i); + v.mkAttrs(attrs); } static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { @@ -1237,12 +1270,12 @@ static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { substituted by the corresponding output path at build time. For example, 'placeholder "out"' returns the string /1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9. At build - time, any occurence of this string in an derivation attribute will + time, any occurrence of this string in an derivation attribute will be replaced with the concrete path in the Nix store of the output ‘out’. */ static void prim_placeholder(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); + v.mkString(hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); } static RegisterPrimOp primop_placeholder({ @@ -1267,7 +1300,7 @@ static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Valu { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); - mkString(v, canonPath(path), context); + v.mkString(canonPath(path), context); } static RegisterPrimOp primop_toPath({ @@ -1311,7 +1344,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V if (!settings.readOnlyMode) state.store->ensurePath(path2); context.insert(state.store->printStorePath(path2)); - mkString(v, path, context); + v.mkString(path, context); } static RegisterPrimOp primop_storePath({ @@ -1334,27 +1367,21 @@ static RegisterPrimOp primop_storePath({ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError({ - .msg = hintfmt( - "cannot check the existence of '%1%', since path '%2%' is not valid", - path, e.path), - .errPos = pos - }); - } + /* We don’t check the path right now, because we don’t want to + throw if the path isn’t allowed, but just return false (and we + can’t just catch the exception here because we still want to + throw if something in the evaluation of `*args[0]` tries to + access an unauthorized path). */ + auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); try { - mkBool(v, pathExists(state.checkSourcePath(path))); + v.mkBool(pathExists(state.checkSourcePath(path))); } catch (SysError & e) { /* Don't give away info from errors while canonicalising ‘path’ in restricted mode. */ - mkBool(v, false); + v.mkBool(false); } catch (RestrictedPathError & e) { - mkBool(v, false); + v.mkBool(false); } } @@ -1373,7 +1400,7 @@ static RegisterPrimOp primop_pathExists({ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - mkString(v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), context); + v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, false, false)), context); } static RegisterPrimOp primop_baseNameOf({ @@ -1393,8 +1420,9 @@ static RegisterPrimOp primop_baseNameOf({ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); - if (args[0]->type() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); + auto path = state.coerceToString(pos, *args[0], context, false, false); + auto dir = dirOf(*path); + if (args[0]->type() == nPath) v.mkPath(dir); else v.mkString(dir, context); } static RegisterPrimOp primop_dirOf({ @@ -1411,20 +1439,19 @@ static RegisterPrimOp primop_dirOf({ /* Return the contents of a file as a string. */ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError({ - .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), - .errPos = pos - }); - } - string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); - if (s.find((char) 0) != string::npos) + auto path = realisePath(state, pos, *args[0]); + auto s = readFile(path); + if (s.find((char) 0) != std::string::npos) throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path); - mkString(v, s.c_str()); + StorePathSet refs; + if (state.store->isInStore(path)) { + try { + refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; + } catch (Error &) { // FIXME: should be InvalidPathError + } + } + auto context = state.store->printStorePathSet(refs); + v.mkString(s, context); } static RegisterPrimOp primop_readFile({ @@ -1444,28 +1471,28 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va SearchPath searchPath; - for (unsigned int n = 0; n < args[0]->listSize(); ++n) { - Value & v2(*args[0]->listElems()[n]); - state.forceAttrs(v2, pos); + for (auto v2 : args[0]->listItems()) { + state.forceAttrs(*v2, pos); - string prefix; - Bindings::iterator i = v2.attrs->find(state.symbols.create("prefix")); - if (i != v2.attrs->end()) + std::string prefix; + Bindings::iterator i = v2->attrs->find(state.sPrefix); + if (i != v2->attrs->end()) prefix = state.forceStringNoCtx(*i->value, pos); i = getAttr( state, "findFile", - "path", - v2.attrs, + state.sPath, + v2->attrs, pos ); PathSet context; - string path = state.coerceToString(pos, *i->value, context, false, false); + auto path = state.coerceToString(pos, *i->value, context, false, false).toOwned(); try { - state.realiseContext(context); + auto rewrites = state.realiseContext(context); + path = rewriteStrings(path, rewrites); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), @@ -1473,12 +1500,13 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va }); } + searchPath.emplace_back(prefix, path); } - string path = state.forceStringNoCtx(*args[1], pos); + auto path = state.forceStringNoCtx(*args[1], pos); - mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); + v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); } static RegisterPrimOp primop_findFile(RegisterPrimOp::Info { @@ -1490,7 +1518,7 @@ static RegisterPrimOp primop_findFile(RegisterPrimOp::Info { /* Return the cryptographic hash of a file in base-16. */ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) { - string type = state.forceStringNoCtx(*args[0], pos); + auto type = state.forceStringNoCtx(*args[0], pos); std::optional<HashType> ht = parseHashType(type); if (!ht) throw Error({ @@ -1498,15 +1526,9 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va .errPos = pos }); - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); - try { - state.realiseContext(context); - } catch (InvalidPathError & e) { - throw EvalError("cannot read '%s' since path '%s' is not valid, at %s", path, e.path, pos); - } + auto path = realisePath(state, pos, *args[1]); - mkString(v, hashFile(*ht, state.checkSourcePath(state.toRealPath(path, context))).to_string(Base16, false)); + v.mkString(hashFile(*ht, path).to_string(Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -1523,32 +1545,23 @@ static RegisterPrimOp primop_hashFile({ /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet ctx; - Path path = state.coerceToPath(pos, *args[0], ctx); - try { - state.realiseContext(ctx); - } catch (InvalidPathError & e) { - throw EvalError({ - .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), - .errPos = pos - }); - } + auto path = realisePath(state, pos, *args[0]); + + DirEntries entries = readDirectory(path); - DirEntries entries = readDirectory(state.checkSourcePath(path)); - state.mkAttrs(v, entries.size()); + auto attrs = state.buildBindings(entries.size()); for (auto & ent : entries) { - Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name)); if (ent.type == DT_UNKNOWN) ent.type = getFileType(path + "/" + ent.name); - ent_val->mkString( + attrs.alloc(ent.name).mkString( ent.type == DT_REG ? "regular" : ent.type == DT_DIR ? "directory" : ent.type == DT_LNK ? "symlink" : "unknown"); } - v.attrs->sort(); + v.mkAttrs(attrs); } static RegisterPrimOp primop_readDir({ @@ -1583,8 +1596,8 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value { std::ostringstream out; PathSet context; - printValueAsXML(state, true, false, *args[0], out, context); - mkString(v, out.str(), context); + printValueAsXML(state, true, false, *args[0], out, context, pos); + v.mkString(out.str(), context); } static RegisterPrimOp primop_toXML({ @@ -1691,8 +1704,8 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu { std::ostringstream out; PathSet context; - printValueAsJSON(state, true, *args[0], out, context); - mkString(v, out.str(), context); + printValueAsJSON(state, true, *args[0], pos, out, context); + v.mkString(out.str(), context); } static RegisterPrimOp primop_toJSON({ @@ -1712,7 +1725,7 @@ static RegisterPrimOp primop_toJSON({ /* Parse a JSON string to a value. */ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) { - string s = state.forceStringNoCtx(*args[0], pos); + auto s = state.forceStringNoCtx(*args[0], pos); try { parseJSON(state, s, v); } catch (JSONParseError &e) { @@ -1741,8 +1754,8 @@ static RegisterPrimOp primop_fromJSON({ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - string name = state.forceStringNoCtx(*args[0], pos); - string contents = state.forceString(*args[1], context, pos); + std::string name(state.forceStringNoCtx(*args[0], pos)); + std::string contents(state.forceString(*args[1], context, pos)); StorePathSet refs; @@ -1766,7 +1779,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu result, since `storePath' itself has references to the paths used in args[1]. */ - mkString(v, storePath, {storePath}); + v.mkString(storePath, {storePath}); } static RegisterPrimOp primop_toFile({ @@ -1847,50 +1860,78 @@ static RegisterPrimOp primop_toFile({ .fun = prim_toFile, }); -static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v) +static void addPath( + EvalState & state, + const Pos & pos, + const std::string & name, + Path path, + Value * filterFun, + FileIngestionMethod method, + const std::optional<Hash> expectedHash, + Value & v, + const PathSet & context) { - const auto path = evalSettings.pureEval && expectedHash ? - path_ : - state.checkSourcePath(path_); - PathFilter filter = filterFun ? ([&](const Path & path) { - auto st = lstat(path); - - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - mkString(arg1, path); - - Value fun2; - state.callFunction(*filterFun, arg1, fun2, noPos); - - Value arg2; - mkString(arg2, - S_ISREG(st.st_mode) ? "regular" : - S_ISDIR(st.st_mode) ? "directory" : - S_ISLNK(st.st_mode) ? "symlink" : - "unknown" /* not supported, will fail! */); - - Value res; - state.callFunction(fun2, arg2, res, noPos); - - return state.forceBool(res, pos); - }) : defaultPathFilter; - - std::optional<StorePath> expectedStorePath; - if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); - Path dstPath; - if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - dstPath = state.store->printStorePath(settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); - if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) - throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); - } else - dstPath = state.store->printStorePath(*expectedStorePath); + try { + // FIXME: handle CA derivation outputs (where path needs to + // be rewritten to the actual output). + auto rewrites = state.realiseContext(context); + path = state.toRealPath(rewriteStrings(path, rewrites), context); + + StorePathSet refs; + + if (state.store->isInStore(path)) { + try { + auto [storePath, subPath] = state.store->toStorePath(path); + // FIXME: we should scanForReferences on the path before adding it + refs = state.store->queryPathInfo(storePath)->references; + path = state.store->toRealPath(storePath) + subPath; + } catch (Error &) { // FIXME: should be InvalidPathError + } + } - mkString(v, dstPath, {dstPath}); + path = evalSettings.pureEval && expectedHash + ? path + : state.checkSourcePath(path); + + PathFilter filter = filterFun ? ([&](const Path & path) { + auto st = lstat(path); + + /* Call the filter function. The first argument is the path, + the second is a string indicating the type of the file. */ + Value arg1; + arg1.mkString(path); + + Value arg2; + arg2.mkString( + S_ISREG(st.st_mode) ? "regular" : + S_ISDIR(st.st_mode) ? "directory" : + S_ISLNK(st.st_mode) ? "symlink" : + "unknown" /* not supported, will fail! */); + + Value * args []{&arg1, &arg2}; + Value res; + state.callFunction(*filterFun, 2, args, res, pos); + + return state.forceBool(res, pos); + }) : defaultPathFilter; + + std::optional<StorePath> expectedStorePath; + if (expectedHash) + expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); + + if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { + StorePath dstPath = settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs); + if (expectedHash && expectedStorePath != dstPath) + throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); + state.allowAndSetStorePathString(dstPath, v); + } else + state.allowAndSetStorePathString(*expectedStorePath, v); + } catch (Error & e) { + e.addTrace(pos, "while adding path '%s'", path); + throw; + } } @@ -1898,11 +1939,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args { PathSet context; Path path = state.coerceToPath(pos, *args[1], context); - if (!context.empty()) - throw EvalError({ - .msg = hintfmt("string '%1%' cannot refer to other paths", path), - .errPos = pos - }); state.forceValue(*args[0], pos); if (args[0]->type() != nFunction) @@ -1913,13 +1949,26 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ .name = "__filterSource", .args = {"e1", "e2"}, .doc = R"( + > **Warning** + > + > `filterSource` should not be used to filter store paths. Since + > `filterSource` uses the name of the input directory while naming + > the output directory, doing so will produce a directory name in + > the form of `<hash2>-<hash>-<name>`, where `<hash>-<name>` is + > the name of the input directory. Since `<hash>` depends on the + > unfiltered directory, the name of the output directory will + > indirectly depend on files that are filtered out by the + > function. This will trigger a rebuild even when a filtered out + > file is changed. Use `builtins.path` instead, which allows + > specifying the name of the output directory. + This function allows you to copy sources into the Nix store while filtering certain files. For instance, suppose that you want to use the directory `source-dir` as an input to a Nix expression, e.g. @@ -1962,22 +2011,17 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value { state.forceAttrs(*args[0], pos); Path path; - string name; + std::string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; std::optional<Hash> expectedHash; + PathSet context; for (auto & attr : *args[0]->attrs) { - const string & n(attr.name); - if (n == "path") { - PathSet context; + auto & n(attr.name); + if (n == "path") path = state.coerceToPath(*attr.pos, *attr.value, context); - if (!context.empty()) - throw EvalError({ - .msg = hintfmt("string '%1%' cannot refer to other paths", path), - .errPos = *attr.pos - }); - } else if (attr.name == state.sName) + else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { state.forceValue(*attr.value, pos); @@ -2000,7 +2044,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value if (name.empty()) name = baseNameOf(path); - addPath(state, pos, name, path, filterFun, method, expectedHash, v); + addPath(state, pos, name, path, filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ @@ -2054,7 +2098,7 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V size_t n = 0; for (auto & i : *args[0]->attrs) - mkString(*(v.listElems()[n++] = state.allocValue()), i.name); + (v.listElems()[n++] = state.allocValue())->mkString(i.name); std::sort(v.listElems(), v.listElems() + n, [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); @@ -2084,7 +2128,10 @@ static void prim_attrValues(EvalState & state, const Pos & pos, Value * * args, v.listElems()[n++] = (Value *) &i; std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return (string) ((Attr *) v1)->name < (string) ((Attr *) v2)->name; }); + [](Value * v1, Value * v2) { + std::string_view s1 = ((Attr *) v1)->name, s2 = ((Attr *) v2)->name; + return s1 < s2; + }); for (unsigned int i = 0; i < n; ++i) v.listElems()[i] = ((Attr *) v.listElems()[i])->value; @@ -2103,13 +2150,12 @@ static RegisterPrimOp primop_attrValues({ /* Dynamic version of the `.' operator. */ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) { - string attr = state.forceStringNoCtx(*args[0], pos); + auto attr = state.forceStringNoCtx(*args[0], pos); state.forceAttrs(*args[1], pos); - // !!! Should we create a symbol here or just do a lookup? Bindings::iterator i = getAttr( state, "getAttr", - attr, + state.symbols.create(attr), args[1]->attrs, pos ); @@ -2134,11 +2180,11 @@ static RegisterPrimOp primop_getAttr({ /* Return position information of the specified attribute. */ static void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, Value & v) { - string attr = state.forceStringNoCtx(*args[0], pos); + auto attr = state.forceStringNoCtx(*args[0], pos); state.forceAttrs(*args[1], pos); Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) - mkNull(v); + v.mkNull(); else state.mkPos(v, i->pos); } @@ -2152,9 +2198,9 @@ static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info { /* Dynamic version of the `?' operator. */ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) { - string attr = state.forceStringNoCtx(*args[0], pos); + auto attr = state.forceStringNoCtx(*args[0], pos); state.forceAttrs(*args[1], pos); - mkBool(v, args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); + v.mkBool(args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); } static RegisterPrimOp primop_hasAttr({ @@ -2172,7 +2218,7 @@ static RegisterPrimOp primop_hasAttr({ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nAttrs); + v.mkBool(args[0]->type() == nAttrs); } static RegisterPrimOp primop_isAttrs({ @@ -2189,21 +2235,26 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, state.forceAttrs(*args[0], pos); state.forceList(*args[1], pos); - /* Get the attribute names to be removed. */ - std::set<Symbol> names; - for (unsigned int i = 0; i < args[1]->listSize(); ++i) { - state.forceStringNoCtx(*args[1]->listElems()[i], pos); - names.insert(state.symbols.create(args[1]->listElems()[i]->string.s)); + /* Get the attribute names to be removed. + We keep them as Attrs instead of Symbols so std::set_difference + can be used to remove them from attrs[0]. */ + boost::container::small_vector<Attr, 64> names; + names.reserve(args[1]->listSize()); + for (auto elem : args[1]->listItems()) { + state.forceStringNoCtx(*elem, pos); + names.emplace_back(state.symbols.create(elem->string.s), nullptr); } + std::sort(names.begin(), names.end()); /* Copy all attributes not in that set. Note that we don't need to sort v.attrs because it's a subset of an already sorted vector. */ - state.mkAttrs(v, args[0]->attrs->size()); - for (auto & i : *args[0]->attrs) { - if (names.find(i.name) == names.end()) - v.attrs->push_back(i); - } + auto attrs = state.buildBindings(args[0]->attrs->size()); + std::set_difference( + args[0]->attrs->begin(), args[0]->attrs->end(), + names.begin(), names.end(), + std::back_inserter(attrs)); + v.mkAttrs(attrs.alreadySorted()); } static RegisterPrimOp primop_removeAttrs({ @@ -2225,29 +2276,28 @@ static RegisterPrimOp primop_removeAttrs({ /* Builds a set from a list specifying (name, value) pairs. To be precise, a list [{name = "name1"; value = value1;} ... {name = "nameN"; value = valueN;}] is transformed to {name1 = value1; - ... nameN = valueN;}. In case of duplicate occurences of the same + ... nameN = valueN;}. In case of duplicate occurrences of the same name, the first takes precedence. */ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceList(*args[0], pos); - state.mkAttrs(v, args[0]->listSize()); + auto attrs = state.buildBindings(args[0]->listSize()); std::set<Symbol> seen; - for (unsigned int i = 0; i < args[0]->listSize(); ++i) { - Value & v2(*args[0]->listElems()[i]); - state.forceAttrs(v2, pos); + for (auto v2 : args[0]->listItems()) { + state.forceAttrs(*v2, pos); Bindings::iterator j = getAttr( state, "listToAttrs", state.sName, - v2.attrs, + v2->attrs, pos ); - string name = state.forceStringNoCtx(*j->value, *j->pos); + auto name = state.forceStringNoCtx(*j->value, *j->pos); Symbol sym = state.symbols.create(name); if (seen.insert(sym).second) { @@ -2255,14 +2305,14 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, state, "listToAttrs", state.sValue, - v2.attrs, + v2->attrs, pos ); - v.attrs->push_back(Attr(sym, j2->value, j2->pos)); + attrs.insert(sym, j2->value, j2->pos); } } - v.attrs->sort(); + v.mkAttrs(attrs); } static RegisterPrimOp primop_listToAttrs({ @@ -2295,13 +2345,15 @@ static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * ar state.forceAttrs(*args[0], pos); state.forceAttrs(*args[1], pos); - state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size())); + auto attrs = state.buildBindings(std::min(args[0]->attrs->size(), args[1]->attrs->size())); for (auto & i : *args[0]->attrs) { Bindings::iterator j = args[1]->attrs->find(i.name); if (j != args[1]->attrs->end()) - v.attrs->push_back(*j); + attrs.insert(*j); } + + v.mkAttrs(attrs.alreadySorted()); } static RegisterPrimOp primop_intersectAttrs({ @@ -2322,11 +2374,10 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va Value * res[args[1]->listSize()]; unsigned int found = 0; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - Value & v2(*args[1]->listElems()[n]); - state.forceAttrs(v2, pos); - Bindings::iterator i = v2.attrs->find(attrName); - if (i != v2.attrs->end()) + for (auto v2 : args[1]->listItems()) { + state.forceAttrs(*v2, pos); + Bindings::iterator i = v2->attrs->find(attrName); + if (i != v2->attrs->end()) res[found++] = i->value; } @@ -2356,7 +2407,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args { state.forceValue(*args[0], pos); if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) { - state.mkAttrs(v, 0); + v.mkAttrs(&state.emptyBindings); return; } if (!args[0]->isLambda()) @@ -2365,19 +2416,16 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - if (!args[0]->lambda.fun->matchAttrs) { - state.mkAttrs(v, 0); + if (!args[0]->lambda.fun->hasFormals()) { + v.mkAttrs(&state.emptyBindings); return; } - state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); - for (auto & i : args[0]->lambda.fun->formals->formals) { + auto attrs = state.buildBindings(args[0]->lambda.fun->formals->formals.size()); + for (auto & i : args[0]->lambda.fun->formals->formals) // !!! should optimise booleans (allocate only once) - Value * value = state.allocValue(); - v.attrs->push_back(Attr(i.name, value, ptr(&i.pos))); - mkBool(*value, i.def); - } - v.attrs->sort(); + attrs.alloc(i.name, ptr(&i.pos)).mkBool(i.def); + v.mkAttrs(attrs); } static RegisterPrimOp primop_functionArgs({ @@ -2402,15 +2450,17 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va { state.forceAttrs(*args[1], pos); - state.mkAttrs(v, args[1]->attrs->size()); + auto attrs = state.buildBindings(args[1]->attrs->size()); for (auto & i : *args[1]->attrs) { Value * vName = state.allocValue(); Value * vFun2 = state.allocValue(); - mkString(*vName, i.name); - mkApp(*vFun2, *args[0], *vName); - mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value); + vName->mkString(i.name); + vFun2->mkApp(args[0], vName); + attrs.alloc(i.name).mkApp(vFun2, i.value); } + + v.mkAttrs(attrs.alreadySorted()); } static RegisterPrimOp primop_mapAttrs({ @@ -2428,6 +2478,91 @@ static RegisterPrimOp primop_mapAttrs({ .fun = prim_mapAttrs, }); +static void prim_zipAttrsWith(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + // we will first count how many values are present for each given key. + // we then allocate a single attrset and pre-populate it with lists of + // appropriate sizes, stash the pointers to the list elements of each, + // and populate the lists. after that we replace the list in the every + // attribute with the merge function application. this way we need not + // use (slightly slower) temporary storage the GC does not know about. + + std::map<Symbol, std::pair<size_t, Value * *>> attrsSeen; + + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + const auto listSize = args[1]->listSize(); + const auto listElems = args[1]->listElems(); + + for (unsigned int n = 0; n < listSize; ++n) { + Value * vElem = listElems[n]; + try { + state.forceAttrs(*vElem, noPos); + for (auto & attr : *vElem->attrs) + attrsSeen[attr.name].first++; + } catch (TypeError & e) { + e.addTrace(pos, hintfmt("while invoking '%s'", "zipAttrsWith")); + throw; + } + } + + auto attrs = state.buildBindings(attrsSeen.size()); + for (auto & [sym, elem] : attrsSeen) { + auto & list = attrs.alloc(sym); + state.mkList(list, elem.first); + elem.second = list.listElems(); + } + v.mkAttrs(attrs.alreadySorted()); + + for (unsigned int n = 0; n < listSize; ++n) { + Value * vElem = listElems[n]; + for (auto & attr : *vElem->attrs) + *attrsSeen[attr.name].second++ = attr.value; + } + + for (auto & attr : *v.attrs) { + auto name = state.allocValue(); + name->mkString(attr.name); + auto call1 = state.allocValue(); + call1->mkApp(args[0], name); + auto call2 = state.allocValue(); + call2->mkApp(call1, attr.value); + attr.value = call2; + } +} + +static RegisterPrimOp primop_zipAttrsWith({ + .name = "__zipAttrsWith", + .args = {"f", "list"}, + .doc = R"( + Transpose a list of attribute sets into an attribute set of lists, + then apply `mapAttrs`. + + `f` receives two arguments: the attribute name and a non-empty + list of all values encountered for that attribute name. + + The result is an attribute set where the attribute names are the + union of the attribute names in each element of `list`. The attribute + values are the return values of `f`. + + ```nix + builtins.zipAttrsWith + (name: values: { inherit name values; }) + [ { a = "x"; } { a = "y"; b = "z"; } ] + ``` + + evaluates to + + ``` + { + a = { name = "a"; values = [ "x" "y" ]; }; + b = { name = "b"; values = [ "z" ]; }; + } + ``` + )", + .fun = prim_zipAttrsWith, +}); + /************************************************************* * Lists @@ -2438,7 +2573,7 @@ static RegisterPrimOp primop_mapAttrs({ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nList); + v.mkBool(args[0]->type() == nList); } static RegisterPrimOp primop_isList({ @@ -2520,7 +2655,7 @@ static RegisterPrimOp primop_tail({ the argument isn’t a list or is an empty list. > **Warning** - > + > > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. @@ -2536,8 +2671,8 @@ static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & state.mkList(v, args[1]->listSize()); for (unsigned int n = 0; n < v.listSize(); ++n) - mkApp(*(v.listElems()[n] = state.allocValue()), - *args[0], *args[1]->listElems()[n]); + (v.listElems()[n] = state.allocValue())->mkApp( + args[0], args[1]->listElems()[n]); } static RegisterPrimOp primop_map({ @@ -2601,12 +2736,12 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value { bool res = false; state.forceList(*args[1], pos); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) - if (state.eqValues(*args[0], *args[1]->listElems()[n])) { + for (auto elem : args[1]->listItems()) + if (state.eqValues(*args[0], *elem)) { res = true; break; } - mkBool(v, res); + v.mkBool(res); } static RegisterPrimOp primop_elem({ @@ -2639,7 +2774,7 @@ static RegisterPrimOp primop_concatLists({ static void prim_length(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceList(*args[0], pos); - mkInt(v, args[0]->listSize()); + v.mkInt(args[0]->listSize()); } static RegisterPrimOp primop_length({ @@ -2661,11 +2796,10 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, if (args[2]->listSize()) { Value * vCur = args[1]; - for (unsigned int n = 0; n < args[2]->listSize(); ++n) { - Value vTmp; - state.callFunction(*args[0], *vCur, vTmp, pos); + for (auto [n, elem] : enumerate(args[2]->listItems())) { + Value * vs []{vCur, elem}; vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); - state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); + state.callFunction(*args[0], 2, vs, *vCur, pos); } state.forceValue(v, pos); } else { @@ -2679,9 +2813,9 @@ static RegisterPrimOp primop_foldlStrict({ .args = {"op", "nul", "list"}, .doc = R"( Reduce a list by applying a binary operator, from left to right, - e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) + e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]` + evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. )", .fun = prim_foldlStrict, @@ -2693,16 +2827,16 @@ static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * arg state.forceList(*args[1], pos); Value vTmp; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos); + for (auto elem : args[1]->listItems()) { + state.callFunction(*args[0], *elem, vTmp, pos); bool res = state.forceBool(vTmp, pos); if (res == any) { - mkBool(v, any); + v.mkBool(any); return; } } - mkBool(v, !any); + v.mkBool(!any); } @@ -2749,9 +2883,9 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val state.mkList(v, len); for (unsigned int n = 0; n < (unsigned int) len; ++n) { - Value * arg = state.allocValue(); - mkInt(*arg, n); - mkApp(*(v.listElems()[n] = state.allocValue()), *args[0], *arg); + auto arg = state.allocValue(); + arg->mkInt(n); + (v.listElems()[n] = state.allocValue())->mkApp(args[0], arg); } } @@ -2786,17 +2920,16 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value v.listElems()[n] = args[1]->listElems()[n]; } - auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) - return CompareValues()(a, b); + return CompareValues(state)(a, b); - Value vTmp1, vTmp2; - state.callFunction(*args[0], *a, vTmp1, pos); - state.callFunction(vTmp1, *b, vTmp2, pos); - return state.forceBool(vTmp2, pos); + Value * vs[] = {a, b}; + Value vBool; + state.callFunction(*args[0], 2, vs, vBool, pos); + return state.forceBool(vBool, pos); }; /* FIXME: std::sort can segfault if the comparator is not a strict @@ -2846,21 +2979,21 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V wrong.push_back(vElem); } - state.mkAttrs(v, 2); + auto attrs = state.buildBindings(2); - Value * vRight = state.allocAttr(v, state.sRight); + auto & vRight = attrs.alloc(state.sRight); auto rsize = right.size(); - state.mkList(*vRight, rsize); + state.mkList(vRight, rsize); if (rsize) - memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize); + memcpy(vRight.listElems(), right.data(), sizeof(Value *) * rsize); - Value * vWrong = state.allocAttr(v, state.sWrong); + auto & vWrong = attrs.alloc(state.sWrong); auto wsize = wrong.size(); - state.mkList(*vWrong, wsize); + state.mkList(vWrong, wsize); if (wsize) - memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wsize); + memcpy(vWrong.listElems(), wrong.data(), sizeof(Value *) * wsize); - v.attrs->sort(); + v.mkAttrs(attrs); } static RegisterPrimOp primop_partition({ @@ -2886,6 +3019,58 @@ static RegisterPrimOp primop_partition({ .fun = prim_partition, }); +static void prim_groupBy(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + ValueVectorMap attrs; + + for (auto vElem : args[1]->listItems()) { + Value res; + state.callFunction(*args[0], *vElem, res, pos); + auto name = state.forceStringNoCtx(res, pos); + Symbol sym = state.symbols.create(name); + auto vector = attrs.try_emplace(sym, ValueVector()).first; + vector->second.push_back(vElem); + } + + auto attrs2 = state.buildBindings(attrs.size()); + + for (auto & i : attrs) { + auto & list = attrs2.alloc(i.first); + auto size = i.second.size(); + state.mkList(list, size); + memcpy(list.listElems(), i.second.data(), sizeof(Value *) * size); + } + + v.mkAttrs(attrs2.alreadySorted()); +} + +static RegisterPrimOp primop_groupBy({ + .name = "__groupBy", + .args = {"f", "list"}, + .doc = R"( + Groups elements of *list* together by the string returned from the + function *f* called on each element. It returns an attribute set + where each attribute value contains the elements of *list* that are + mapped to the same corresponding attribute name returned by *f*. + + For example, + + ```nix + builtins.groupBy (builtins.substring 0 1) ["foo" "bar" "baz"] + ``` + + evaluates to + + ```nix + { b = [ "bar" "baz" ]; f = [ "foo" ]; } + ``` + )", + .fun = prim_groupBy, +}); + static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceFunction(*args[0], pos); @@ -2938,9 +3123,9 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else - mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_add({ @@ -2957,9 +3142,9 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else - mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_sub({ @@ -2976,9 +3161,9 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else - mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_mul({ @@ -3003,7 +3188,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & }); if (args[0]->type() == nFloat || args[1]->type() == nFloat) { - mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); } else { NixInt i1 = state.forceInt(*args[0], pos); NixInt i2 = state.forceInt(*args[1], pos); @@ -3014,7 +3199,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & .errPos = pos }); - mkInt(v, i1 / i2); + v.mkInt(i1 / i2); } } @@ -3029,7 +3214,7 @@ static RegisterPrimOp primop_div({ static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_bitAnd({ @@ -3043,7 +3228,7 @@ static RegisterPrimOp primop_bitAnd({ static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_bitOr({ @@ -3057,7 +3242,7 @@ static RegisterPrimOp primop_bitOr({ static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_bitXor({ @@ -3073,8 +3258,8 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - CompareValues comp; - mkBool(v, comp(args[0], args[1])); + CompareValues comp{state}; + v.mkBool(comp(args[0], args[1])); } static RegisterPrimOp primop_lessThan({ @@ -3100,8 +3285,8 @@ static RegisterPrimOp primop_lessThan({ static void prim_toString(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - string s = state.coerceToString(pos, *args[0], context, true, false); - mkString(v, s, context); + auto s = state.coerceToString(pos, *args[0], context, true, false); + v.mkString(*s, context); } static RegisterPrimOp primop_toString({ @@ -3137,7 +3322,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V int start = state.forceInt(*args[0], pos); int len = state.forceInt(*args[1], pos); PathSet context; - string s = state.coerceToString(pos, *args[2], context); + auto s = state.coerceToString(pos, *args[2], context); if (start < 0) throw EvalError({ @@ -3145,7 +3330,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V .errPos = pos }); - mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); + v.mkString((unsigned int) start >= s->size() ? "" : s->substr(start, len), context); } static RegisterPrimOp primop_substring({ @@ -3171,8 +3356,8 @@ static RegisterPrimOp primop_substring({ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - string s = state.coerceToString(pos, *args[0], context); - mkInt(v, s.size()); + auto s = state.coerceToString(pos, *args[0], context); + v.mkInt(s->size()); } static RegisterPrimOp primop_stringLength({ @@ -3188,7 +3373,7 @@ static RegisterPrimOp primop_stringLength({ /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) { - string type = state.forceStringNoCtx(*args[0], pos); + auto type = state.forceStringNoCtx(*args[0], pos); std::optional<HashType> ht = parseHashType(type); if (!ht) throw Error({ @@ -3197,9 +3382,9 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, }); PathSet context; // discarded - string s = state.forceString(*args[1], context, pos); + auto s = state.forceString(*args[1], context, pos); - mkString(v, hashString(*ht, s).to_string(Base16, false)); + v.mkString(hashString(*ht, s).to_string(Base16, false)); } static RegisterPrimOp primop_hashString({ @@ -3215,7 +3400,18 @@ static RegisterPrimOp primop_hashString({ struct RegexCache { - std::unordered_map<std::string, std::regex> cache; + // TODO use C++20 transparent comparison when available + std::unordered_map<std::string_view, std::regex> cache; + std::list<std::string> keys; + + std::regex get(std::string_view re) + { + auto it = cache.find(re); + if (it != cache.end()) + return it->second; + keys.emplace_back(re); + return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second; + } }; std::shared_ptr<RegexCache> makeRegexCache() @@ -3229,16 +3425,14 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) try { - auto regex = state.regexCache->cache.find(re); - if (regex == state.regexCache->cache.end()) - regex = state.regexCache->cache.emplace(re, std::regex(re, std::regex::extended)).first; + auto regex = state.regexCache->get(re); PathSet context; - const std::string str = state.forceString(*args[1], context, pos); + const auto str = state.forceString(*args[1], context, pos); - std::smatch match; - if (!std::regex_match(str, match, regex->second)) { - mkNull(v); + std::cmatch match; + if (!std::regex_match(str.begin(), str.end(), match, regex)) { + v.mkNull(); return; } @@ -3247,9 +3441,9 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) state.mkList(v, len); for (size_t i = 0; i < len; ++i) { if (!match[i+1].matched) - mkNull(*(v.listElems()[i] = state.allocValue())); + (v.listElems()[i] = state.allocValue())->mkNull(); else - mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str()); + (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); } } catch (std::regex_error &e) { @@ -3306,58 +3500,55 @@ static RegisterPrimOp primop_match({ /* Split a string with a regular expression, and return a list of the non-matching parts interleaved by the lists of the matching groups. */ -static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v) +void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto re = state.forceStringNoCtx(*args[0], pos); try { - std::regex regex(re, std::regex::extended); + auto regex = state.regexCache->get(re); PathSet context; - const std::string str = state.forceString(*args[1], context, pos); + const auto str = state.forceString(*args[1], context, pos); - auto begin = std::sregex_iterator(str.begin(), str.end(), regex); - auto end = std::sregex_iterator(); + auto begin = std::cregex_iterator(str.begin(), str.end(), regex); + auto end = std::cregex_iterator(); // Any matches results are surrounded by non-matching results. const size_t len = std::distance(begin, end); state.mkList(v, 2 * len + 1); size_t idx = 0; - Value * elem; if (len == 0) { v.listElems()[idx++] = args[1]; return; } - for (std::sregex_iterator i = begin; i != end; ++i) { + for (auto i = begin; i != end; ++i) { assert(idx <= 2 * len + 1 - 3); - std::smatch match = *i; + auto match = *i; // Add a string for non-matched characters. - elem = v.listElems()[idx++] = state.allocValue(); - mkString(*elem, match.prefix().str().c_str()); + (v.listElems()[idx++] = state.allocValue())->mkString(match.prefix().str()); // Add a list for matched substrings. const size_t slen = match.size() - 1; - elem = v.listElems()[idx++] = state.allocValue(); + auto elem = v.listElems()[idx++] = state.allocValue(); // Start at 1, beacause the first match is the whole string. state.mkList(*elem, slen); for (size_t si = 0; si < slen; ++si) { if (!match[si + 1].matched) - mkNull(*(elem->listElems()[si] = state.allocValue())); + (elem->listElems()[si] = state.allocValue())->mkNull(); else - mkString(*(elem->listElems()[si] = state.allocValue()), match[si + 1].str().c_str()); + (elem->listElems()[si] = state.allocValue())->mkString(match[si + 1].str()); } // Add a string for non-matched suffix characters. - if (idx == 2 * len) { - elem = v.listElems()[idx++] = state.allocValue(); - mkString(*elem, match.suffix().str().c_str()); - } + if (idx == 2 * len) + (v.listElems()[idx++] = state.allocValue())->mkString(match.suffix().str()); } + assert(idx == 2 * len + 1); } catch (std::regex_error &e) { @@ -3420,16 +3611,16 @@ static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * * auto sep = state.forceString(*args[0], context, pos); state.forceList(*args[1], pos); - string res; + std::string res; res.reserve((args[1]->listSize() + 32) * sep.size()); bool first = true; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + for (auto elem : args[1]->listItems()) { if (first) first = false; else res += sep; - res += state.coerceToString(pos, *args[1]->listElems()[n], context); + res += *state.coerceToString(pos, *elem, context); } - mkString(v, res, context); + v.mkString(res, context); } static RegisterPrimOp primop_concatStringsSep({ @@ -3453,23 +3644,23 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar .errPos = pos }); - vector<string> from; + std::vector<std::string> from; from.reserve(args[0]->listSize()); - for (unsigned int n = 0; n < args[0]->listSize(); ++n) - from.push_back(state.forceString(*args[0]->listElems()[n], pos)); + for (auto elem : args[0]->listItems()) + from.emplace_back(state.forceString(*elem, pos)); - vector<std::pair<string, PathSet>> to; + std::vector<std::pair<std::string, PathSet>> to; to.reserve(args[1]->listSize()); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + for (auto elem : args[1]->listItems()) { PathSet ctx; - auto s = state.forceString(*args[1]->listElems()[n], ctx, pos); - to.push_back(std::make_pair(std::move(s), std::move(ctx))); + auto s = state.forceString(*elem, ctx, pos); + to.emplace_back(s, std::move(ctx)); } PathSet context; auto s = state.forceString(*args[2], context, pos); - string res; + std::string res; // Loops one past last character to handle the case where 'from' contains an empty string. for (size_t p = 0; p <= s.size(); ) { bool found = false; @@ -3498,7 +3689,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar } } - mkString(v, res, context); + v.mkString(res, context); } static RegisterPrimOp primop_replaceStrings({ @@ -3525,12 +3716,12 @@ static RegisterPrimOp primop_replaceStrings({ static void prim_parseDrvName(EvalState & state, const Pos & pos, Value * * args, Value & v) { - string name = state.forceStringNoCtx(*args[0], pos); + auto name = state.forceStringNoCtx(*args[0], pos); DrvName parsed(name); - state.mkAttrs(v, 2); - mkString(*state.allocAttr(v, state.sName), parsed.name); - mkString(*state.allocAttr(v, state.symbols.create("version")), parsed.version); - v.attrs->sort(); + auto attrs = state.buildBindings(2); + attrs.alloc(state.sName).mkString(parsed.name); + attrs.alloc("version").mkString(parsed.version); + v.mkAttrs(attrs); } static RegisterPrimOp primop_parseDrvName({ @@ -3549,9 +3740,9 @@ static RegisterPrimOp primop_parseDrvName({ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * args, Value & v) { - string version1 = state.forceStringNoCtx(*args[0], pos); - string version2 = state.forceStringNoCtx(*args[1], pos); - mkInt(v, compareVersions(version1, version2)); + auto version1 = state.forceStringNoCtx(*args[0], pos); + auto version2 = state.forceStringNoCtx(*args[1], pos); + v.mkInt(compareVersions(version1, version2)); } static RegisterPrimOp primop_compareVersions({ @@ -3569,21 +3760,18 @@ static RegisterPrimOp primop_compareVersions({ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v) { - string version = state.forceStringNoCtx(*args[0], pos); + auto version = state.forceStringNoCtx(*args[0], pos); auto iter = version.cbegin(); Strings components; while (iter != version.cend()) { auto component = nextComponent(iter, version.cend()); if (component.empty()) break; - components.emplace_back(std::move(component)); + components.emplace_back(component); } state.mkList(v, components.size()); - unsigned int n = 0; - for (auto & component : components) { - auto listElem = v.listElems()[n++] = state.allocValue(); - mkString(*listElem, std::move(component)); - } + for (const auto & [n, component] : enumerate(components)) + (v.listElems()[n] = state.allocValue())->mkString(std::move(component)); } static RegisterPrimOp primop_splitVersion({ @@ -3633,37 +3821,37 @@ void EvalState::createBaseEnv() Value v; /* `builtins' must be first! */ - mkAttrs(v, 128); + v.mkAttrs(buildBindings(128).finish()); addConstant("builtins", v); - mkBool(v, true); + v.mkBool(true); addConstant("true", v); - mkBool(v, false); + v.mkBool(false); addConstant("false", v); - mkNull(v); + v.mkNull(); addConstant("null", v); if (!evalSettings.pureEval) { - mkInt(v, time(0)); + v.mkInt(time(0)); addConstant("__currentTime", v); - mkString(v, settings.thisSystem.get()); + v.mkString(settings.thisSystem.get()); addConstant("__currentSystem", v); } - mkString(v, nixVersion); + v.mkString(nixVersion); addConstant("__nixVersion", v); - mkString(v, store->storeDir); + v.mkString(store->storeDir); addConstant("__storeDir", v); /* Language version. This should be increased every time a new language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ - mkInt(v, 5); + v.mkInt(6); addConstant("__langVersion", v); // Miscellaneous @@ -3676,11 +3864,10 @@ void EvalState::createBaseEnv() mkList(v, searchPath.size()); int n = 0; for (auto & i : searchPath) { - auto v2 = v.listElems()[n++] = allocValue(); - mkAttrs(*v2, 2); - mkString(*allocAttr(*v2, symbols.create("path")), i.second); - mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); - v2->attrs->sort(); + auto attrs = buildBindings(2); + attrs.alloc("path").mkString(i.second); + attrs.alloc("prefix").mkString(i.first); + (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } addConstant("__nixPath", v); @@ -3690,21 +3877,30 @@ void EvalState::createBaseEnv() .fun = primOp.fun, .arity = std::max(primOp.args.size(), primOp.arity), .name = symbols.create(primOp.name), - .args = std::move(primOp.args), + .args = primOp.args, .doc = primOp.doc, }); /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ sDerivationNix = symbols.create("//builtin/derivation.nix"); - eval(parse( - #include "primops/derivation.nix.gen.hh" - , foFile, sDerivationNix, "/", staticBaseEnv), v); - addConstant("derivation", v); + auto vDerivation = allocValue(); + addConstant("derivation", vDerivation); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); + + staticBaseEnv.sort(); + + /* Note: we have to initialize the 'derivation' constant *after* + building baseEnv/staticBaseEnv because it uses 'builtins'. */ + char code[] = + #include "primops/derivation.nix.gen.hh" + // the parser needs two NUL bytes as terminators; one of them + // is implied by being a C string. + "\0"; + eval(parse(code, sizeof(code), foFile, sDerivationNix, "/", staticBaseEnv), *vDerivation); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 31cf812b4..3701bd442 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -7,8 +7,8 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - string s = state.coerceToString(pos, *args[0], context); - mkString(v, s, PathSet()); + auto s = state.coerceToString(pos, *args[0], context); + v.mkString(*s); } static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); @@ -18,7 +18,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, { PathSet context; state.forceString(*args[0], context, pos); - mkBool(v, !context.empty()); + v.mkBool(!context.empty()); } static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); @@ -33,13 +33,13 @@ static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - string s = state.coerceToString(pos, *args[0], context); + auto s = state.coerceToString(pos, *args[0], context); PathSet context2; for (auto & p : context) - context2.insert(p.at(0) == '=' ? string(p, 1) : p); + context2.insert(p.at(0) == '=' ? std::string(p, 1) : p); - mkString(v, s, context2); + v.mkString(*s, context2); } static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); @@ -76,13 +76,13 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, auto contextInfos = std::map<Path, ContextInfo>(); for (const auto & p : context) { Path drv; - string output; + std::string output; const Path * path = &p; if (p.at(0) == '=') { - drv = string(p, 1); + drv = std::string(p, 1); path = &drv; } else if (p.at(0) == '!') { - std::pair<string, string> ctx = decodeContext(p); + std::pair<std::string, std::string> ctx = decodeContext(p); drv = ctx.first; output = ctx.second; path = &drv; @@ -103,28 +103,26 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, } } - state.mkAttrs(v, contextInfos.size()); + auto attrs = state.buildBindings(contextInfos.size()); auto sPath = state.symbols.create("path"); auto sAllOutputs = state.symbols.create("allOutputs"); for (const auto & info : contextInfos) { - auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first)); - state.mkAttrs(infoVal, 3); + auto infoAttrs = state.buildBindings(3); if (info.second.path) - mkBool(*state.allocAttr(infoVal, sPath), true); + infoAttrs.alloc(sPath).mkBool(true); if (info.second.allOutputs) - mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + infoAttrs.alloc(sAllOutputs).mkBool(true); if (!info.second.outputs.empty()) { - auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); + auto & outputsVal = infoAttrs.alloc(state.sOutputs); state.mkList(outputsVal, info.second.outputs.size()); - size_t i = 0; - for (const auto & output : info.second.outputs) { - mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); - } + for (const auto & [i, output] : enumerate(info.second.outputs)) + (outputsVal.listElems()[i] = state.allocValue())->mkString(output); } - infoVal.attrs->sort(); + attrs.alloc(info.first).mkAttrs(infoAttrs); } - v.attrs->sort(); + + v.mkAttrs(attrs); } static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); @@ -168,7 +166,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg .errPos = *i.pos }); } - context.insert("=" + string(i.name)); + context.insert("=" + std::string(i.name)); } } @@ -181,14 +179,14 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg .errPos = *i.pos }); } - for (unsigned int n = 0; n < iter->value->listSize(); ++n) { - auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); - context.insert("!" + name + "!" + string(i.name)); + for (auto elem : iter->value->listItems()) { + auto name = state.forceStringNoCtx(*elem, *iter->pos); + context.insert(concatStrings("!", name, "!", i.name)); } } } - mkString(v, orig, context); + v.mkString(orig, context); } static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 3f88ccb91..b7f715859 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -12,24 +12,24 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar std::string url; std::optional<Hash> rev; std::optional<std::string> ref; - std::string name = "source"; + std::string_view name = "source"; PathSet context; - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); for (auto & attr : *args[0]->attrs) { - string n(attr.name); + std::string_view n(attr.name); if (n == "url") - url = state.coerceToString(*attr.pos, *attr.value, context, false, false); + url = state.coerceToString(*attr.pos, *attr.value, context, false, false).toOwned(); else if (n == "rev") { // Ugly: unlike fetchGit, here the "rev" attribute can // be both a revision or a branch/tag name. auto value = state.forceStringNoCtx(*attr.value, *attr.pos); - if (std::regex_match(value, revRegex)) + if (std::regex_match(value.begin(), value.end(), revRegex)) rev = Hash::parseAny(value, htSHA1); else ref = value; @@ -50,7 +50,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar }); } else - url = state.coerceToString(pos, *args[0], context, false, false); + url = state.coerceToString(pos, *args[0], context, false, false).toOwned(); // FIXME: git externals probably can be used to bypass the URI // whitelist. Ah well. @@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar fetchers::Attrs attrs; attrs.insert_or_assign("type", "hg"); attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); - attrs.insert_or_assign("name", name); + attrs.insert_or_assign("name", std::string(name)); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); auto input = fetchers::Input::fromAttrs(std::move(attrs)); @@ -70,22 +70,21 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // FIXME: use name auto [tree, input2] = input.fetch(state.store); - state.mkAttrs(v, 8); + auto attrs2 = state.buildBindings(8); auto storePath = state.store->printStorePath(tree.storePath); - mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + attrs2.alloc(state.sOutPath).mkString(storePath, {storePath}); if (input2.getRef()) - mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef()); + attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. auto rev2 = input2.getRev().value_or(Hash(htSHA1)); - mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); + attrs2.alloc("rev").mkString(rev2.gitRev()); + attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12)); if (auto revCount = input2.getRevCount()) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); - v.attrs->sort(); + attrs2.alloc("revCount").mkInt(*revCount); + v.mkAttrs(attrs2); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); } static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 06bdec003..9c2da2178 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -19,54 +19,53 @@ void emitTreeAttrs( bool emptyRevFallback, bool forceDirty) { - assert(input.isImmutable()); + assert(input.isLocked()); - state.mkAttrs(v, 8); + auto attrs = state.buildBindings(8); auto storePath = state.store->printStorePath(tree.storePath); - mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + attrs.alloc(state.sOutPath).mkString(storePath, {storePath}); // FIXME: support arbitrary input attributes. auto narHash = input.getNarHash(); assert(narHash); - mkString(*state.allocAttr(v, state.symbols.create("narHash")), - narHash->to_string(SRI, true)); + attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); if (input.getType() == "git") - mkBool(*state.allocAttr(v, state.symbols.create("submodules")), + attrs.alloc("submodules").mkBool( fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); if (!forceDirty) { if (auto rev = input.getRev()) { - mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); + attrs.alloc("rev").mkString(rev->gitRev()); + attrs.alloc("shortRev").mkString(rev->gitShortRev()); } else if (emptyRevFallback) { // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev auto emptyHash = Hash(htSHA1); - mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev()); + attrs.alloc("rev").mkString(emptyHash.gitRev()); + attrs.alloc("shortRev").mkString(emptyHash.gitShortRev()); } if (auto revCount = input.getRevCount()) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); + attrs.alloc("revCount").mkInt(*revCount); else if (emptyRevFallback) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0); + attrs.alloc("revCount").mkInt(0); } if (auto lastModified = input.getLastModified()) { - mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); - mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")), + attrs.alloc("lastModified").mkInt(*lastModified); + attrs.alloc("lastModifiedDate").mkString( fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S"))); } - v.attrs->sort(); + v.mkAttrs(attrs); } -std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file") +std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") { state.checkURI(uri); return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; @@ -74,53 +73,66 @@ std::string fixURI(std::string uri, EvalState &state, const std::string & defaul std::string fixURIForGit(std::string uri, EvalState & state) { - static std::regex scp_uri("([^/].*)@(.*):(.*)"); + /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + * them by removing the `:` and assuming a scheme of `ssh://` + * */ + static std::regex scp_uri("([^/]*)@(.*):(.*)"); if (uri[0] != '/' && std::regex_match(uri, scp_uri)) return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); else return fixURI(uri, state); } -void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) -{ - string n(name); - attrs.emplace(name, n == "url" ? fixURI(v, state) : v); -} - struct FetchTreeParams { bool emptyRevFallback = false; bool allowNameArgument = false; }; static void fetchTree( - EvalState &state, - const Pos &pos, - Value **args, - Value &v, - const std::optional<std::string> type, + EvalState & state, + const Pos & pos, + Value * * args, + Value & v, + std::optional<std::string> type, const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; PathSet context; - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); fetchers::Attrs attrs; + if (auto aType = args[0]->attrs->get(state.sType)) { + if (type) + throw Error({ + .msg = hintfmt("unexpected attribute 'type'"), + .errPos = pos + }); + type = state.forceStringNoCtx(*aType->value, *aType->pos); + } else if (!type) + throw Error({ + .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .errPos = pos + }); + + attrs.emplace("type", type.value()); + for (auto & attr : *args[0]->attrs) { - state.forceValue(*attr.value); - if (attr.value->type() == nPath || attr.value->type() == nString) - addURI( - state, - attrs, - attr.name, - state.coerceToString(*attr.pos, *attr.value, context, false, false) - ); - else if (attr.value->type() == nString) - addURI(state, attrs, attr.name, attr.value->string.s); + if (attr.name == state.sType) continue; + state.forceValue(*attr.value, *attr.pos); + if (attr.value->type() == nPath || attr.value->type() == nString) { + auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false).toOwned(); + attrs.emplace(attr.name, + attr.name == "url" + ? type == "git" + ? fixURIForGit(s, state) + : fixURI(s, state) + : s); + } else if (attr.value->type() == nBool) attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean}); else if (attr.value->type() == nInt) @@ -130,15 +142,6 @@ static void fetchTree( attr.name, showType(*attr.value)); } - if (type) - attrs.emplace("type", type.value()); - - if (!attrs.count("type")) - throw Error({ - .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), - .errPos = pos - }); - if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) throw Error({ @@ -146,10 +149,9 @@ static void fetchTree( .errPos = pos }); - input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - auto url = state.coerceToString(pos, *args[0], context, false, false); + auto url = state.coerceToString(pos, *args[0], context, false, false).toOwned(); if (type == "git") { fetchers::Attrs attrs; @@ -164,20 +166,19 @@ static void fetchTree( if (!evalSettings.pureEval && !input.isDirect()) input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input.isImmutable()) - throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos); + if (evalSettings.pureEval && !input.isLocked()) + throw Error("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", pos); auto [tree, input2] = input.fetch(state.store); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) { - settings.requireExperimentalFeature("flakes"); + settings.requireExperimentalFeature(Xp::Flakes); fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } @@ -185,19 +186,19 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, - const string & who, bool unpack, std::string name) + const std::string & who, bool unpack, std::string name) { std::optional<std::string> url; std::optional<Hash> expectedHash; - state.forceValue(*args[0]); + state.forceValue(*args[0], pos); if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); for (auto & attr : *args[0]->attrs) { - string n(attr.name); + std::string n(attr.name); if (n == "url") url = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "sha256") @@ -229,27 +230,36 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, if (evalSettings.pureEval && !expectedHash) throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + // early exit if pinned and already in the store + if (expectedHash && expectedHash->type == htSHA256) { + auto expectedPath = + unpack + ? state.store->makeFixedOutputPath(FileIngestionMethod::Recursive, *expectedHash, name, {}) + : state.store->makeFixedOutputPath(FileIngestionMethod::Flat, *expectedHash, name, {}); + + if (state.store->isValidPath(expectedPath)) { + state.allowAndSetStorePathString(expectedPath, v); + return; + } + } + + // TODO: fetching may fail, yet the path may be substitutable. + // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; - auto realPath = state.store->toRealPath(storePath); - if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, realPath); + : hashFile(htSHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } - if (state.allowedPaths) - state.allowedPaths->insert(realPath); - - auto path = state.store->printStorePath(storePath); - mkString(v, path, PathSet({path})); + state.allowAndSetStorePathString(storePath, v); } static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -291,13 +301,13 @@ static RegisterPrimOp primop_fetchTarball({ stdenv.mkDerivation { … } ``` - The fetched tarball is cached for a certain amount of time (1 hour - by default) in `~/.cache/nix/tarballs/`. You can change the cache - timeout either on the command line with `--option tarball-ttl number - of seconds` or in the Nix configuration file with this option: ` - number of seconds to cache `. + The fetched tarball is cached for a certain amount of time (1 + hour by default) in `~/.cache/nix/tarballs/`. You can change the + cache timeout either on the command line with `--tarball-ttl` + *number-of-seconds* or in the Nix configuration file by adding + the line `tarball-ttl = ` *number-of-seconds*. - Note that when obtaining the hash with ` nix-prefetch-url ` the + Note that when obtaining the hash with `nix-prefetch-url` the option `--unpack` is required. This function can also verify the contents against a hash. In that @@ -397,7 +407,7 @@ static RegisterPrimOp primop_fetchGit({ ``` > **Note** - > + > > It is nice to always specify the branch which a revision > belongs to. Without the branch being specified, the fetcher > might fail if the default branch changes. Additionally, it can @@ -434,12 +444,12 @@ static RegisterPrimOp primop_fetchGit({ ``` > **Note** - > + > > Nix will refetch the branch in accordance with > the option `tarball-ttl`. > **Note** - > + > > This behavior is disabled in *Pure evaluation mode*. )", .fun = prim_fetchGit, diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4c6682dfd..dd4280030 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -1,86 +1,76 @@ #include "primops.hh" #include "eval-inline.hh" -#include "../../cpptoml/cpptoml.h" +#include "../../toml11/toml.hpp" namespace nix { -static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & val) { - using namespace cpptoml; - auto toml = state.forceStringNoCtx(*args[0], pos); - std::istringstream tomlStream(toml); + std::istringstream tomlStream(std::string{toml}); - std::function<void(Value &, std::shared_ptr<base>)> visit; + std::function<void(Value &, toml::value)> visit; - visit = [&](Value & v, std::shared_ptr<base> t) { + visit = [&](Value & v, toml::value t) { - if (auto t2 = t->as_table()) { + switch(t.type()) + { + case toml::value_t::table: + { + auto table = toml::get<toml::table>(t); - size_t size = 0; - for (auto & i : *t2) { (void) i; size++; } + size_t size = 0; + for (auto & i : table) { (void) i; size++; } - state.mkAttrs(v, size); + auto attrs = state.buildBindings(size); - for (auto & i : *t2) { - auto & v2 = *state.allocAttr(v, state.symbols.create(i.first)); + for(auto & elem : table) + visit(attrs.alloc(elem.first), elem.second); - if (auto i2 = i.second->as_table_array()) { - size_t size2 = i2->get().size(); - state.mkList(v2, size2); - for (size_t j = 0; j < size2; ++j) - visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]); + v.mkAttrs(attrs); } - else - visit(v2, i.second); - } - - v.attrs->sort(); - } - - else if (auto t2 = t->as_array()) { - size_t size = t2->get().size(); - - state.mkList(v, size); - - for (size_t i = 0; i < size; ++i) - visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); - } - - // Handle cases like 'a = [[{ a = true }]]', which IMHO should be - // parsed as a array containing an array containing a table, - // but instead are parsed as an array containing a table array - // containing a table. - else if (auto t2 = t->as_table_array()) { - size_t size = t2->get().size(); - - state.mkList(v, size); - - for (size_t j = 0; j < size; ++j) - visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); - } + break;; + case toml::value_t::array: + { + auto array = toml::get<std::vector<toml::value>>(t); + + size_t size = array.size(); + state.mkList(v, size); + for (size_t i = 0; i < size; ++i) + visit(*(v.listElems()[i] = state.allocValue()), array[i]); + } + break;; + case toml::value_t::boolean: + v.mkBool(toml::get<bool>(t)); + break;; + case toml::value_t::integer: + v.mkInt(toml::get<int64_t>(t)); + break;; + case toml::value_t::floating: + v.mkFloat(toml::get<NixFloat>(t)); + break;; + case toml::value_t::string: + v.mkString(toml::get<std::string>(t)); + break;; + case toml::value_t::local_datetime: + case toml::value_t::offset_datetime: + case toml::value_t::local_date: + case toml::value_t::local_time: + // We fail since Nix doesn't have date and time types + throw std::runtime_error("Dates and times are not supported"); + break;; + case toml::value_t::empty: + v.mkNull(); + break;; - else if (t->is_value()) { - if (auto val = t->as<int64_t>()) - mkInt(v, val->get()); - else if (auto val = t->as<NixFloat>()) - mkFloat(v, val->get()); - else if (auto val = t->as<bool>()) - mkBool(v, val->get()); - else if (auto val = t->as<std::string>()) - mkString(v, val->get()); - else - throw EvalError("unsupported value type in TOML"); } - - else abort(); }; try { - visit(v, parser(tomlStream).parse()); - } catch (std::runtime_error & e) { + visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */)); + } catch (std::exception & e) { // TODO: toml::syntax_error throw EvalError({ .msg = hintfmt("while parsing a TOML string: %s", e.what()), .errPos = pos diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 4eb6dac81..48d20c29d 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -1,7 +1,8 @@ #pragma once +#include <list> #include <map> -#include <unordered_set> +#include <unordered_map> #include "types.hh" @@ -16,8 +17,8 @@ namespace nix { class Symbol { private: - const string * s; // pointer into SymbolTable - Symbol(const string * s) : s(s) { }; + const std::string * s; // pointer into SymbolTable + Symbol(const std::string * s) : s(s) { }; friend class SymbolTable; public: @@ -70,15 +71,21 @@ public: class SymbolTable { private: - typedef std::unordered_set<string> Symbols; - Symbols symbols; + std::unordered_map<std::string_view, Symbol> symbols; + std::list<std::string> store; public: Symbol create(std::string_view s) { - // FIXME: avoid allocation if 's' already exists in the symbol table. - std::pair<Symbols::iterator, bool> res = symbols.emplace(std::string(s)); - return Symbol(&*res.first); + // Most symbols are looked up more than once, so we trade off insertion performance + // for lookup performance. + // TODO: could probably be done more efficiently with transparent Hash and Equals + // on the original implementation using unordered_set + auto it = symbols.find(s); + if (it != symbols.end()) return it->second; + + auto & rawSym = store.emplace_back(s); + return symbols.emplace(rawSym, Symbol(&rawSym)).first->second; } size_t size() const @@ -91,7 +98,7 @@ public: template<typename T> void dump(T callback) { - for (auto & s : symbols) + for (auto & s : store) callback(s); } }; diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index bfea24d40..517da4c01 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -10,11 +10,11 @@ namespace nix { void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context) + Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context) { checkInterrupt(); - if (strict) state.forceValue(v); + if (strict) state.forceValue(v, pos); switch (v.type()) { @@ -40,7 +40,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nAttrs: { - auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); + auto maybeString = state.tryAttrsToString(pos, v, context, false, false); if (maybeString) { out.write(*maybeString); break; @@ -54,18 +54,18 @@ void printValueAsJSON(EvalState & state, bool strict, for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, placeholder, context); + printValueAsJSON(state, strict, *a.value, *a.pos, placeholder, context); } } else - printValueAsJSON(state, strict, *i->value, out, context); + printValueAsJSON(state, strict, *i->value, *i->pos, out, context); break; } case nList: { auto list(out.list()); - for (unsigned int n = 0; n < v.listSize(); ++n) { + for (auto elem : v.listItems()) { auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context); + printValueAsJSON(state, strict, *elem, pos, placeholder, context); } break; } @@ -79,18 +79,20 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nThunk: - throw TypeError("cannot convert %1% to JSON", showType(v)); - case nFunction: - throw TypeError("cannot convert %1% to JSON", showType(v)); + auto e = TypeError({ + .msg = hintfmt("cannot convert %1% to JSON", showType(v)), + .errPos = v.determinePos(pos) + }); + throw e.addTrace(pos, hintfmt("message for the trace")); } } void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context) + Value & v, const Pos & pos, std::ostream & str, PathSet & context) { JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, out, context); + printValueAsJSON(state, strict, v, pos, out, context); } void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 67fed6487..c2f797b29 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { class JSONPlaceholder; void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context); + Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context); void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context); + Value & v, const Pos & pos, std::ostream & str, PathSet & context); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 2ddc5f751..afeaf5694 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -9,7 +9,7 @@ namespace nix { -static XMLAttrs singletonAttrs(const string & name, const string & value) +static XMLAttrs singletonAttrs(const std::string & name, const std::string & value) { XMLAttrs attrs; attrs[name] = value; @@ -18,7 +18,8 @@ static XMLAttrs singletonAttrs(const string & name, const string & value) static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos); static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) @@ -46,17 +47,18 @@ static void showAttrs(EvalState & state, bool strict, bool location, XMLOpenElement _(doc, "attr", xmlAttrs); printValueAsXML(state, strict, location, - *a.value, doc, context, drvsSeen); + *a.value, doc, context, drvsSeen, *a.pos); } } static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) { checkInterrupt(); - if (strict) state.forceValue(v); + if (strict) state.forceValue(v, pos); switch (v.type()) { @@ -91,14 +93,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, Path drvPath; a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); + if (strict) state.forceValue(*a->value, *a->pos); if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); + if (strict) state.forceValue(*a->value, *a->pos); if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->string.s; } @@ -120,8 +122,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nList: { XMLOpenElement _(doc, "list"); - for (unsigned int n = 0; n < v.listSize(); ++n) - printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); + for (auto v2 : v.listItems()) + printValueAsXML(state, strict, location, *v2, doc, context, drvsSeen, pos); break; } @@ -135,12 +137,12 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (location) posToXML(xmlAttrs, v.lambda.fun->pos); XMLOpenElement _(doc, "function", xmlAttrs); - if (v.lambda.fun->matchAttrs) { + if (v.lambda.fun->hasFormals()) { XMLAttrs attrs; if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; XMLOpenElement _(doc, "attrspat", attrs); - for (auto & i : v.lambda.fun->formals->formals) + for (auto & i : v.lambda.fun->formals->lexicographicOrder()) doc.writeEmptyElement("attr", singletonAttrs("name", i.name)); } else doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg)); @@ -149,7 +151,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, } case nExternal: - v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); + v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); break; case nFloat: @@ -163,19 +165,20 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const + bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) const { doc.writeEmptyElement("unevaluated"); } void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context) + Value & v, std::ostream & out, PathSet & context, const Pos & pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); PathSet drvsSeen; - printValueAsXML(state, strict, location, v, doc, context, drvsSeen); + printValueAsXML(state, strict, location, v, doc, context, drvsSeen, pos); } diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..cc778a2cb 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -9,6 +9,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context); - + Value & v, std::ostream & out, PathSet & context, const Pos & pos); + } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index a1f131f9e..d0fa93e92 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,5 +1,7 @@ #pragma once +#include <cassert> + #include "symbol-table.hh" #if HAVE_BOEHMGC @@ -8,6 +10,8 @@ namespace nix { +class BindingsBuilder; + typedef enum { tInt = 1, @@ -73,20 +77,20 @@ class ExternalValueBase public: /* Return a simple string describing the type */ - virtual string showType() const = 0; + virtual std::string showType() const = 0; /* Return a string to be used in builtins.typeOf */ - virtual string typeOf() const = 0; + virtual std::string typeOf() const = 0; /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an - * error + * error. */ - virtual string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; + virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; /* Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ - virtual bool operator==(const ExternalValueBase & b) const; + virtual bool operator ==(const ExternalValueBase & b) const; /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ virtual void printValueAsJSON(EvalState & state, bool strict, @@ -94,7 +98,8 @@ class ExternalValueBase /* Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const; + XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) const; virtual ~ExternalValueBase() { @@ -109,8 +114,8 @@ struct Value private: InternalType internalType; -friend std::string showType(const Value & v); -friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v); + friend std::string showType(const Value & v); + friend void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v); public: @@ -232,6 +237,17 @@ public: string.context = context; } + void mkString(std::string_view s); + + void mkString(std::string_view s, const PathSet & context); + + void mkStringMove(const char * s, const PathSet & context); + + inline void mkString(const Symbol & s) + { + mkString(((const std::string &) s).c_str()); + } + inline void mkPath(const char * s) { clearValue(); @@ -239,6 +255,8 @@ public: path = s; } + void mkPath(std::string_view s); + inline void mkNull() { clearValue(); @@ -252,6 +270,8 @@ public: attrs = a; } + Value & mkAttrs(BindingsBuilder & bindings); + inline void mkList(size_t size) { clearValue(); @@ -341,7 +361,7 @@ public: return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size; } - Pos determinePos(const Pos &pos) const; + Pos determinePos(const Pos & pos) const; /* Check whether forcing this value requires a trivial amount of computation. In particular, function applications are @@ -349,54 +369,45 @@ public: bool isTrivial() const; std::vector<std::pair<Path, std::string>> getContext(); -}; - - - -// TODO: Remove these static functions, replace call sites with v.mk* instead -static inline void mkInt(Value & v, NixInt n) -{ - v.mkInt(n); -} - -static inline void mkFloat(Value & v, NixFloat n) -{ - v.mkFloat(n); -} - -static inline void mkBool(Value & v, bool b) -{ - v.mkBool(b); -} - -static inline void mkNull(Value & v) -{ - v.mkNull(); -} - -static inline void mkApp(Value & v, Value & left, Value & right) -{ - v.mkApp(&left, &right); -} - -static inline void mkString(Value & v, const Symbol & s) -{ - v.mkString(((const string &) s).c_str()); -} - - -void mkString(Value & v, const char * s); + auto listItems() + { + struct ListIterable + { + typedef Value * const * iterator; + iterator _begin, _end; + iterator begin() const { return _begin; } + iterator end() const { return _end; } + }; + assert(isList()); + auto begin = listElems(); + return ListIterable { begin, begin + listSize() }; + } -void mkPath(Value & v, const char * s); + auto listItems() const + { + struct ConstListIterable + { + typedef const Value * const * iterator; + iterator _begin, _end; + iterator begin() const { return _begin; } + iterator end() const { return _end; } + }; + assert(isList()); + auto begin = listElems(); + return ConstListIterable { begin, begin + listSize() }; + } +}; #if HAVE_BOEHMGC typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector; typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap; +typedef std::map<Symbol, ValueVector, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, ValueVector> > > ValueVectorMap; #else typedef std::vector<Value *> ValueVector; typedef std::map<Symbol, Value *> ValueMap; +typedef std::map<Symbol, ValueVector> ValueVectorMap; #endif |