diff options
Diffstat (limited to 'src/libexpr')
-rw-r--r-- | src/libexpr/attr-set.cc | 40 | ||||
-rw-r--r-- | src/libexpr/attr-set.hh | 40 | ||||
-rw-r--r-- | src/libexpr/common-eval-args.cc | 11 | ||||
-rw-r--r-- | src/libexpr/eval.cc | 148 | ||||
-rw-r--r-- | src/libexpr/eval.hh | 19 | ||||
-rw-r--r-- | src/libexpr/flake/flake.cc | 44 | ||||
-rw-r--r-- | src/libexpr/get-drvs.cc | 13 | ||||
-rw-r--r-- | src/libexpr/json-to-value.cc | 38 | ||||
-rw-r--r-- | src/libexpr/nixexpr.hh | 13 | ||||
-rw-r--r-- | src/libexpr/primops.cc | 541 | ||||
-rw-r--r-- | src/libexpr/primops/context.cc | 30 | ||||
-rw-r--r-- | src/libexpr/primops/fetchMercurial.cc | 14 | ||||
-rw-r--r-- | src/libexpr/primops/fetchTree.cc | 29 | ||||
-rw-r--r-- | src/libexpr/primops/fromTOML.cc | 110 | ||||
-rw-r--r-- | src/libexpr/value.hh | 56 |
15 files changed, 677 insertions, 469 deletions
diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index b6091c955..52ac47e9b 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -7,26 +7,19 @@ namespace nix { + /* Allocate a new array of attributes for an attribute set with a specific capacity. The space is implicitly reserved after the Bindings structure. */ Bindings * EvalState::allocBindings(size_t capacity) { + if (capacity == 0) + return &emptyBindings; if (capacity > std::numeric_limits<Bindings::size_t>::max()) throw Error("attribute set of size %d is too big", capacity); - return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); -} - - -void EvalState::mkAttrs(Value & v, size_t capacity) -{ - if (capacity == 0) { - v = vEmptySet; - return; - } - v.mkAttrs(allocBindings(capacity)); nrAttrsets++; nrAttrsInAttrsets += capacity; + return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); } @@ -41,15 +34,36 @@ Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) } -Value * EvalState::allocAttr(Value & vAttrs, const std::string & name) +Value * EvalState::allocAttr(Value & vAttrs, std::string_view name) { return allocAttr(vAttrs, symbols.create(name)); } +Value & BindingsBuilder::alloc(const Symbol & name, ptr<Pos> pos) +{ + auto value = state.allocValue(); + bindings->push_back(Attr(name, value, pos)); + return *value; +} + + +Value & BindingsBuilder::alloc(std::string_view name, ptr<Pos> pos) +{ + return alloc(state.symbols.create(name), pos); +} + + void Bindings::sort() { - std::sort(begin(), end()); + if (size_) std::sort(begin(), end()); +} + + +Value & Value::mkAttrs(BindingsBuilder & bindings) +{ + mkAttrs(bindings.finish()); + return *this; } diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 7d6ffc9f3..82c348287 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -113,5 +113,45 @@ public: friend class EvalState; }; +/* A wrapper around Bindings that ensures that its always in sorted + order at the end. The only way to consume a BindingsBuilder is to + call finish(), which sorts the bindings. */ +class BindingsBuilder +{ + Bindings * bindings; + +public: + + EvalState & state; + + BindingsBuilder(EvalState & state, Bindings * bindings) + : bindings(bindings), state(state) + { } + + void insert(Symbol name, Value * value, ptr<Pos> pos = ptr(&noPos)) + { + insert(Attr(name, value, pos)); + } + + void insert(const Attr & attr) + { + bindings->push_back(attr); + } + + Value & alloc(const Symbol & name, ptr<Pos> pos = ptr(&noPos)); + + Value & alloc(std::string_view name, ptr<Pos> pos = ptr(&noPos)); + + Bindings * finish() + { + bindings->sort(); + return bindings; + } + + Bindings * alreadySorted() + { + return bindings; + } +}; } diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index fb0932c00..fffca4ac5 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -73,17 +73,16 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { - Bindings * res = state.allocBindings(autoArgs.size()); + auto res = state.buildBindings(autoArgs.size()); for (auto & i : autoArgs) { - Value * v = state.allocValue(); + auto v = state.allocValue(); if (i.second[0] == 'E') state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath("."))); else - mkString(*v, string(i.second, 1)); - res->push_back(Attr(state.symbols.create(i.first), v)); + v->mkString(((std::string_view) i.second).substr(1)); + res.insert(state.symbols.create(i.first), v); } - res->sort(); - return res; + return res.finish(); } Path lookupFileArg(EvalState & state, string s) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index a95726f5f..61bccd6e2 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -36,6 +36,19 @@ namespace nix { +static char * allocString(size_t size) +{ + char * t; +#if HAVE_BOEHMGC + t = (char *) GC_MALLOC_ATOMIC(size); +#else + t = malloc(size); +#endif + if (!t) throw std::bad_alloc(); + return t; +} + + static char * dupString(const char * s) { char * t; @@ -145,7 +158,7 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu str << v.fpoint; break; default: - throw Error("invalid value"); + abort(); } active.erase(&v); @@ -413,6 +426,7 @@ EvalState::EvalState( , sSelf(symbols.create("self")) , sEpsilon(symbols.create("")) , repair(NoRepair) + , emptyBindings(0) , store(store) , buildStore(buildStore ? buildStore : store) , regexCache(makeRegexCache()) @@ -454,8 +468,6 @@ EvalState::EvalState( } } - vEmptySet.mkAttrs(allocBindings(0)); - createBaseEnv(); } @@ -613,7 +625,7 @@ Value * EvalState::addPrimOp(const string & name, auto vPrimOp = allocValue(); vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym }); Value v; - mkApp(v, *vPrimOp, *vPrimOp); + v.mkApp(vPrimOp, vPrimOp); return addConstant(name, v); } @@ -635,7 +647,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) auto vPrimOp = allocValue(); vPrimOp->mkPrimOp(new PrimOp(std::move(primOp))); Value v; - mkApp(v, *vPrimOp, *vPrimOp); + v.mkApp(vPrimOp, vPrimOp); return addConstant(primOp.name, v); } @@ -766,15 +778,14 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con } -void mkString(Value & v, const char * s) +void Value::mkString(std::string_view s) { - v.mkString(dupString(s)); + mkString(dupStringWithLen(s.data(), s.size())); } -Value & mkString(Value & v, std::string_view s, const PathSet & context) +static void copyContextToValue(Value & v, const PathSet & context) { - v.mkString(dupStringWithLen(s.data(), s.size())); if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) @@ -783,13 +794,24 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context) v.string.context[n++] = dupString(i.c_str()); v.string.context[n] = 0; } - return v; } +void Value::mkString(std::string_view s, const PathSet & context) +{ + mkString(s); + copyContextToValue(*this, context); +} -void mkPath(Value & v, const char * s) +void Value::mkStringMove(const char * s, const PathSet & context) { - v.mkPath(dupString(s)); + mkString(s); + copyContextToValue(*this, context); +} + + +void Value::mkPath(std::string_view s) +{ + mkPath(dupStringWithLen(s.data(), s.size())); } @@ -882,13 +904,13 @@ void EvalState::mkThunk_(Value & v, Expr * expr) void EvalState::mkPos(Value & v, ptr<Pos> pos) { if (pos->file.set()) { - mkAttrs(v, 3); - mkString(*allocAttr(v, sFile), pos->file); - mkInt(*allocAttr(v, sLine), pos->line); - mkInt(*allocAttr(v, sColumn), pos->column); - v.attrs->sort(); + auto attrs = buildBindings(3); + attrs.alloc(sFile).mkString(pos->file); + attrs.alloc(sLine).mkInt(pos->line); + attrs.alloc(sColumn).mkInt(pos->column); + v.mkAttrs(attrs); } else - mkNull(v); + v.mkNull(); } @@ -1067,8 +1089,8 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v) void ExprAttrs::eval(EvalState & state, Env & env, Value & v) { - state.mkAttrs(v, attrs.size() + dynamicAttrs.size()); - Env *dynamicEnv = &env; + v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish()); + auto dynamicEnv = &env; if (recursive) { /* Create a new environment that contains the attributes in @@ -1259,14 +1281,14 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { - mkBool(v, false); + v.mkBool(false); return; } else { vAttrs = j->value; } } - mkBool(v, true); + v.mkBool(true); } @@ -1341,7 +1363,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* Nope, so show the first unexpected argument to the user. */ for (auto & i : *args[0]->attrs) - if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) + if (!lambda.formals->argNames.count(i.name)) throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); abort(); // can't happen } @@ -1484,22 +1506,20 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) return; } - Value * actualArgs = allocValue(); - mkAttrs(*actualArgs, std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size())); + auto attrs = buildBindings(std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size())); if (fun.lambda.fun->formals->ellipsis) { // If the formals have an ellipsis (eg the function accepts extra args) pass // all available automatic arguments (which includes arguments specified on // the command line via --arg/--argstr) - for (auto& v : args) { - actualArgs->attrs->push_back(v); - } + for (auto & v : args) + attrs.insert(v); } else { // Otherwise, only pass the arguments that the function accepts for (auto & i : fun.lambda.fun->formals->formals) { Bindings::iterator j = args.find(i.name); if (j != args.end()) { - actualArgs->attrs->push_back(*j); + attrs.insert(*j); } else if (!i.def) { throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') @@ -1512,9 +1532,7 @@ https://nixos.org/manual/nix/stable/#ss-functions.)", i.name); } } - actualArgs->attrs->sort(); - - callFunction(fun, *actualArgs, res, noPos); + callFunction(fun, allocValue()->mkAttrs(attrs), res, noPos); } @@ -1549,7 +1567,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) void ExprOpNot::eval(EvalState & state, Env & env, Value & v) { - mkBool(v, !state.evalBool(env, e)); + v.mkBool(!state.evalBool(env, e)); } @@ -1557,7 +1575,7 @@ void ExprOpEq::eval(EvalState & state, Env & env, Value & v) { Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); - mkBool(v, state.eqValues(v1, v2)); + v.mkBool(state.eqValues(v1, v2)); } @@ -1565,25 +1583,25 @@ void ExprOpNEq::eval(EvalState & state, Env & env, Value & v) { Value v1; e1->eval(state, env, v1); Value v2; e2->eval(state, env, v2); - mkBool(v, !state.eqValues(v1, v2)); + v.mkBool(!state.eqValues(v1, v2)); } void ExprOpAnd::eval(EvalState & state, Env & env, Value & v) { - mkBool(v, state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); + v.mkBool(state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); } void ExprOpOr::eval(EvalState & state, Env & env, Value & v) { - mkBool(v, state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); + v.mkBool(state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); } void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) { - mkBool(v, !state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); + v.mkBool(!state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); } @@ -1598,7 +1616,7 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) if (v1.attrs->size() == 0) { v = v2; return; } if (v2.attrs->size() == 0) { v = v1; return; } - state.mkAttrs(v, v1.attrs->size() + v2.attrs->size()); + auto attrs = state.buildBindings(v1.attrs->size() + v2.attrs->size()); /* Merge the sets, preferring values from the second set. Make sure to keep the resulting vector in sorted order. */ @@ -1607,17 +1625,19 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) while (i != v1.attrs->end() && j != v2.attrs->end()) { if (i->name == j->name) { - v.attrs->push_back(*j); + attrs.insert(*j); ++i; ++j; } else if (i->name < j->name) - v.attrs->push_back(*i++); + attrs.insert(*i++); else - v.attrs->push_back(*j++); + attrs.insert(*j++); } - while (i != v1.attrs->end()) v.attrs->push_back(*i++); - while (j != v2.attrs->end()) v.attrs->push_back(*j++); + while (i != v1.attrs->end()) attrs.insert(*i++); + while (j != v2.attrs->end()) attrs.insert(*j++); + + v.mkAttrs(attrs.alreadySorted()); state.nrOpUpdateValuesCopied += v.attrs->size(); } @@ -1664,13 +1684,34 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) { PathSet context; - std::ostringstream s; + std::vector<std::string> s; + size_t sSize = 0; NixInt n = 0; NixFloat nf = 0; bool first = !forceString; ValueType firstType = nString; + const auto str = [&] { + std::string result; + result.reserve(sSize); + for (const auto & part : s) result += part; + return result; + }; + /* c_str() is not str().c_str() because we want to create a string + Value. allocating a GC'd string directly and moving it into a + Value lets us avoid an allocation and copy. */ + const auto c_str = [&] { + char * result = allocString(sSize + 1); + char * tmp = result; + for (const auto & part : s) { + memcpy(tmp, part.c_str(), part.size()); + tmp += part.size(); + } + *tmp = 0; + return result; + }; + for (auto & [i_pos, i] : *es) { Value vTmp; i->eval(state, env, vTmp); @@ -1700,26 +1741,29 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) nf += vTmp.fpoint; } else throwEvalError(i_pos, "cannot add %1% to a float", showType(vTmp)); - } else + } else { + if (s.empty()) s.reserve(es->size()); /* skip canonization of first path, which would only be not canonized in the first place if it's coming from a ./${foo} type path */ - s << state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first); + s.emplace_back( + state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first)); + sSize += s.back().size(); + } first = false; } if (firstType == nInt) - mkInt(v, n); + v.mkInt(n); else if (firstType == nFloat) - mkFloat(v, nf); + v.mkFloat(nf); else if (firstType == nPath) { if (!context.empty()) throwEvalError(pos, "a string that refers to a store path cannot be appended to a path"); - auto path = canonPath(s.str()); - mkPath(v, path.c_str()); + v.mkPath(canonPath(str())); } else - mkString(v, s.str(), context); + v.mkStringMove(c_str(), context); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 15925a6b4..850c5bae6 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -44,8 +44,6 @@ struct Env }; -Value & mkString(Value & v, std::string_view s, const PathSet & context = PathSet()); - void copyContext(const Value & v, PathSet & context); @@ -93,7 +91,7 @@ public: mode. */ std::optional<PathSet> allowedPaths; - Value vEmptySet; + Bindings emptyBindings; /* Store used to materialise .drv files. */ const ref<Store> store; @@ -339,12 +337,16 @@ public: Env & allocEnv(size_t size); Value * allocAttr(Value & vAttrs, const Symbol & name); - Value * allocAttr(Value & vAttrs, const std::string & name); + Value * allocAttr(Value & vAttrs, std::string_view name); Bindings * allocBindings(size_t capacity); + BindingsBuilder buildBindings(size_t capacity) + { + return BindingsBuilder(*this, allocBindings(capacity)); + } + void mkList(Value & v, size_t length); - void mkAttrs(Value & v, size_t capacity); void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, ptr<Pos> pos); @@ -353,7 +355,10 @@ public: /* Print statistics. */ void printStats(); - void realiseContext(const PathSet & context); + /* Realise the given context, and return a mapping from the placeholders + * used to construct the associated value to their final store path + */ + [[nodiscard]] StringMap realiseContext(const PathSet & context); private: @@ -394,6 +399,8 @@ private: friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v); + + friend struct Value; }; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index b15878d5c..190a128d7 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -251,6 +251,10 @@ static Flake getFlake( forceTrivialValue(state, *setting.value, *setting.pos); if (setting.value->type() == nString) flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)}); + else if (setting.value->type() == nPath) { + PathSet emptyContext = {}; + flake.config.settings.insert({setting.name, state.coerceToString(*setting.pos, *setting.value, emptyContext, false, true, true)}); + } else if (setting.value->type() == nInt) flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); else if (setting.value->type() == nBool) @@ -344,7 +348,8 @@ LockedFlake lockFlake( const InputPath & inputPathPrefix, std::shared_ptr<const Node> oldNode, const LockParent & parent, - const Path & parentPath)> + const Path & parentPath, + bool trustLock)> computeLocks; computeLocks = [&]( @@ -353,7 +358,8 @@ LockedFlake lockFlake( const InputPath & inputPathPrefix, std::shared_ptr<const Node> oldNode, const LockParent & parent, - const Path & parentPath) + const Path & parentPath, + bool trustLock) { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); @@ -464,14 +470,20 @@ LockedFlake lockFlake( .isFlake = (*lockedNode)->isFlake, }); } else if (auto follows = std::get_if<1>(&i.second)) { - auto o = input.overrides.find(i.first); - // If the override disappeared, we have to refetch the flake, - // since some of the inputs may not be present in the lockfile. - if (o == input.overrides.end()) { - mustRefetch = true; - // There's no point populating the rest of the fake inputs, - // since we'll refetch the flake anyways. - break; + if (! trustLock) { + // It is possible that the flake has changed, + // so we must confirm all the follows that are in the lockfile are also in the flake. + auto overridePath(inputPath); + overridePath.push_back(i.first); + auto o = overrides.find(overridePath); + // If the override disappeared, we have to refetch the flake, + // since some of the inputs may not be present in the lockfile. + if (o == overrides.end()) { + mustRefetch = true; + // There's no point populating the rest of the fake inputs, + // since we'll refetch the flake anyways. + break; + } } fakeInputs.emplace(i.first, FlakeInput { .follows = *follows, @@ -482,14 +494,14 @@ LockedFlake lockFlake( LockParent newParent { .path = inputPath, - .absolute = false + .absolute = true }; computeLocks( mustRefetch ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs : fakeInputs, - childNode, inputPath, oldLock, newParent, parentPath); + childNode, inputPath, oldLock, newParent, parentPath, !mustRefetch); } else { /* We need to create a new lock file entry. So fetch @@ -546,7 +558,7 @@ LockedFlake lockFlake( ? std::dynamic_pointer_cast<const Node>(oldLock) : LockFile::read( inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root, - newParent, localPath); + newParent, localPath, false); } else { @@ -574,7 +586,7 @@ LockedFlake lockFlake( computeLocks( flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath); + lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath, false); for (auto & i : lockFlags.inputOverrides) if (!overridesUsed.count(i.first)) @@ -674,7 +686,7 @@ void callFlake(EvalState & state, auto vTmp1 = state.allocValue(); auto vTmp2 = state.allocValue(); - mkString(*vLocks, lockedFlake.lockFile.to_string()); + vLocks->mkString(lockedFlake.lockFile.to_string()); emitTreeAttrs( state, @@ -684,7 +696,7 @@ void callFlake(EvalState & state, false, lockedFlake.flake.forceDirty); - mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); + vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); if (!state.vCallFlake) { state.vCallFlake = allocRootValue(state.allocValue()); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index ed4c47fbb..25fd9b949 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -254,15 +254,14 @@ bool DrvInfo::queryMetaBool(const string & name, bool def) void DrvInfo::setMeta(const string & name, Value * v) { getMeta(); - Bindings * old = meta; - meta = state->allocBindings(1 + (old ? old->size() : 0)); + auto attrs = state->buildBindings(1 + (meta ? meta->size() : 0)); Symbol sym = state->symbols.create(name); - if (old) - for (auto i : *old) + if (meta) + for (auto i : *meta) if (i.name != sym) - meta->push_back(i); - if (v) meta->push_back(Attr(sym, v)); - meta->sort(); + attrs.insert(i); + if (v) attrs.insert(sym, v); + meta = attrs.finish(); } diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 9ca5ac86d..88716250c 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -37,10 +37,10 @@ class JSONSax : nlohmann::json_sax<json> { ValueMap attrs; std::unique_ptr<JSONState> resolve(EvalState & state) override { - Value & v = parent->value(state); - state.mkAttrs(v, attrs.size()); + auto attrs2 = state.buildBindings(attrs.size()); for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second)); + attrs2.insert(i.first, i.second); + parent->value(state).mkAttrs(attrs2.alreadySorted()); return std::move(parent); } void add() override { v = nullptr; } @@ -76,45 +76,51 @@ class JSONSax : nlohmann::json_sax<json> { EvalState & state; std::unique_ptr<JSONState> rs; - template<typename T, typename... Args> inline bool handle_value(T f, Args... args) - { - f(rs->value(state), args...); - rs->add(); - return true; - } - public: JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; bool null() { - return handle_value(mkNull); + rs->value(state).mkNull(); + rs->add(); + return true; } bool boolean(bool val) { - return handle_value(mkBool, val); + rs->value(state).mkBool(val); + rs->add(); + return true; } bool number_integer(number_integer_t val) { - return handle_value(mkInt, val); + rs->value(state).mkInt(val); + rs->add(); + return true; } bool number_unsigned(number_unsigned_t val) { - return handle_value(mkInt, val); + rs->value(state).mkInt(val); + rs->add(); + return true; } bool number_float(number_float_t val, const string_t & s) { - return handle_value(mkFloat, val); + rs->value(state).mkFloat(val); + rs->add(); + return true; } bool string(string_t & val) { - return handle_value<void(Value&, const char*)>(mkString, val.c_str()); + rs->value(state).mkString(val); + rs->add(); + return true; } + #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 bool binary(binary_t&) { diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index c013f5deb..0a60057e5 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -94,7 +94,7 @@ struct ExprInt : Expr { NixInt n; Value v; - ExprInt(NixInt n) : n(n) { mkInt(v, n); }; + ExprInt(NixInt n) : n(n) { v.mkInt(n); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; @@ -103,7 +103,7 @@ struct ExprFloat : Expr { NixFloat nf; Value v; - ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); }; + ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; @@ -112,7 +112,7 @@ struct ExprString : Expr { Symbol s; Value v; - ExprString(const Symbol & s) : s(s) { mkString(v, s); }; + ExprString(const Symbol & s) : s(s) { v.mkString(s); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; @@ -368,6 +368,13 @@ struct StaticEnv [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; }); } + void deduplicate() + { + const auto last = std::unique(vars.begin(), vars.end(), + [] (const Vars::value_type & a, const Vars::value_type & b) { return a.first == b.first; }); + vars.erase(last, vars.end()); + } + Vars::const_iterator find(const Symbol & name) const { Vars::value_type key(name, 0); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 852317aa3..b819918ad 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -35,9 +35,10 @@ namespace nix { InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} -void EvalState::realiseContext(const PathSet & context) +StringMap EvalState::realiseContext(const PathSet & context) { std::vector<DerivedPath::Built> drvs; + StringMap res; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -46,10 +47,12 @@ void EvalState::realiseContext(const PathSet & context) throw InvalidPathError(store->printStorePath(ctx)); if (!outputName.empty() && ctx.isDerivation()) { drvs.push_back({ctx, {outputName}}); + } else { + res.insert_or_assign(ctxS, ctxS); } } - if (drvs.empty()) return; + if (drvs.empty()) return {}; if (!evalSettings.enableImportFromDerivation) throw Error( @@ -61,19 +64,53 @@ void EvalState::realiseContext(const PathSet & context) for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); store->buildPaths(buildReqs); + /* Get all the output paths corresponding to the placeholders we had */ + for (auto & [drvPath, outputs] : drvs) { + auto outputPaths = store->queryDerivationOutputMap(drvPath); + for (auto & outputName : outputs) { + if (outputPaths.count(outputName) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store->printStorePath(drvPath), outputName); + res.insert_or_assign( + downstreamPlaceholder(*store, drvPath, outputName), + store->printStorePath(outputPaths.at(outputName)) + ); + } + } + /* Add the output of this derivations to the allowed paths. */ if (allowedPaths) { - for (auto & [drvPath, outputs] : drvs) { - auto outputPaths = store->queryDerivationOutputMap(drvPath); - for (auto & outputName : outputs) { - if (outputPaths.count(outputName) == 0) - throw Error("derivation '%s' does not have an output named '%s'", - store->printStorePath(drvPath), outputName); - allowPath(outputPaths.at(outputName)); - } + for (auto & [_placeholder, outputPath] : res) { + allowPath(store->toRealPath(outputPath)); } } + + return res; +} + +struct RealisePathFlags { + // Whether to check whether the path is a valid absolute path + bool requireAbsolutePath = true; + // Whether to check that the path is allowed in pure eval mode + bool checkForPureEval = true; +}; + +static Path realisePath(EvalState & state, const Pos & pos, Value & v, const RealisePathFlags flags = {}) +{ + PathSet context; + + Path path = flags.requireAbsolutePath + ? state.coerceToPath(pos, v, context) + : state.coerceToString(pos, v, context, false, false); + + StringMap rewrites = state.realiseContext(context); + + auto realPath = state.toRealPath(rewriteStrings(path, rewrites), context); + + return flags.checkForPureEval + ? state.checkSourcePath(realPath) + : realPath; } /* Add and attribute to the given attribute map from the output name to @@ -88,13 +125,15 @@ void EvalState::realiseContext(const PathSet & context) the actual path. The 'drv' and 'drvPath' outputs must correspond. */ -static void mkOutputString(EvalState & state, Value & v, - const StorePath & drvPath, const BasicDerivation & drv, - std::pair<string, DerivationOutput> o) +static void mkOutputString( + EvalState & state, + BindingsBuilder & attrs, + const StorePath & drvPath, + const BasicDerivation & drv, + const std::pair<string, DerivationOutput> & o) { auto optOutputPath = o.second.path(*state.store, drv.name, o.first); - mkString( - *state.allocAttr(v, state.symbols.create(o.first)), + attrs.alloc(o.first).mkString( optOutputPath ? state.store->printStorePath(*optOutputPath) /* Downstream we would substitute this for an actual path once @@ -109,11 +148,9 @@ static void mkOutputString(EvalState & state, Value & v, argument. */ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vScope, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, vPath, context); - + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, vPath); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), @@ -124,8 +161,6 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS throw; } - Path realPath = state.checkSourcePath(state.toRealPath(path, context)); - // FIXME auto isValidDerivationInStore = [&]() -> std::optional<StorePath> { if (!state.store->isStorePath(path)) @@ -139,23 +174,19 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS if (auto optStorePath = isValidDerivationInStore()) { auto storePath = *optStorePath; Derivation drv = state.store->readDerivation(storePath); - Value & w = *state.allocValue(); - state.mkAttrs(w, 3 + drv.outputs.size()); - Value * v2 = state.allocAttr(w, state.sDrvPath); - mkString(*v2, path, {"=" + path}); - v2 = state.allocAttr(w, state.sName); - mkString(*v2, drv.env["name"]); - Value * outputsVal = - state.allocAttr(w, state.symbols.create("outputs")); - state.mkList(*outputsVal, drv.outputs.size()); - unsigned int outputs_index = 0; - - for (const auto & o : drv.outputs) { - mkOutputString(state, w, storePath, drv, o); - outputsVal->listElems()[outputs_index] = state.allocValue(); - mkString(*(outputsVal->listElems()[outputs_index++]), o.first); + auto attrs = state.buildBindings(3 + drv.outputs.size()); + attrs.alloc(state.sDrvPath).mkString(path, {"=" + path}); + attrs.alloc(state.sName).mkString(drv.env["name"]); + auto & outputsVal = attrs.alloc(state.sOutputs); + state.mkList(outputsVal, drv.outputs.size()); + + for (const auto & [i, o] : enumerate(drv.outputs)) { + mkOutputString(state, attrs, storePath, drv, o); + (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first); } - w.attrs->sort(); + + auto w = state.allocValue(); + w->mkAttrs(attrs); if (!state.vImportedDrvToDerivation) { state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); @@ -165,7 +196,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS } state.forceFunction(**state.vImportedDrvToDerivation, pos); - mkApp(v, **state.vImportedDrvToDerivation, w); + v.mkApp(*state.vImportedDrvToDerivation, w); state.forceAttrs(v, pos); } @@ -177,7 +208,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS else { if (!vScope) - state.evalFile(realPath, v); + state.evalFile(path, v); else { state.forceAttrs(*vScope); @@ -195,8 +226,8 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS // No need to call staticEnv.sort(), because // args[0]->attrs is already sorted. - printTalkative("evaluating file '%1%'", realPath); - Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); + printTalkative("evaluating file '%1%'", path); + Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); e->eval(state, *env, v); } @@ -281,22 +312,19 @@ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); /* Load a ValueInitializer from a DSO and return whatever it initializes */ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, *args[0]); } catch (InvalidPathError & e) { throw EvalError({ - .msg = hintfmt( - "cannot import '%1%', since path '%2%' is not valid", - path, e.path), + .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); + } catch (Error & e) { + e.addTrace(pos, "while importing '%s'", path); + throw; } - path = state.checkSourcePath(path); - string sym = state.forceStringNoCtx(*args[1], pos); void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); @@ -338,7 +366,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) for (unsigned int i = 1; i < args[0]->listSize(); ++i) commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false)); try { - state.realiseContext(context); + auto _ = state.realiseContext(context); // FIXME: Handle CA derivations } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid", @@ -384,7 +412,7 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu case nFloat: t = "float"; break; case nThunk: abort(); } - mkString(v, state.symbols.create(t)); + v.mkString(state.symbols.create(t)); } static RegisterPrimOp primop_typeOf({ @@ -402,7 +430,7 @@ static RegisterPrimOp primop_typeOf({ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nNull); + v.mkBool(args[0]->type() == nNull); } static RegisterPrimOp primop_isNull({ @@ -422,7 +450,7 @@ static RegisterPrimOp primop_isNull({ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nFunction); + v.mkBool(args[0]->type() == nFunction); } static RegisterPrimOp primop_isFunction({ @@ -438,7 +466,7 @@ static RegisterPrimOp primop_isFunction({ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nInt); + v.mkBool(args[0]->type() == nInt); } static RegisterPrimOp primop_isInt({ @@ -454,7 +482,7 @@ static RegisterPrimOp primop_isInt({ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nFloat); + v.mkBool(args[0]->type() == nFloat); } static RegisterPrimOp primop_isFloat({ @@ -470,7 +498,7 @@ static RegisterPrimOp primop_isFloat({ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nString); + v.mkBool(args[0]->type() == nString); } static RegisterPrimOp primop_isString({ @@ -486,7 +514,7 @@ static RegisterPrimOp primop_isString({ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nBool); + v.mkBool(args[0]->type() == nBool); } static RegisterPrimOp primop_isBool({ @@ -502,7 +530,7 @@ static RegisterPrimOp primop_isBool({ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nPath); + v.mkBool(args[0]->type() == nPath); } static RegisterPrimOp primop_isPath({ @@ -657,7 +685,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar /* Call the `operator' function with `e' as argument. */ Value call; - mkApp(call, *op->value, *e); + call.mkApp(op->value, e); state.forceList(call, pos); /* Add the values returned by the operator to the work set. */ @@ -733,7 +761,7 @@ static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { static void prim_ceil(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); - mkInt(v, ceil(value)); + v.mkInt(ceil(value)); } static RegisterPrimOp primop_ceil({ @@ -752,7 +780,7 @@ static RegisterPrimOp primop_ceil({ static void prim_floor(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); - mkInt(v, floor(value)); + v.mkInt(floor(value)); } static RegisterPrimOp primop_floor({ @@ -772,16 +800,16 @@ static RegisterPrimOp primop_floor({ * else => {success=false; value=false;} */ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Value & v) { - state.mkAttrs(v, 2); + auto attrs = state.buildBindings(2); try { state.forceValue(*args[0], pos); - v.attrs->push_back(Attr(state.sValue, args[0])); - mkBool(*state.allocAttr(v, state.symbols.create("success")), true); + attrs.insert(state.sValue, args[0]); + attrs.alloc("success").mkBool(true); } catch (AssertionError & e) { - mkBool(*state.allocAttr(v, state.sValue), false); - mkBool(*state.allocAttr(v, state.symbols.create("success")), false); + attrs.alloc(state.sValue).mkBool(false); + attrs.alloc("success").mkBool(false); } - v.attrs->sort(); + v.mkAttrs(attrs); } static RegisterPrimOp primop_tryEval({ @@ -809,7 +837,7 @@ static RegisterPrimOp primop_tryEval({ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) { string name = state.forceStringNoCtx(*args[0], pos); - mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); + v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); } static RegisterPrimOp primop_getEnv({ @@ -1235,11 +1263,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * drvHashes.lock()->insert_or_assign(drvPath, h); } - state.mkAttrs(v, 1 + drv.outputs.size()); - mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); + auto attrs = state.buildBindings(1 + drv.outputs.size()); + attrs.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS}); for (auto & i : drv.outputs) - mkOutputString(state, v, drvPath, drv, i); - v.attrs->sort(); + mkOutputString(state, attrs, drvPath, drv, i); + v.mkAttrs(attrs); } static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { @@ -1257,7 +1285,7 @@ static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { ‘out’. */ static void prim_placeholder(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); + v.mkString(hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); } static RegisterPrimOp primop_placeholder({ @@ -1282,7 +1310,7 @@ static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Valu { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); - mkString(v, canonPath(path), context); + v.mkString(canonPath(path), context); } static RegisterPrimOp primop_toPath({ @@ -1326,7 +1354,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V if (!settings.readOnlyMode) state.store->ensurePath(path2); context.insert(state.store->printStorePath(path2)); - mkString(v, path, context); + v.mkString(path, context); } static RegisterPrimOp primop_storePath({ @@ -1349,10 +1377,14 @@ static RegisterPrimOp primop_storePath({ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); + Path path; try { - state.realiseContext(context); + // We don’t check the path right now, because we don’t want to throw if + // the path isn’t allowed, but just return false + // (and we can’t just catch the exception here because we still want to + // throw if something in the evaluation of `*args[0]` tries to access an + // unauthorized path) + path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt( @@ -1363,13 +1395,13 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, } try { - mkBool(v, pathExists(state.checkSourcePath(path))); + v.mkBool(pathExists(state.checkSourcePath(path))); } catch (SysError & e) { /* Don't give away info from errors while canonicalising ‘path’ in restricted mode. */ - mkBool(v, false); + v.mkBool(false); } catch (RestrictedPathError & e) { - mkBool(v, false); + v.mkBool(false); } } @@ -1388,7 +1420,7 @@ static RegisterPrimOp primop_pathExists({ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - mkString(v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), context); + v.mkString(baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), context); } static RegisterPrimOp primop_baseNameOf({ @@ -1409,7 +1441,7 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value { PathSet context; Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); - if (args[0]->type() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); + if (args[0]->type() == nPath) v.mkPath(dir); else v.mkString(dir, context); } static RegisterPrimOp primop_dirOf({ @@ -1426,20 +1458,23 @@ static RegisterPrimOp primop_dirOf({ /* Return the contents of a file as a string. */ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, *args[0]); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } - string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); + string s = readFile(path); if (s.find((char) 0) != string::npos) throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path); - mkString(v, s.c_str()); + auto refs = state.store->isInStore(path) ? + state.store->queryPathInfo(state.store->toStorePath(path).first)->references : + StorePathSet{}; + auto context = state.store->printStorePathSet(refs); + v.mkString(s, context); } static RegisterPrimOp primop_readFile({ @@ -1475,11 +1510,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va pos ); - PathSet context; - string path = state.coerceToString(pos, *i->value, context, false, false); + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, *i->value, { .requireAbsolutePath = false }); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), @@ -1492,7 +1526,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va string path = state.forceStringNoCtx(*args[1], pos); - mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); + v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); } static RegisterPrimOp primop_findFile(RegisterPrimOp::Info { @@ -1512,15 +1546,14 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va .errPos = pos }); - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, *args[1]); } catch (InvalidPathError & e) { throw EvalError("cannot read '%s' since path '%s' is not valid, at %s", path, e.path, pos); } - mkString(v, hashFile(*ht, state.checkSourcePath(state.toRealPath(path, context))).to_string(Base16, false)); + v.mkString(hashFile(*ht, path).to_string(Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -1537,10 +1570,9 @@ static RegisterPrimOp primop_hashFile({ /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet ctx; - Path path = state.coerceToPath(pos, *args[0], ctx); + Path path; try { - state.realiseContext(ctx); + path = realisePath(state, pos, *args[0]); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), @@ -1548,21 +1580,21 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val }); } - DirEntries entries = readDirectory(state.checkSourcePath(path)); - state.mkAttrs(v, entries.size()); + DirEntries entries = readDirectory(path); + + auto attrs = state.buildBindings(entries.size()); for (auto & ent : entries) { - Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name)); if (ent.type == DT_UNKNOWN) ent.type = getFileType(path + "/" + ent.name); - ent_val->mkString( + attrs.alloc(ent.name).mkString( ent.type == DT_REG ? "regular" : ent.type == DT_DIR ? "directory" : ent.type == DT_LNK ? "symlink" : "unknown"); } - v.attrs->sort(); + v.mkAttrs(attrs); } static RegisterPrimOp primop_readDir({ @@ -1598,7 +1630,7 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value std::ostringstream out; PathSet context; printValueAsXML(state, true, false, *args[0], out, context, pos); - mkString(v, out.str(), context); + v.mkString(out.str(), context); } static RegisterPrimOp primop_toXML({ @@ -1706,7 +1738,7 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu std::ostringstream out; PathSet context; printValueAsJSON(state, true, *args[0], pos, out, context); - mkString(v, out.str(), context); + v.mkString(out.str(), context); } static RegisterPrimOp primop_toJSON({ @@ -1780,7 +1812,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu result, since `storePath' itself has references to the paths used in args[1]. */ - mkString(v, storePath, {storePath}); + v.mkString(storePath, {storePath}); } static RegisterPrimOp primop_toFile({ @@ -1875,7 +1907,8 @@ static void addPath( try { // FIXME: handle CA derivation outputs (where path needs to // be rewritten to the actual output). - state.realiseContext(context); + auto rewrites = state.realiseContext(context); + path = state.toRealPath(rewriteStrings(path, rewrites), context); StorePathSet refs; @@ -1896,10 +1929,10 @@ static void addPath( /* Call the filter function. The first argument is the path, the second is a string indicating the type of the file. */ Value arg1; - mkString(arg1, path); + arg1.mkString(path); Value arg2; - mkString(arg2, + arg2.mkString( S_ISREG(st.st_mode) ? "regular" : S_ISDIR(st.st_mode) ? "directory" : S_ISLNK(st.st_mode) ? "symlink" : @@ -1926,7 +1959,7 @@ static void addPath( } else dstPath = state.store->printStorePath(*expectedStorePath); - mkString(v, dstPath, {dstPath}); + v.mkString(dstPath, {dstPath}); state.allowPath(dstPath); @@ -2100,7 +2133,7 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V size_t n = 0; for (auto & i : *args[0]->attrs) - mkString(*(v.listElems()[n++] = state.allocValue()), i.name); + (v.listElems()[n++] = state.allocValue())->mkString(i.name); std::sort(v.listElems(), v.listElems() + n, [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); @@ -2184,7 +2217,7 @@ static void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * state.forceAttrs(*args[1], pos); Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); if (i == args[1]->attrs->end()) - mkNull(v); + v.mkNull(); else state.mkPos(v, i->pos); } @@ -2200,7 +2233,7 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val { string attr = state.forceStringNoCtx(*args[0], pos); state.forceAttrs(*args[1], pos); - mkBool(v, args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); + v.mkBool(args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); } static RegisterPrimOp primop_hasAttr({ @@ -2218,7 +2251,7 @@ static RegisterPrimOp primop_hasAttr({ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nAttrs); + v.mkBool(args[0]->type() == nAttrs); } static RegisterPrimOp primop_isAttrs({ @@ -2245,11 +2278,12 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, /* Copy all attributes not in that set. Note that we don't need to sort v.attrs because it's a subset of an already sorted vector. */ - state.mkAttrs(v, args[0]->attrs->size()); + auto attrs = state.buildBindings(args[0]->attrs->size()); for (auto & i : *args[0]->attrs) { if (!names.count(i.name)) - v.attrs->push_back(i); + attrs.insert(i); } + v.mkAttrs(attrs.alreadySorted()); } static RegisterPrimOp primop_removeAttrs({ @@ -2277,7 +2311,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, { state.forceList(*args[0], pos); - state.mkAttrs(v, args[0]->listSize()); + auto attrs = state.buildBindings(args[0]->listSize()); std::set<Symbol> seen; @@ -2303,11 +2337,11 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, v2->attrs, pos ); - v.attrs->push_back(Attr(sym, j2->value, j2->pos)); + attrs.insert(sym, j2->value, j2->pos); } } - v.attrs->sort(); + v.mkAttrs(attrs); } static RegisterPrimOp primop_listToAttrs({ @@ -2340,13 +2374,15 @@ static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * ar state.forceAttrs(*args[0], pos); state.forceAttrs(*args[1], pos); - state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size())); + auto attrs = state.buildBindings(std::min(args[0]->attrs->size(), args[1]->attrs->size())); for (auto & i : *args[0]->attrs) { Bindings::iterator j = args[1]->attrs->find(i.name); if (j != args[1]->attrs->end()) - v.attrs->push_back(*j); + attrs.insert(*j); } + + v.mkAttrs(attrs.alreadySorted()); } static RegisterPrimOp primop_intersectAttrs({ @@ -2400,7 +2436,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args { state.forceValue(*args[0], pos); if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) { - state.mkAttrs(v, 0); + v.mkAttrs(&state.emptyBindings); return; } if (!args[0]->isLambda()) @@ -2410,18 +2446,15 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args }); if (!args[0]->lambda.fun->hasFormals()) { - state.mkAttrs(v, 0); + v.mkAttrs(&state.emptyBindings); return; } - state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); - for (auto & i : args[0]->lambda.fun->formals->formals) { + auto attrs = state.buildBindings(args[0]->lambda.fun->formals->formals.size()); + for (auto & i : args[0]->lambda.fun->formals->formals) // !!! should optimise booleans (allocate only once) - Value * value = state.allocValue(); - v.attrs->push_back(Attr(i.name, value, ptr(&i.pos))); - mkBool(*value, i.def); - } - v.attrs->sort(); + attrs.alloc(i.name, ptr(&i.pos)).mkBool(i.def); + v.mkAttrs(attrs); } static RegisterPrimOp primop_functionArgs({ @@ -2446,15 +2479,17 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va { state.forceAttrs(*args[1], pos); - state.mkAttrs(v, args[1]->attrs->size()); + auto attrs = state.buildBindings(args[1]->attrs->size()); for (auto & i : *args[1]->attrs) { Value * vName = state.allocValue(); Value * vFun2 = state.allocValue(); - mkString(*vName, i.name); - mkApp(*vFun2, *args[0], *vName); - mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value); + vName->mkString(i.name); + vFun2->mkApp(args[0], vName); + attrs.alloc(i.name).mkApp(vFun2, i.value); } + + v.mkAttrs(attrs.alreadySorted()); } static RegisterPrimOp primop_mapAttrs({ @@ -2472,6 +2507,91 @@ static RegisterPrimOp primop_mapAttrs({ .fun = prim_mapAttrs, }); +static void prim_zipAttrsWith(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + // we will first count how many values are present for each given key. + // we then allocate a single attrset and pre-populate it with lists of + // appropriate sizes, stash the pointers to the list elements of each, + // and populate the lists. after that we replace the list in the every + // attribute with the merge function application. this way we need not + // use (slightly slower) temporary storage the GC does not know about. + + std::map<Symbol, std::pair<size_t, Value * *>> attrsSeen; + + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + const auto listSize = args[1]->listSize(); + const auto listElems = args[1]->listElems(); + + for (unsigned int n = 0; n < listSize; ++n) { + Value * vElem = listElems[n]; + try { + state.forceAttrs(*vElem); + for (auto & attr : *vElem->attrs) + attrsSeen[attr.name].first++; + } catch (TypeError & e) { + e.addTrace(pos, hintfmt("while invoking '%s'", "zipAttrsWith")); + throw; + } + } + + auto attrs = state.buildBindings(attrsSeen.size()); + for (auto & [sym, elem] : attrsSeen) { + auto & list = attrs.alloc(sym); + state.mkList(list, elem.first); + elem.second = list.listElems(); + } + v.mkAttrs(attrs.alreadySorted()); + + for (unsigned int n = 0; n < listSize; ++n) { + Value * vElem = listElems[n]; + for (auto & attr : *vElem->attrs) + *attrsSeen[attr.name].second++ = attr.value; + } + + for (auto & attr : *v.attrs) { + auto name = state.allocValue(); + name->mkString(attr.name); + auto call1 = state.allocValue(); + call1->mkApp(args[0], name); + auto call2 = state.allocValue(); + call2->mkApp(call1, attr.value); + attr.value = call2; + } +} + +static RegisterPrimOp primop_zipAttrsWith({ + .name = "__zipAttrsWith", + .args = {"f", "list"}, + .doc = R"( + Transpose a list of attribute sets into an attribute set of lists, + then apply `mapAttrs`. + + `f` receives two arguments: the attribute name and a non-empty + list of all values encountered for that attribute name. + + The result is an attribute set where the attribute names are the + union of the attribute names in each element of `list`. The attribute + values are the return values of `f`. + + ```nix + builtins.zipAttrsWith + (name: values: { inherit name values; }) + [ { a = "x"; } { a = "y"; b = "z"; } ] + ``` + + evaluates to + + ``` + { + a = { name = "a"; values = [ "x" "y" ]; }; + b = { name = "b"; values = [ "z" ]; }; + } + ``` + )", + .fun = prim_zipAttrsWith, +}); + /************************************************************* * Lists @@ -2482,7 +2602,7 @@ static RegisterPrimOp primop_mapAttrs({ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type() == nList); + v.mkBool(args[0]->type() == nList); } static RegisterPrimOp primop_isList({ @@ -2580,8 +2700,8 @@ static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & state.mkList(v, args[1]->listSize()); for (unsigned int n = 0; n < v.listSize(); ++n) - mkApp(*(v.listElems()[n] = state.allocValue()), - *args[0], *args[1]->listElems()[n]); + (v.listElems()[n] = state.allocValue())->mkApp( + args[0], args[1]->listElems()[n]); } static RegisterPrimOp primop_map({ @@ -2650,7 +2770,7 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value res = true; break; } - mkBool(v, res); + v.mkBool(res); } static RegisterPrimOp primop_elem({ @@ -2683,7 +2803,7 @@ static RegisterPrimOp primop_concatLists({ static void prim_length(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceList(*args[0], pos); - mkInt(v, args[0]->listSize()); + v.mkInt(args[0]->listSize()); } static RegisterPrimOp primop_length({ @@ -2740,12 +2860,12 @@ static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * arg state.callFunction(*args[0], *elem, vTmp, pos); bool res = state.forceBool(vTmp, pos); if (res == any) { - mkBool(v, any); + v.mkBool(any); return; } } - mkBool(v, !any); + v.mkBool(!any); } @@ -2792,9 +2912,9 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val state.mkList(v, len); for (unsigned int n = 0; n < (unsigned int) len; ++n) { - Value * arg = state.allocValue(); - mkInt(*arg, n); - mkApp(*(v.listElems()[n] = state.allocValue()), *args[0], *arg); + auto arg = state.allocValue(); + arg->mkInt(n); + (v.listElems()[n] = state.allocValue())->mkApp(args[0], arg); } } @@ -2888,21 +3008,21 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V wrong.push_back(vElem); } - state.mkAttrs(v, 2); + auto attrs = state.buildBindings(2); - Value * vRight = state.allocAttr(v, state.sRight); + auto & vRight = attrs.alloc(state.sRight); auto rsize = right.size(); - state.mkList(*vRight, rsize); + state.mkList(vRight, rsize); if (rsize) - memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize); + memcpy(vRight.listElems(), right.data(), sizeof(Value *) * rsize); - Value * vWrong = state.allocAttr(v, state.sWrong); + auto & vWrong = attrs.alloc(state.sWrong); auto wsize = wrong.size(); - state.mkList(*vWrong, wsize); + state.mkList(vWrong, wsize); if (wsize) - memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wsize); + memcpy(vWrong.listElems(), wrong.data(), sizeof(Value *) * wsize); - v.attrs->sort(); + v.mkAttrs(attrs); } static RegisterPrimOp primop_partition({ @@ -2944,14 +3064,16 @@ static void prim_groupBy(EvalState & state, const Pos & pos, Value * * args, Val vector->second.push_back(vElem); } - state.mkAttrs(v, attrs.size()); + auto attrs2 = state.buildBindings(attrs.size()); for (auto & i : attrs) { - Value * list = state.allocAttr(v, i.first); + auto & list = attrs2.alloc(i.first); auto size = i.second.size(); - state.mkList(*list, size); - memcpy(list->listElems(), i.second.data(), sizeof(Value *) * size); + state.mkList(list, size); + memcpy(list.listElems(), i.second.data(), sizeof(Value *) * size); } + + v.mkAttrs(attrs2.alreadySorted()); } static RegisterPrimOp primop_groupBy({ @@ -3030,9 +3152,9 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else - mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_add({ @@ -3049,9 +3171,9 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else - mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_sub({ @@ -3068,9 +3190,9 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); if (args[0]->type() == nFloat || args[1]->type() == nFloat) - mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else - mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_mul({ @@ -3095,7 +3217,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & }); if (args[0]->type() == nFloat || args[1]->type() == nFloat) { - mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); + v.mkFloat(state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); } else { NixInt i1 = state.forceInt(*args[0], pos); NixInt i2 = state.forceInt(*args[1], pos); @@ -3106,7 +3228,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & .errPos = pos }); - mkInt(v, i1 / i2); + v.mkInt(i1 / i2); } } @@ -3121,7 +3243,7 @@ static RegisterPrimOp primop_div({ static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_bitAnd({ @@ -3135,7 +3257,7 @@ static RegisterPrimOp primop_bitAnd({ static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_bitOr({ @@ -3149,7 +3271,7 @@ static RegisterPrimOp primop_bitOr({ static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v) { - mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); + v.mkInt(state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); } static RegisterPrimOp primop_bitXor({ @@ -3166,7 +3288,7 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); CompareValues comp{state}; - mkBool(v, comp(args[0], args[1])); + v.mkBool(comp(args[0], args[1])); } static RegisterPrimOp primop_lessThan({ @@ -3193,7 +3315,7 @@ static void prim_toString(EvalState & state, const Pos & pos, Value * * args, Va { PathSet context; string s = state.coerceToString(pos, *args[0], context, true, false); - mkString(v, s, context); + v.mkString(s, context); } static RegisterPrimOp primop_toString({ @@ -3237,7 +3359,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V .errPos = pos }); - mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); + v.mkString((unsigned int) start >= s.size() ? "" : string(s, start, len), context); } static RegisterPrimOp primop_substring({ @@ -3264,7 +3386,7 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args { PathSet context; string s = state.coerceToString(pos, *args[0], context); - mkInt(v, s.size()); + v.mkInt(s.size()); } static RegisterPrimOp primop_stringLength({ @@ -3291,7 +3413,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, PathSet context; // discarded string s = state.forceString(*args[1], context, pos); - mkString(v, hashString(*ht, s).to_string(Base16, false)); + v.mkString(hashString(*ht, s).to_string(Base16, false)); } static RegisterPrimOp primop_hashString({ @@ -3330,7 +3452,7 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) std::smatch match; if (!std::regex_match(str, match, regex->second)) { - mkNull(v); + v.mkNull(); return; } @@ -3339,9 +3461,9 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) state.mkList(v, len); for (size_t i = 0; i < len; ++i) { if (!match[i+1].matched) - mkNull(*(v.listElems()[i] = state.allocValue())); + (v.listElems()[i] = state.allocValue())->mkNull(); else - mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str()); + (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); } } catch (std::regex_error &e) { @@ -3416,7 +3538,6 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value const size_t len = std::distance(begin, end); state.mkList(v, 2 * len + 1); size_t idx = 0; - Value * elem; if (len == 0) { v.listElems()[idx++] = args[1]; @@ -3428,28 +3549,26 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value std::smatch match = *i; // Add a string for non-matched characters. - elem = v.listElems()[idx++] = state.allocValue(); - mkString(*elem, match.prefix().str().c_str()); + (v.listElems()[idx++] = state.allocValue())->mkString(match.prefix().str()); // Add a list for matched substrings. const size_t slen = match.size() - 1; - elem = v.listElems()[idx++] = state.allocValue(); + auto elem = v.listElems()[idx++] = state.allocValue(); // Start at 1, beacause the first match is the whole string. state.mkList(*elem, slen); for (size_t si = 0; si < slen; ++si) { if (!match[si + 1].matched) - mkNull(*(elem->listElems()[si] = state.allocValue())); + (elem->listElems()[si] = state.allocValue())->mkNull(); else - mkString(*(elem->listElems()[si] = state.allocValue()), match[si + 1].str().c_str()); + (elem->listElems()[si] = state.allocValue())->mkString(match[si + 1].str()); } // Add a string for non-matched suffix characters. - if (idx == 2 * len) { - elem = v.listElems()[idx++] = state.allocValue(); - mkString(*elem, match.suffix().str().c_str()); - } + if (idx == 2 * len) + (v.listElems()[idx++] = state.allocValue())->mkString(match.suffix().str()); } + assert(idx == 2 * len + 1); } catch (std::regex_error &e) { @@ -3521,7 +3640,7 @@ static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * * res += state.coerceToString(pos, *elem, context); } - mkString(v, res, context); + v.mkString(res, context); } static RegisterPrimOp primop_concatStringsSep({ @@ -3590,7 +3709,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar } } - mkString(v, res, context); + v.mkString(res, context); } static RegisterPrimOp primop_replaceStrings({ @@ -3619,10 +3738,10 @@ static void prim_parseDrvName(EvalState & state, const Pos & pos, Value * * args { string name = state.forceStringNoCtx(*args[0], pos); DrvName parsed(name); - state.mkAttrs(v, 2); - mkString(*state.allocAttr(v, state.sName), parsed.name); - mkString(*state.allocAttr(v, state.symbols.create("version")), parsed.version); - v.attrs->sort(); + auto attrs = state.buildBindings(2); + attrs.alloc(state.sName).mkString(parsed.name); + attrs.alloc("version").mkString(parsed.version); + v.mkAttrs(attrs); } static RegisterPrimOp primop_parseDrvName({ @@ -3643,7 +3762,7 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a { string version1 = state.forceStringNoCtx(*args[0], pos); string version2 = state.forceStringNoCtx(*args[1], pos); - mkInt(v, compareVersions(version1, version2)); + v.mkInt(compareVersions(version1, version2)); } static RegisterPrimOp primop_compareVersions({ @@ -3671,11 +3790,8 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args components.emplace_back(std::move(component)); } state.mkList(v, components.size()); - unsigned int n = 0; - for (auto & component : components) { - auto listElem = v.listElems()[n++] = state.allocValue(); - mkString(*listElem, std::move(component)); - } + for (const auto & [n, component] : enumerate(components)) + (v.listElems()[n] = state.allocValue())->mkString(std::move(component)); } static RegisterPrimOp primop_splitVersion({ @@ -3725,37 +3841,37 @@ void EvalState::createBaseEnv() Value v; /* `builtins' must be first! */ - mkAttrs(v, 128); + v.mkAttrs(buildBindings(128).finish()); addConstant("builtins", v); - mkBool(v, true); + v.mkBool(true); addConstant("true", v); - mkBool(v, false); + v.mkBool(false); addConstant("false", v); - mkNull(v); + v.mkNull(); addConstant("null", v); if (!evalSettings.pureEval) { - mkInt(v, time(0)); + v.mkInt(time(0)); addConstant("__currentTime", v); - mkString(v, settings.thisSystem.get()); + v.mkString(settings.thisSystem.get()); addConstant("__currentSystem", v); } - mkString(v, nixVersion); + v.mkString(nixVersion); addConstant("__nixVersion", v); - mkString(v, store->storeDir); + v.mkString(store->storeDir); addConstant("__storeDir", v); /* Language version. This should be increased every time a new language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ - mkInt(v, 6); + v.mkInt(6); addConstant("__langVersion", v); // Miscellaneous @@ -3768,11 +3884,10 @@ void EvalState::createBaseEnv() mkList(v, searchPath.size()); int n = 0; for (auto & i : searchPath) { - auto v2 = v.listElems()[n++] = allocValue(); - mkAttrs(*v2, 2); - mkString(*allocAttr(*v2, symbols.create("path")), i.second); - mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); - v2->attrs->sort(); + auto attrs = buildBindings(2); + attrs.alloc("path").mkString(i.second); + attrs.alloc("prefix").mkString(i.first); + (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } addConstant("__nixPath", v); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 20545afd0..a239c06da 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -7,8 +7,7 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; - string s = state.coerceToString(pos, *args[0], context); - mkString(v, s, PathSet()); + v.mkString(state.coerceToString(pos, *args[0], context)); } static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); @@ -18,7 +17,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, { PathSet context; state.forceString(*args[0], context, pos); - mkBool(v, !context.empty()); + v.mkBool(!context.empty()); } static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); @@ -39,7 +38,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po for (auto & p : context) context2.insert(p.at(0) == '=' ? string(p, 1) : p); - mkString(v, s, context2); + v.mkString(s, context2); } static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); @@ -103,27 +102,26 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, } } - state.mkAttrs(v, contextInfos.size()); + auto attrs = state.buildBindings(contextInfos.size()); auto sPath = state.symbols.create("path"); auto sAllOutputs = state.symbols.create("allOutputs"); for (const auto & info : contextInfos) { - auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first)); - state.mkAttrs(infoVal, 3); + auto infoAttrs = state.buildBindings(3); if (info.second.path) - mkBool(*state.allocAttr(infoVal, sPath), true); + infoAttrs.alloc(sPath).mkBool(true); if (info.second.allOutputs) - mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + infoAttrs.alloc(sAllOutputs).mkBool(true); if (!info.second.outputs.empty()) { - auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); + auto & outputsVal = infoAttrs.alloc(state.sOutputs); state.mkList(outputsVal, info.second.outputs.size()); - size_t i = 0; - for (const auto & output : info.second.outputs) - mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); + for (const auto & [i, output] : enumerate(info.second.outputs)) + (outputsVal.listElems()[i] = state.allocValue())->mkString(output); } - infoVal.attrs->sort(); + attrs.alloc(info.first).mkAttrs(infoAttrs); } - v.attrs->sort(); + + v.mkAttrs(attrs); } static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); @@ -187,7 +185,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg } } - mkString(v, orig, context); + v.mkString(orig, context); } static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index c23480853..42214c207 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -70,19 +70,19 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // FIXME: use name auto [tree, input2] = input.fetch(state.store); - state.mkAttrs(v, 8); + auto attrs2 = state.buildBindings(8); auto storePath = state.store->printStorePath(tree.storePath); - mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + attrs2.alloc(state.sOutPath).mkString(storePath, {storePath}); if (input2.getRef()) - mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef()); + attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. auto rev2 = input2.getRev().value_or(Hash(htSHA1)); - mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); + attrs2.alloc("rev").mkString(rev2.gitRev()); + attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12)); if (auto revCount = input2.getRevCount()) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); - v.attrs->sort(); + attrs2.alloc("revCount").mkInt(*revCount); + v.mkAttrs(attrs2); state.allowPath(tree.storePath); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 079513873..6647bd35c 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -21,49 +21,48 @@ void emitTreeAttrs( { assert(input.isImmutable()); - state.mkAttrs(v, 8); + auto attrs = state.buildBindings(8); auto storePath = state.store->printStorePath(tree.storePath); - mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); + attrs.alloc(state.sOutPath).mkString(storePath, {storePath}); // FIXME: support arbitrary input attributes. auto narHash = input.getNarHash(); assert(narHash); - mkString(*state.allocAttr(v, state.symbols.create("narHash")), - narHash->to_string(SRI, true)); + attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); if (input.getType() == "git") - mkBool(*state.allocAttr(v, state.symbols.create("submodules")), + attrs.alloc("submodules").mkBool( fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false)); if (!forceDirty) { if (auto rev = input.getRev()) { - mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); + attrs.alloc("rev").mkString(rev->gitRev()); + attrs.alloc("shortRev").mkString(rev->gitShortRev()); } else if (emptyRevFallback) { // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev auto emptyHash = Hash(htSHA1); - mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev()); + attrs.alloc("rev").mkString(emptyHash.gitRev()); + attrs.alloc("shortRev").mkString(emptyHash.gitShortRev()); } if (auto revCount = input.getRevCount()) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); + attrs.alloc("revCount").mkInt(*revCount); else if (emptyRevFallback) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0); + attrs.alloc("revCount").mkInt(0); } if (auto lastModified = input.getLastModified()) { - mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); - mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")), + attrs.alloc("lastModified").mkInt(*lastModified); + attrs.alloc("lastModifiedDate").mkString( fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S"))); } - v.attrs->sort(); + v.mkAttrs(attrs); } std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") @@ -248,7 +247,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, state.allowPath(storePath); auto path = state.store->printStorePath(storePath); - mkString(v, path, PathSet({path})); + v.mkString(path, PathSet({path})); } static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4c6682dfd..80c7e0b82 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -1,86 +1,76 @@ #include "primops.hh" #include "eval-inline.hh" -#include "../../cpptoml/cpptoml.h" +#include "../../toml11/toml.hpp" namespace nix { -static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & val) { - using namespace cpptoml; - auto toml = state.forceStringNoCtx(*args[0], pos); std::istringstream tomlStream(toml); - std::function<void(Value &, std::shared_ptr<base>)> visit; + std::function<void(Value &, toml::value)> visit; - visit = [&](Value & v, std::shared_ptr<base> t) { + visit = [&](Value & v, toml::value t) { - if (auto t2 = t->as_table()) { + switch(t.type()) + { + case toml::value_t::table: + { + auto table = toml::get<toml::table>(t); - size_t size = 0; - for (auto & i : *t2) { (void) i; size++; } + size_t size = 0; + for (auto & i : table) { (void) i; size++; } - state.mkAttrs(v, size); + auto attrs = state.buildBindings(size); - for (auto & i : *t2) { - auto & v2 = *state.allocAttr(v, state.symbols.create(i.first)); + for(auto & elem : table) + visit(attrs.alloc(elem.first), elem.second); - if (auto i2 = i.second->as_table_array()) { - size_t size2 = i2->get().size(); - state.mkList(v2, size2); - for (size_t j = 0; j < size2; ++j) - visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]); + v.mkAttrs(attrs); } - else - visit(v2, i.second); - } - - v.attrs->sort(); - } - - else if (auto t2 = t->as_array()) { - size_t size = t2->get().size(); - - state.mkList(v, size); - - for (size_t i = 0; i < size; ++i) - visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); - } - - // Handle cases like 'a = [[{ a = true }]]', which IMHO should be - // parsed as a array containing an array containing a table, - // but instead are parsed as an array containing a table array - // containing a table. - else if (auto t2 = t->as_table_array()) { - size_t size = t2->get().size(); - - state.mkList(v, size); - - for (size_t j = 0; j < size; ++j) - visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); - } + break;; + case toml::value_t::array: + { + auto array = toml::get<std::vector<toml::value>>(t); + + size_t size = array.size(); + state.mkList(v, size); + for (size_t i = 0; i < size; ++i) + visit(*(v.listElems()[i] = state.allocValue()), array[i]); + } + break;; + case toml::value_t::boolean: + v.mkBool(toml::get<bool>(t)); + break;; + case toml::value_t::integer: + v.mkInt(toml::get<int64_t>(t)); + break;; + case toml::value_t::floating: + v.mkFloat(toml::get<NixFloat>(t)); + break;; + case toml::value_t::string: + v.mkString(toml::get<std::string>(t)); + break;; + case toml::value_t::local_datetime: + case toml::value_t::offset_datetime: + case toml::value_t::local_date: + case toml::value_t::local_time: + // We fail since Nix doesn't have date and time types + throw std::runtime_error("Dates and times are not supported"); + break;; + case toml::value_t::empty: + v.mkNull(); + break;; - else if (t->is_value()) { - if (auto val = t->as<int64_t>()) - mkInt(v, val->get()); - else if (auto val = t->as<NixFloat>()) - mkFloat(v, val->get()); - else if (auto val = t->as<bool>()) - mkBool(v, val->get()); - else if (auto val = t->as<std::string>()) - mkString(v, val->get()); - else - throw EvalError("unsupported value type in TOML"); } - - else abort(); }; try { - visit(v, parser(tomlStream).parse()); - } catch (std::runtime_error & e) { + visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */)); + } catch (std::exception & e) { // TODO: toml::syntax_error throw EvalError({ .msg = hintfmt("while parsing a TOML string: %s", e.what()), .errPos = pos diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 6b4f3c0ae..1896c7563 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -10,6 +10,8 @@ namespace nix { +class BindingsBuilder; + typedef enum { tInt = 1, @@ -235,6 +237,17 @@ public: string.context = context; } + void mkString(std::string_view s); + + void mkString(std::string_view s, const PathSet & context); + + void mkStringMove(const char * s, const PathSet & context); + + inline void mkString(const Symbol & s) + { + mkString(((const std::string &) s).c_str()); + } + inline void mkPath(const char * s) { clearValue(); @@ -242,6 +255,8 @@ public: path = s; } + void mkPath(std::string_view s); + inline void mkNull() { clearValue(); @@ -255,6 +270,8 @@ public: attrs = a; } + Value & mkAttrs(BindingsBuilder & bindings); + inline void mkList(size_t size) { clearValue(); @@ -383,45 +400,6 @@ public: }; - -// TODO: Remove these static functions, replace call sites with v.mk* instead -static inline void mkInt(Value & v, NixInt n) -{ - v.mkInt(n); -} - -static inline void mkFloat(Value & v, NixFloat n) -{ - v.mkFloat(n); -} - -static inline void mkBool(Value & v, bool b) -{ - v.mkBool(b); -} - -static inline void mkNull(Value & v) -{ - v.mkNull(); -} - -static inline void mkApp(Value & v, Value & left, Value & right) -{ - v.mkApp(&left, &right); -} - -static inline void mkString(Value & v, const Symbol & s) -{ - v.mkString(((const string &) s).c_str()); -} - - -void mkString(Value & v, const char * s); - - -void mkPath(Value & v, const char * s); - - #if HAVE_BOEHMGC typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector; typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap; |