diff options
Diffstat (limited to 'src/libexpr')
40 files changed, 5564 insertions, 988 deletions
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 83854df49..c50c6d92b 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -19,7 +19,7 @@ static Strings parseAttrPath(std::string_view s) ++i; while (1) { if (i == s.end()) - throw Error("missing closing quote in selection path '%1%'", s); + throw ParseError("missing closing quote in selection path '%1%'", s); if (*i == '"') break; cur.push_back(*i++); } @@ -52,9 +52,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr for (auto & attr : tokens) { /* Is i an index (integer) or a normal attribute name? */ - enum { apAttr, apIndex } apType = apAttr; - unsigned int attrIndex; - if (string2Int(attr, attrIndex)) apType = apIndex; + auto attrIndex = string2Int<unsigned int>(attr); /* Evaluate the expression. */ Value * vNew = state.allocValue(); @@ -65,9 +63,9 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr /* It should evaluate to either a set or an expression, according to what is specified in the attrPath. */ - if (apType == apAttr) { + if (!attrIndex) { - if (v->type != tAttrs) + if (v->type() != nAttrs) throw TypeError( "the expression selected by the selection path '%1%' should be a set but is %2%", attrPath, @@ -82,17 +80,17 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr pos = *a->pos; } - else if (apType == apIndex) { + else { if (!v->isList()) throw TypeError( "the expression selected by the selection path '%1%' should be a list but is %2%", attrPath, showType(*v)); - if (attrIndex >= v->listSize()) - throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); + if (*attrIndex >= v->listSize()) + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath); - v = v->listElems()[attrIndex]; + v = v->listElems()[*attrIndex]; pos = noPos; } @@ -102,7 +100,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr } -Pos findDerivationFilename(EvalState & state, Value & v, std::string what) +Pos findPackageFilename(EvalState & state, Value & v, std::string what) { Value * v2; try { @@ -118,14 +116,14 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what) auto colon = pos.rfind(':'); if (colon == std::string::npos) - throw Error("cannot parse meta.position attribute '%s'", pos); + throw ParseError("cannot parse meta.position attribute '%s'", pos); std::string filename(pos, 0, colon); unsigned int lineno; try { lineno = std::stoi(std::string(pos, colon + 1)); } catch (std::invalid_argument & e) { - throw Error("cannot parse line number '%s'", pos); + throw ParseError("cannot parse line number '%s'", pos); } Symbol file = state.symbols.create(filename); diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index d9d74ab2d..2ee3ea089 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -14,7 +14,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr Bindings & autoArgs, Value & vIn); /* Heuristic to find the filename and lineno or a nix value. */ -Pos findDerivationFilename(EvalState & state, Value & v, std::string what); +Pos findPackageFilename(EvalState & state, Value & v, std::string what); std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s); diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index b1d61a285..b6091c955 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -24,9 +24,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity) v = vEmptySet; return; } - clearValue(v); - v.type = tAttrs; - v.attrs = allocBindings(capacity); + v.mkAttrs(allocBindings(capacity)); nrAttrsets++; nrAttrsInAttrsets += capacity; } diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index 7eaa16c59..7d6ffc9f3 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -17,8 +17,8 @@ struct Attr { Symbol name; Value * value; - Pos * pos; - Attr(Symbol name, Value * value, Pos * pos = &noPos) + ptr<Pos> pos; + Attr(Symbol name, Value * value, ptr<Pos> pos = ptr(&noPos)) : name(name), value(value), pos(pos) { }; Attr() : pos(&noPos) { }; bool operator < (const Attr & a) const @@ -35,12 +35,13 @@ class Bindings { public: typedef uint32_t size_t; + ptr<Pos> pos; private: size_t size_, capacity_; Attr attrs[0]; - Bindings(size_t capacity) : size_(0), capacity_(capacity) { } + Bindings(size_t capacity) : pos(&noPos), size_(0), capacity_(capacity) { } Bindings(const Bindings & bindings) = delete; public: @@ -77,7 +78,7 @@ public: auto a = get(name); if (!a) throw Error({ - .hint = hintfmt("attribute '%s' missing", name), + .msg = hintfmt("attribute '%s' missing", name), .errPos = pos }); diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 44baadd53..fb0932c00 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -4,22 +4,28 @@ #include "util.hh" #include "eval.hh" #include "fetchers.hh" +#include "registry.hh" +#include "flake/flakeref.hh" #include "store-api.hh" namespace nix { MixEvalArgs::MixEvalArgs() { + auto category = "Common evaluation options"; + addFlag({ .longName = "arg", - .description = "argument to be passed to Nix functions", + .description = "Pass the value *expr* as the argument *name* to Nix functions.", + .category = category, .labels = {"name", "expr"}, .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} }); addFlag({ .longName = "argstr", - .description = "string-valued argument to be passed to Nix functions", + .description = "Pass the string *string* as the argument *name* to Nix functions.", + .category = category, .labels = {"name", "string"}, .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, }); @@ -27,10 +33,42 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "include", .shortName = 'I', - .description = "add a path to the list of locations used to look up <...> file names", + .description = "Add *path* to the list of locations used to look up `<...>` file names.", + .category = category, .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} }); + + addFlag({ + .longName = "impure", + .description = "Allow access to mutable paths and repositories.", + .category = category, + .handler = {[&]() { + evalSettings.pureEval = false; + }}, + }); + + addFlag({ + .longName = "override-flake", + .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.", + .category = category, + .labels = {"original-ref", "resolved-ref"}, + .handler = {[&](std::string _from, std::string _to) { + auto from = parseFlakeRef(_from, absPath(".")); + auto to = parseFlakeRef(_to, absPath(".")); + fetchers::Attrs extraAttrs; + if (to.subdir != "") extraAttrs["dir"] = to.subdir; + fetchers::overrideRegistry(from.input, to.input, extraAttrs); + }} + }); + + addFlag({ + .longName = "eval-store", + .description = "The Nix store to use for evaluations.", + .category = category, + .labels = {"store-url"}, + .handler = {&evalStoreUrl}, + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) @@ -53,7 +91,7 @@ Path lookupFileArg(EvalState & state, string s) if (isUri(s)) { return state.store->toRealPath( fetchers::downloadTarball( - state.store, resolveUri(s), "source", false).storePath); + state.store, resolveUri(s), "source", false).first.storePath); } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { Path p = s.substr(1, s.size() - 2); return state.findFile(p); diff --git a/src/libexpr/common-eval-args.hh b/src/libexpr/common-eval-args.hh index be7fda783..0e113fff1 100644 --- a/src/libexpr/common-eval-args.hh +++ b/src/libexpr/common-eval-args.hh @@ -16,8 +16,9 @@ struct MixEvalArgs : virtual Args Strings searchPath; -private: + std::optional<std::string> evalStoreUrl; +private: std::map<std::string, std::string> autoArgs; }; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc new file mode 100644 index 000000000..d7e21783d --- /dev/null +++ b/src/libexpr/eval-cache.cc @@ -0,0 +1,629 @@ +#include "eval-cache.hh" +#include "sqlite.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "store-api.hh" + +namespace nix::eval_cache { + +static const char * schema = R"sql( +create table if not exists Attributes ( + parent integer not null, + name text, + type integer not null, + value text, + context text, + primary key (parent, name) +); +)sql"; + +struct AttrDb +{ + std::atomic_bool failed{false}; + + struct State + { + SQLite db; + SQLiteStmt insertAttribute; + SQLiteStmt insertAttributeWithContext; + SQLiteStmt queryAttribute; + SQLiteStmt queryAttributes; + std::unique_ptr<SQLiteTxn> txn; + }; + + std::unique_ptr<Sync<State>> _state; + + AttrDb(const Hash & fingerprint) + : _state(std::make_unique<Sync<State>>()) + { + auto state(_state->lock()); + + Path cacheDir = getCacheDir() + "/nix/eval-cache-v2"; + createDirs(cacheDir); + + Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->insertAttribute.create(state->db, + "insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)"); + + state->insertAttributeWithContext.create(state->db, + "insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)"); + + state->queryAttribute.create(state->db, + "select rowid, type, value, context from Attributes where parent = ? and name = ?"); + + state->queryAttributes.create(state->db, + "select name from Attributes where parent = ?"); + + state->txn = std::make_unique<SQLiteTxn>(state->db); + } + + ~AttrDb() + { + try { + auto state(_state->lock()); + if (!failed) + state->txn->commit(); + state->txn.reset(); + } catch (...) { + ignoreException(); + } + } + + template<typename F> + AttrId doSQLite(F && fun) + { + if (failed) return 0; + try { + return fun(); + } catch (SQLiteError &) { + ignoreException(); + failed = true; + return 0; + } + } + + AttrId setAttrs( + AttrKey key, + const std::vector<Symbol> & attrs) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::FullAttrs) + (0, false).exec(); + + AttrId rowId = state->db.getLastInsertedRowId(); + assert(rowId); + + for (auto & attr : attrs) + state->insertAttribute.use() + (rowId) + (attr) + (AttrType::Placeholder) + (0, false).exec(); + + return rowId; + }); + } + + AttrId setString( + AttrKey key, + std::string_view s, + const char * * context = nullptr) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + if (context) { + std::string ctx; + for (const char * * p = context; *p; ++p) { + if (p != context) ctx.push_back(' '); + ctx.append(*p); + } + state->insertAttributeWithContext.use() + (key.first) + (key.second) + (AttrType::String) + (s) + (ctx).exec(); + } else { + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::String) + (s).exec(); + } + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setBool( + AttrKey key, + bool b) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Bool) + (b ? 1 : 0).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setPlaceholder(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Placeholder) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setMissing(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Missing) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setMisc(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Misc) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + AttrId setFailed(AttrKey key) + { + return doSQLite([&]() + { + auto state(_state->lock()); + + state->insertAttribute.use() + (key.first) + (key.second) + (AttrType::Failed) + (0, false).exec(); + + return state->db.getLastInsertedRowId(); + }); + } + + std::optional<std::pair<AttrId, AttrValue>> getAttr( + AttrKey key, + SymbolTable & symbols) + { + auto state(_state->lock()); + + auto queryAttribute(state->queryAttribute.use()(key.first)(key.second)); + if (!queryAttribute.next()) return {}; + + auto rowId = (AttrType) queryAttribute.getInt(0); + auto type = (AttrType) queryAttribute.getInt(1); + + switch (type) { + case AttrType::Placeholder: + return {{rowId, placeholder_t()}}; + case AttrType::FullAttrs: { + // FIXME: expensive, should separate this out. + std::vector<Symbol> attrs; + auto queryAttributes(state->queryAttributes.use()(rowId)); + while (queryAttributes.next()) + attrs.push_back(symbols.create(queryAttributes.getStr(0))); + return {{rowId, attrs}}; + } + case AttrType::String: { + std::vector<std::pair<Path, std::string>> context; + if (!queryAttribute.isNull(3)) + for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";")) + context.push_back(decodeContext(s)); + return {{rowId, string_t{queryAttribute.getStr(2), context}}}; + } + case AttrType::Bool: + return {{rowId, queryAttribute.getInt(2) != 0}}; + case AttrType::Missing: + return {{rowId, missing_t()}}; + case AttrType::Misc: + return {{rowId, misc_t()}}; + case AttrType::Failed: + return {{rowId, failed_t()}}; + default: + throw Error("unexpected type in evaluation cache"); + } + } +}; + +static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint) +{ + try { + return std::make_shared<AttrDb>(fingerprint); + } catch (SQLiteError &) { + ignoreException(); + return nullptr; + } +} + +EvalCache::EvalCache( + std::optional<std::reference_wrapper<const Hash>> useCache, + EvalState & state, + RootLoader rootLoader) + : db(useCache ? makeAttrDb(*useCache) : nullptr) + , state(state) + , rootLoader(rootLoader) +{ +} + +Value * EvalCache::getRootValue() +{ + if (!value) { + debug("getting root value"); + value = allocRootValue(rootLoader()); + } + return *value; +} + +std::shared_ptr<AttrCursor> EvalCache::getRoot() +{ + return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt); +} + +AttrCursor::AttrCursor( + ref<EvalCache> root, + Parent parent, + Value * value, + std::optional<std::pair<AttrId, AttrValue>> && cachedValue) + : root(root), parent(parent), cachedValue(std::move(cachedValue)) +{ + if (value) + _value = allocRootValue(value); +} + +AttrKey AttrCursor::getKey() +{ + if (!parent) + return {0, root->state.sEpsilon}; + if (!parent->first->cachedValue) { + parent->first->cachedValue = root->db->getAttr( + parent->first->getKey(), root->state.symbols); + assert(parent->first->cachedValue); + } + return {parent->first->cachedValue->first, parent->second}; +} + +Value & AttrCursor::getValue() +{ + if (!_value) { + if (parent) { + auto & vParent = parent->first->getValue(); + root->state.forceAttrs(vParent); + auto attr = vParent.attrs->get(parent->second); + if (!attr) + throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr()); + _value = allocRootValue(attr->value); + } else + _value = allocRootValue(root->getRootValue()); + } + return **_value; +} + +std::vector<Symbol> AttrCursor::getAttrPath() const +{ + if (parent) { + auto attrPath = parent->first->getAttrPath(); + attrPath.push_back(parent->second); + return attrPath; + } else + return {}; +} + +std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const +{ + auto attrPath = getAttrPath(); + attrPath.push_back(name); + return attrPath; +} + +std::string AttrCursor::getAttrPathStr() const +{ + return concatStringsSep(".", getAttrPath()); +} + +std::string AttrCursor::getAttrPathStr(Symbol name) const +{ + return concatStringsSep(".", getAttrPath(name)); +} + +Value & AttrCursor::forceValue() +{ + debug("evaluating uncached attribute %s", getAttrPathStr()); + + auto & v = getValue(); + + try { + root->state.forceValue(v); + } catch (EvalError &) { + debug("setting '%s' to failed", getAttrPathStr()); + if (root->db) + cachedValue = {root->db->setFailed(getKey()), failed_t()}; + throw; + } + + if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) { + if (v.type() == nString) + cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), + string_t{v.string.s, {}}}; + else if (v.type() == nPath) + cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; + else if (v.type() == nBool) + cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; + else if (v.type() == nAttrs) + ; // FIXME: do something? + else + cachedValue = {root->db->setMisc(getKey()), misc_t()}; + } + + return v; +} + +std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + + if (cachedValue) { + if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) { + for (auto & attr : *attrs) + if (attr == name) + return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name)); + return nullptr; + } else if (std::get_if<placeholder_t>(&cachedValue->second)) { + auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols); + if (attr) { + if (std::get_if<missing_t>(&attr->second)) + return nullptr; + else if (std::get_if<failed_t>(&attr->second)) { + if (forceErrors) + debug("reevaluating failed cached attribute '%s'"); + else + throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); + } else + return std::make_shared<AttrCursor>(root, + std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); + } + // Incomplete attrset, so need to fall thru and + // evaluate to see whether 'name' exists + } else + return nullptr; + //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type() != nAttrs) + return nullptr; + //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + + auto attr = v.attrs->get(name); + + if (!attr) { + if (root->db) { + if (!cachedValue) + cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; + root->db->setMissing({cachedValue->first, name}); + } + return nullptr; + } + + std::optional<std::pair<AttrId, AttrValue>> cachedValue2; + if (root->db) { + if (!cachedValue) + cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()}; + cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()}; + } + + return std::make_shared<AttrCursor>( + root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2)); +} + +std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name) +{ + return maybeGetAttr(root->state.symbols.create(name)); +} + +std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name, bool forceErrors) +{ + auto p = maybeGetAttr(name, forceErrors); + if (!p) + throw Error("attribute '%s' does not exist", getAttrPathStr(name)); + return p; +} + +std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name) +{ + return getAttr(root->state.symbols.create(name)); +} + +std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force) +{ + auto res = shared_from_this(); + for (auto & attr : attrPath) { + res = res->maybeGetAttr(attr, force); + if (!res) return {}; + } + return res; +} + +std::string AttrCursor::getString() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) { + if (auto s = std::get_if<string_t>(&cachedValue->second)) { + debug("using cached string attribute '%s'", getAttrPathStr()); + return s->first; + } else + throw TypeError("'%s' is not a string", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type() != nString && v.type() != nPath) + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); + + return v.type() == nString ? v.string.s : v.path; +} + +string_t AttrCursor::getStringWithContext() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) { + if (auto s = std::get_if<string_t>(&cachedValue->second)) { + bool valid = true; + for (auto & c : s->second) { + if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) { + valid = false; + break; + } + } + if (valid) { + debug("using cached string attribute '%s'", getAttrPathStr()); + return *s; + } + } else + throw TypeError("'%s' is not a string", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type() == nString) + return {v.string.s, v.getContext()}; + else if (v.type() == nPath) + return {v.path, {}}; + else + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); +} + +bool AttrCursor::getBool() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) { + if (auto b = std::get_if<bool>(&cachedValue->second)) { + debug("using cached Boolean attribute '%s'", getAttrPathStr()); + return *b; + } else + throw TypeError("'%s' is not a Boolean", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type() != nBool) + throw TypeError("'%s' is not a Boolean", getAttrPathStr()); + + return v.boolean; +} + +std::vector<Symbol> AttrCursor::getAttrs() +{ + if (root->db) { + if (!cachedValue) + cachedValue = root->db->getAttr(getKey(), root->state.symbols); + if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) { + if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) { + debug("using cached attrset attribute '%s'", getAttrPathStr()); + return *attrs; + } else + throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + } + } + + auto & v = forceValue(); + + if (v.type() != nAttrs) + throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + + std::vector<Symbol> attrs; + for (auto & attr : *getValue().attrs) + attrs.push_back(attr.name); + std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) { + return (const string &) a < (const string &) b; + }); + + if (root->db) + cachedValue = {root->db->setAttrs(getKey(), attrs), attrs}; + + return attrs; +} + +bool AttrCursor::isDerivation() +{ + auto aType = maybeGetAttr("type"); + return aType && aType->getString() == "derivation"; +} + +StorePath AttrCursor::forceDerivation() +{ + auto aDrvPath = getAttr(root->state.sDrvPath, true); + auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); + if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { + /* The eval cache contains 'drvPath', but the actual path has + been garbage-collected. So force it to be regenerated. */ + aDrvPath->forceValue(); + if (!root->state.store->isValidPath(drvPath)) + throw Error("don't know how to recreate store derivation '%s'!", + root->state.store->printStorePath(drvPath)); + } + return drvPath; +} + +} diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh new file mode 100644 index 000000000..43b34ebcb --- /dev/null +++ b/src/libexpr/eval-cache.hh @@ -0,0 +1,123 @@ +#pragma once + +#include "sync.hh" +#include "hash.hh" +#include "eval.hh" + +#include <functional> +#include <variant> + +namespace nix::eval_cache { + +MakeError(CachedEvalError, EvalError); + +struct AttrDb; +class AttrCursor; + +class EvalCache : public std::enable_shared_from_this<EvalCache> +{ + friend class AttrCursor; + + std::shared_ptr<AttrDb> db; + EvalState & state; + typedef std::function<Value *()> RootLoader; + RootLoader rootLoader; + RootValue value; + + Value * getRootValue(); + +public: + + EvalCache( + std::optional<std::reference_wrapper<const Hash>> useCache, + EvalState & state, + RootLoader rootLoader); + + std::shared_ptr<AttrCursor> getRoot(); +}; + +enum AttrType { + Placeholder = 0, + FullAttrs = 1, + String = 2, + Missing = 3, + Misc = 4, + Failed = 5, + Bool = 6, +}; + +struct placeholder_t {}; +struct missing_t {}; +struct misc_t {}; +struct failed_t {}; +typedef uint64_t AttrId; +typedef std::pair<AttrId, Symbol> AttrKey; +typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t; + +typedef std::variant< + std::vector<Symbol>, + string_t, + placeholder_t, + missing_t, + misc_t, + failed_t, + bool + > AttrValue; + +class AttrCursor : public std::enable_shared_from_this<AttrCursor> +{ + friend class EvalCache; + + ref<EvalCache> root; + typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent; + Parent parent; + RootValue _value; + std::optional<std::pair<AttrId, AttrValue>> cachedValue; + + AttrKey getKey(); + + Value & getValue(); + +public: + + AttrCursor( + ref<EvalCache> root, + Parent parent, + Value * value = nullptr, + std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {}); + + std::vector<Symbol> getAttrPath() const; + + std::vector<Symbol> getAttrPath(Symbol name) const; + + std::string getAttrPathStr() const; + + std::string getAttrPathStr(Symbol name) const; + + std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name, bool forceErrors = false); + + std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name); + + std::shared_ptr<AttrCursor> getAttr(Symbol name, bool forceErrors = false); + + std::shared_ptr<AttrCursor> getAttr(std::string_view name); + + std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath, bool force = false); + + std::string getString(); + + string_t getStringWithContext(); + + bool getBool(); + + std::vector<Symbol> getAttrs(); + + bool isDerivation(); + + Value & forceValue(); + + /* Force creation of the .drv file in the Nix store. */ + StorePath forceDerivation(); +}; + +} diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 30f6ec7db..655408cd3 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -10,7 +10,7 @@ namespace nix { LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s)) { throw EvalError({ - .hint = hintfmt(s), + .msg = hintfmt(s), .errPos = pos }); } @@ -24,7 +24,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v)) { throw TypeError({ - .hint = hintfmt(s, showType(v)), + .msg = hintfmt(s, showType(v)), .errPos = pos }); } @@ -32,23 +32,21 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const void EvalState::forceValue(Value & v, const Pos & pos) { - if (v.type == tThunk) { + if (v.isThunk()) { Env * env = v.thunk.env; Expr * expr = v.thunk.expr; try { - v.type = tBlackhole; + v.mkBlackhole(); //checkInterrupt(); expr->eval(*this, *env, v); } catch (...) { - v.type = tThunk; - v.thunk.env = env; - v.thunk.expr = expr; + v.mkThunk(env, expr); throw; } } - else if (v.type == tApp) + else if (v.isApp()) callFunction(*v.app.left, *v.app.right, v, noPos); - else if (v.type == tBlackhole) + else if (v.isBlackhole()) throwEvalError(pos, "infinite recursion encountered"); } @@ -56,7 +54,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) inline void EvalState::forceAttrs(Value & v) { forceValue(v); - if (v.type != tAttrs) + if (v.type() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -64,7 +62,7 @@ inline void EvalState::forceAttrs(Value & v) inline void EvalState::forceAttrs(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tAttrs) + if (v.type() != nAttrs) throwTypeError(pos, "value is %1% while a set was expected", v); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c1a9af9b2..bc41a2cd9 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -27,6 +27,10 @@ #include <gc/gc.h> #include <gc/gc_cpp.h> +#include <boost/coroutine2/coroutine.hpp> +#include <boost/coroutine2/protected_fixedsize_stack.hpp> +#include <boost/context/stack_context.hpp> + #endif namespace nix { @@ -60,11 +64,15 @@ static char * dupStringWithLen(const char * s, size_t size) RootValue allocRootValue(Value * v) { +#if HAVE_BOEHMGC return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v); +#else + return std::make_shared<Value *>(v); +#endif } -static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) +void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) { checkInterrupt(); @@ -73,7 +81,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con return; } - switch (v.type) { + switch (v.internalType) { case tInt: str << v.integer; break; @@ -87,6 +95,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con else if (*i == '\n') str << "\\n"; else if (*i == '\r') str << "\\r"; else if (*i == '\t') str << "\\t"; + else if (*i == '$' && *(i+1) == '{') str << "\\" << *i; else str << *i; str << "\""; break; @@ -153,32 +162,27 @@ std::ostream & operator << (std::ostream & str, const Value & v) const Value *getPrimOp(const Value &v) { const Value * primOp = &v; - while (primOp->type == tPrimOpApp) { + while (primOp->isPrimOpApp()) { primOp = primOp->primOpApp.left; } - assert(primOp->type == tPrimOp); + assert(primOp->isPrimOp()); return primOp; } - string showType(ValueType type) { switch (type) { - case tInt: return "an integer"; - case tBool: return "a Boolean"; - case tString: return "a string"; - case tPath: return "a path"; - case tNull: return "null"; - case tAttrs: return "a set"; - case tList1: case tList2: case tListN: return "a list"; - case tThunk: return "a thunk"; - case tApp: return "a function application"; - case tLambda: return "a function"; - case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; - case tExternal: return "an external value"; - case tFloat: return "a float"; + case nInt: return "an integer"; + case nBool: return "a Boolean"; + case nString: return "a string"; + case nPath: return "a path"; + case nNull: return "null"; + case nAttrs: return "a set"; + case nList: return "a list"; + case nFunction: return "a function"; + case nExternal: return "an external value"; + case nFloat: return "a float"; + case nThunk: return "a thunk"; } abort(); } @@ -186,18 +190,43 @@ string showType(ValueType type) string showType(const Value & v) { - switch (v.type) { + switch (v.internalType) { case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: return fmt("the built-in function '%s'", string(v.primOp->name)); case tPrimOpApp: return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); + case tThunk: return "a thunk"; + case tApp: return "a function application"; + case tBlackhole: return "a black hole"; default: - return showType(v.type); + return showType(v.type()); } } +Pos Value::determinePos(const Pos &pos) const +{ + switch (internalType) { + case tAttrs: return *attrs->pos; + case tLambda: return lambda.fun->pos; + case tApp: return app.left->determinePos(pos); + default: return pos; + } +} + +bool Value::isTrivial() const +{ + return + internalType != tApp + && internalType != tPrimOpApp + && (internalType != tThunk + || (dynamic_cast<ExprAttrs *>(thunk.expr) + && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) + || dynamic_cast<ExprLambda *>(thunk.expr) + || dynamic_cast<ExprList *>(thunk.expr)); +} + #if HAVE_BOEHMGC /* Called when the Boehm GC runs out of memory. */ @@ -206,6 +235,43 @@ static void * oomHandler(size_t requested) /* Convert this to a proper C++ exception. */ throw std::bad_alloc(); } + +class BoehmGCStackAllocator : public StackAllocator { + boost::coroutines2::protected_fixedsize_stack stack { + // We allocate 8 MB, the default max stack size on NixOS. + // A smaller stack might be quicker to allocate but reduces the stack + // depth available for source filter expressions etc. + std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024)) + }; + + // This is specific to boost::coroutines2::protected_fixedsize_stack. + // The stack protection page is included in sctx.size, so we have to + // subtract one page size from the stack size. + std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) { + return sctx.size - boost::context::stack_traits::page_size(); + } + + public: + boost::context::stack_context allocate() override { + auto sctx = stack.allocate(); + + // Stacks generally start at a high address and grow to lower addresses. + // Architectures that do the opposite are rare; in fact so rare that + // boost_routine does not implement it. + // So we subtract the stack size. + GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); + return sctx; + } + + void deallocate(boost::context::stack_context sctx) override { + GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); + stack.deallocate(sctx); + } + +}; + +static BoehmGCStackAllocator boehmGCStackAllocator; + #endif @@ -243,6 +309,8 @@ void initGC() GC_set_oom_fn(oomHandler); + StackAllocator::defaultAllocator = &boehmGCStackAllocator; + /* Set the initial heap size to something fairly big (25% of physical RAM, up to a maximum of 384 MiB) so that in most cases we don't need to garbage collect at all. (Collection has a @@ -310,7 +378,10 @@ static Strings parseNixPath(const string & s) } -EvalState::EvalState(const Strings & _searchPath, ref<Store> store) +EvalState::EvalState( + const Strings & _searchPath, + ref<Store> store, + std::shared_ptr<Store> buildStore) : sWith(symbols.create("<with>")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) @@ -333,12 +404,18 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sStructuredAttrs(symbols.create("__structuredAttrs")) , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) + , sContentAddressed(symbols.create("__contentAddressed")) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) , sRecurseForDerivations(symbols.create("recurseForDerivations")) + , sDescription(symbols.create("description")) + , sSelf(symbols.create("self")) + , sEpsilon(symbols.create("")) , repair(NoRepair) , store(store) + , buildStore(buildStore ? buildStore : store) + , regexCache(makeRegexCache()) , baseEnv(allocEnv(128)) , staticBaseEnv(false, 0) { @@ -353,7 +430,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) for (auto & i : _searchPath) addToSearchPath(i); for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); } - addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); @@ -365,18 +441,20 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) auto path = r.second; if (store->isInStore(r.second)) { - StorePathSet closure; - store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure); - for (auto & path : closure) - allowedPaths->insert(store->printStorePath(path)); + try { + StorePathSet closure; + store->computeFSClosure(store->toStorePath(r.second).first, closure); + for (auto & path : closure) + allowedPaths->insert(store->printStorePath(path)); + } catch (InvalidPath &) { + allowedPaths->insert(r.second); + } } else allowedPaths->insert(r.second); } } - clearValue(vEmptySet); - vEmptySet.type = tAttrs; - vEmptySet.attrs = allocBindings(0); + vEmptySet.mkAttrs(allocBindings(0)); createBaseEnv(); } @@ -403,6 +481,8 @@ Path EvalState::checkSourcePath(const Path & path_) */ Path abspath = canonPath(path_); + if (hasPrefix(abspath, corepkgsPrefix)) return abspath; + for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { found = true; @@ -492,16 +572,14 @@ Value * EvalState::addPrimOp(const string & name, the primop to a dummy value. */ if (arity == 0) { auto vPrimOp = allocValue(); - vPrimOp->type = tPrimOp; - vPrimOp->primOp = new PrimOp(primOp, 1, sym); + vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym }); Value v; mkApp(v, *vPrimOp, *vPrimOp); return addConstant(name, v); } Value * v = allocValue(); - v->type = tPrimOp; - v->primOp = new PrimOp(primOp, arity, sym); + v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym }); staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); @@ -509,12 +587,55 @@ Value * EvalState::addPrimOp(const string & name, } +Value * EvalState::addPrimOp(PrimOp && primOp) +{ + /* Hack to make constants lazy: turn them into a application of + the primop to a dummy value. */ + if (primOp.arity == 0) { + primOp.arity = 1; + auto vPrimOp = allocValue(); + vPrimOp->mkPrimOp(new PrimOp(std::move(primOp))); + Value v; + mkApp(v, *vPrimOp, *vPrimOp); + return addConstant(primOp.name, v); + } + + Symbol envName = primOp.name; + if (hasPrefix(primOp.name, "__")) + primOp.name = symbols.create(std::string(primOp.name, 2)); + + Value * v = allocValue(); + v->mkPrimOp(new PrimOp(std::move(primOp))); + staticBaseEnv.vars[envName] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); + return v; +} + + Value & EvalState::getBuiltin(const string & name) { return *baseEnv.values[0]->attrs->find(symbols.create(name))->value; } +std::optional<EvalState::Doc> EvalState::getDoc(Value & v) +{ + if (v.isPrimOp()) { + auto v2 = &v; + if (v2->primOp->doc) + return Doc { + .pos = noPos, + .name = v2->primOp->name, + .arity = v2->primOp->arity, + .args = v2->primOp->args, + .doc = v2->primOp->doc, + }; + } + return {}; +} + + /* Every "format" object (even temporary) takes up a few hundred bytes of stack space, which is a real killer in the recursive evaluator. So here are some helper functions for throwing @@ -528,7 +649,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2)) { throw EvalError({ - .hint = hintfmt(s, s2), + .msg = hintfmt(s, s2), .errPos = pos }); } @@ -541,7 +662,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3)) { throw EvalError({ - .hint = hintfmt(s, s2, s3), + .msg = hintfmt(s, s2, s3), .errPos = pos }); } @@ -550,7 +671,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const { // p1 is where the error occurred; p2 is a position mentioned in the message. throw EvalError({ - .hint = hintfmt(s, sym, p2), + .msg = hintfmt(s, sym, p2), .errPos = p1 }); } @@ -558,20 +679,15 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) { throw TypeError({ - .hint = hintfmt(s), + .msg = hintfmt(s), .errPos = pos }); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) -{ - throw TypeError(s, s1); -} - LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { throw TypeError({ - .hint = hintfmt(s, fun.showNamePos(), s2), + .msg = hintfmt(s, fun.showNamePos(), s2), .errPos = pos }); } @@ -579,7 +695,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1)) { throw AssertionError({ - .hint = hintfmt(s, s1), + .msg = hintfmt(s, s1), .errPos = pos }); } @@ -587,7 +703,15 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1)) { throw UndefinedVarError({ - .hint = hintfmt(s, s1), + .msg = hintfmt(s, s1), + .errPos = pos + }); +} + +LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1)) +{ + throw MissingArgumentError({ + .msg = hintfmt(s, s1), .errPos = pos }); } @@ -605,15 +729,13 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con void mkString(Value & v, const char * s) { - mkStringNoCopy(v, dupString(s)); + v.mkString(dupString(s)); } Value & mkString(Value & v, std::string_view s, const PathSet & context) { - v.type = tString; - v.string.s = dupStringWithLen(s.data(), s.size()); - v.string.context = 0; + v.mkString(dupStringWithLen(s.data(), s.size())); if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) @@ -628,7 +750,7 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context) void mkPath(Value & v, const char * s) { - mkPathNoCopy(v, dupString(s)); + v.mkPath(dupString(s)); } @@ -648,7 +770,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) } Bindings::iterator j = env->values[0]->attrs->find(var.name); if (j != env->values[0]->attrs->end()) { - if (countCalls && j->pos) attrSelects[*j->pos]++; + if (countCalls) attrSelects[*j->pos]++; return j->value; } if (!env->prevWith) @@ -658,18 +780,10 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) } -std::atomic<uint64_t> nrValuesFreed{0}; - -void finalizeValue(void * obj, void * data) -{ - nrValuesFreed++; -} - Value * EvalState::allocValue() { nrValues++; auto v = (Value *) allocBytes(sizeof(Value)); - //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr); return v; } @@ -689,16 +803,9 @@ Env & EvalState::allocEnv(size_t size) void EvalState::mkList(Value & v, size_t size) { - clearValue(v); - if (size == 1) - v.type = tList1; - else if (size == 2) - v.type = tList2; - else { - v.type = tListN; - v.bigList.size = size; - v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0; - } + v.mkList(size); + if (size > 2) + v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *)); nrListElems += size; } @@ -707,9 +814,7 @@ unsigned long nrThunks = 0; static inline void mkThunk(Value & v, Env & env, Expr * expr) { - v.type = tThunk; - v.thunk.env = &env; - v.thunk.expr = expr; + v.mkThunk(&env, expr); nrThunks++; } @@ -720,9 +825,9 @@ void EvalState::mkThunk_(Value & v, Expr * expr) } -void EvalState::mkPos(Value & v, Pos * pos) +void EvalState::mkPos(Value & v, ptr<Pos> pos) { - if (pos && pos->file.set()) { + if (pos->file.set()) { mkAttrs(v, 3); mkString(*allocAttr(v, sFile), pos->file); mkInt(*allocAttr(v, sLine), pos->line); @@ -745,44 +850,42 @@ Value * Expr::maybeThunk(EvalState & state, Env & env) } -unsigned long nrAvoided = 0; - Value * ExprVar::maybeThunk(EvalState & state, Env & env) { Value * v = state.lookupVar(&env, *this, true); /* The value might not be initialised in the environment yet. In that case, ignore it. */ - if (v) { nrAvoided++; return v; } + if (v) { state.nrAvoided++; return v; } return Expr::maybeThunk(state, env); } Value * ExprString::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } Value * ExprInt::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } Value * ExprFloat::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } Value * ExprPath::maybeThunk(EvalState & state, Env & env) { - nrAvoided++; + state.nrAvoided++; return &v; } -void EvalState::evalFile(const Path & path_, Value & v) +void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) { auto path = checkSourcePath(path_); @@ -792,33 +895,23 @@ void EvalState::evalFile(const Path & path_, Value & v) return; } - Path path2 = resolveExprPath(path); - if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) { + Path resolvedPath = resolveExprPath(path); + if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) { v = i->second; return; } - printTalkative("evaluating file '%1%'", path2); + printTalkative("evaluating file '%1%'", resolvedPath); Expr * e = nullptr; - auto j = fileParseCache.find(path2); + auto j = fileParseCache.find(resolvedPath); if (j != fileParseCache.end()) e = j->second; if (!e) - e = parseExprFromFile(checkSourcePath(path2)); - - fileParseCache[path2] = e; + e = parseExprFromFile(checkSourcePath(resolvedPath)); - try { - eval(e, v); - } catch (Error & e) { - addErrorTrace(e, "while evaluating the file '%1%':", path2); - throw; - } - - fileEvalCache[path2] = v; - if (path != path2) fileEvalCache[path] = v; + cacheFile(path, resolvedPath, e, v, mustBeTrivial); } @@ -829,6 +922,32 @@ void EvalState::resetFileCache() } +void EvalState::cacheFile( + const Path & path, + const Path & resolvedPath, + Expr * e, + Value & v, + bool mustBeTrivial) +{ + fileParseCache[resolvedPath] = e; + + try { + // Enforce that 'flake.nix' is a direct attrset, not a + // computation. + if (mustBeTrivial && + !(dynamic_cast<ExprAttrs *>(e))) + throw EvalError("file '%s' must be an attribute set", path); + eval(e, v); + } catch (Error & e) { + addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath); + throw; + } + + fileEvalCache[resolvedPath] = v; + if (path != resolvedPath) fileEvalCache[path] = v; +} + + void EvalState::eval(Expr * e, Value & v) { e->eval(*this, baseEnv, v); @@ -839,7 +958,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e) { Value v; e->eval(*this, env, v); - if (v.type != tBool) + if (v.type() != nBool) throwTypeError("value is %1% while a Boolean was expected", v); return v.boolean; } @@ -849,7 +968,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) { Value v; e->eval(*this, env, v); - if (v.type != tBool) + if (v.type() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -858,7 +977,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) { e->eval(*this, env, v); - if (v.type != tAttrs) + if (v.type() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -919,7 +1038,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) } else vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); env2.values[displ++] = vAttr; - v.attrs->push_back(Attr(i.first, vAttr, &i.second.pos)); + v.attrs->push_back(Attr(i.first, vAttr, ptr(&i.second.pos))); } /* If the rec contains an attribute called `__overrides', then @@ -951,14 +1070,14 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) else for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos)); + v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), ptr(&i.second.pos))); /* Dynamic attrs apply *after* rec and __overrides. */ for (auto & i : dynamicAttrs) { Value nameVal; i.nameExpr->eval(state, *dynamicEnv, nameVal); state.forceValue(nameVal, i.pos); - if (nameVal.type == tNull) + if (nameVal.type() == nNull) continue; state.forceStringNoCtx(nameVal); Symbol nameSym = state.symbols.create(nameVal.string.s); @@ -968,9 +1087,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ - v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos)); + v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), ptr(&i.pos))); v.attrs->sort(); // FIXME: inefficient } + + v.attrs->pos = ptr(&pos); } @@ -1025,12 +1146,10 @@ static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPa } -unsigned long nrLookups = 0; - void ExprSelect::eval(EvalState & state, Env & env, Value & v) { Value vTmp; - Pos * pos2 = 0; + ptr<Pos> pos2(&noPos); Value * vAttrs = &vTmp; e->eval(state, env, vTmp); @@ -1038,12 +1157,12 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) try { for (auto & i : attrPath) { - nrLookups++; + state.nrLookups++; Bindings::iterator j; Symbol name = getName(i, state, env); if (def) { state.forceValue(*vAttrs, pos); - if (vAttrs->type != tAttrs || + if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { def->eval(state, env, v); @@ -1056,13 +1175,13 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } vAttrs = j->value; pos2 = j->pos; - if (state.countCalls && pos2) state.attrSelects[*pos2]++; + if (state.countCalls) state.attrSelects[*pos2]++; } - state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) ); + state.forceValue(*vAttrs, (*pos2 != noPos ? *pos2 : this->pos ) ); } catch (Error & e) { - if (pos2 && pos2->file != state.sDerivationNix) + if (*pos2 != noPos && pos2->file != state.sDerivationNix) addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath)); throw; @@ -1083,7 +1202,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) state.forceValue(*vAttrs); Bindings::iterator j; Symbol name = getName(i, state, env); - if (vAttrs->type != tAttrs || + if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { mkBool(v, false); @@ -1099,9 +1218,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) void ExprLambda::eval(EvalState & state, Env & env, Value & v) { - v.type = tLambda; - v.lambda.env = &env; - v.lambda.fun = this; + v.mkLambda(&env, this); } @@ -1119,11 +1236,11 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) /* Figure out the number of arguments still needed. */ size_t argsDone = 0; Value * primOp = &fun; - while (primOp->type == tPrimOpApp) { + while (primOp->isPrimOpApp()) { argsDone++; primOp = primOp->primOpApp.left; } - assert(primOp->type == tPrimOp); + assert(primOp->isPrimOp()); auto arity = primOp->primOp->arity; auto argsLeft = arity - argsDone; @@ -1134,7 +1251,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) Value * vArgs[arity]; auto n = arity - 1; vArgs[n--] = &arg; - for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) + for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[n--] = arg->primOpApp.right; /* And call the primop. */ @@ -1144,9 +1261,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } else { Value * fun2 = allocValue(); *fun2 = fun; - v.type = tPrimOpApp; - v.primOpApp.left = fun2; - v.primOpApp.right = &arg; + v.mkPrimOpApp(fun2, &arg); } } @@ -1156,12 +1271,12 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po forceValue(fun, pos); - if (fun.type == tPrimOp || fun.type == tPrimOpApp) { + if (fun.isPrimOp() || fun.isPrimOpApp()) { callPrimOp(fun, arg, v, pos); return; } - if (fun.type == tAttrs) { + if (fun.type() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { /* fun may be allocated on the stack of the calling function, @@ -1177,7 +1292,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po } } - if (fun.type != tLambda) + if (!fun.isLambda()) throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); ExprLambda & lambda(*fun.lambda.fun); @@ -1236,10 +1351,10 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po try { lambda.body->eval(*this, env2, v); } catch (Error & e) { - addErrorTrace(e, lambda.pos, "while evaluating %s", - (lambda.name.set() - ? "'" + (string) lambda.name + "'" - : "anonymous lambdaction")); + addErrorTrace(e, lambda.pos, "while evaluating %s", + (lambda.name.set() + ? "'" + (string) lambda.name + "'" + : "anonymous lambda")); addErrorTrace(e, pos, "from call site%s", ""); throw; } @@ -1260,7 +1375,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) { forceValue(fun); - if (fun.type == tAttrs) { + if (fun.type() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { Value * v = allocValue(); @@ -1270,20 +1385,37 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } } - if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { + if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) { res = fun; return; } Value * actualArgs = allocValue(); - mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size()); - - for (auto & i : fun.lambda.fun->formals->formals) { - Bindings::iterator j = args.find(i.name); - if (j != args.end()) - actualArgs->attrs->push_back(*j); - else if (!i.def) - throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name); + mkAttrs(*actualArgs, std::max(static_cast<uint32_t>(fun.lambda.fun->formals->formals.size()), args.size())); + + if (fun.lambda.fun->formals->ellipsis) { + // If the formals have an ellipsis (eg the function accepts extra args) pass + // all available automatic arguments (which includes arguments specified on + // the command line via --arg/--argstr) + for (auto& v : args) { + actualArgs->attrs->push_back(v); + } + } else { + // Otherwise, only pass the arguments that the function accepts + for (auto & i : fun.lambda.fun->formals->formals) { + Bindings::iterator j = args.find(i.name); + if (j != args.end()) { + actualArgs->attrs->push_back(*j); + } else if (!i.def) { + throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') + +Nix attempted to evaluate a function as a top level expression; in +this case it must have its arguments supplied either by default +values, or passed explicitly with '--arg' or '--argstr'. See +https://nixos.org/manual/nix/stable/#ss-functions.)", i.name); + + } + } } actualArgs->attrs->sort(); @@ -1315,7 +1447,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) if (!state.evalBool(env, cond, pos)) { std::ostringstream out; cond->show(out); - throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str()); + throwAssertionError(pos, "assertion '%1%' failed", out.str()); } body->eval(state, env, v); } @@ -1443,7 +1575,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) NixFloat nf = 0; bool first = !forceString; - ValueType firstType = tString; + ValueType firstType = nString; for (auto & i : *es) { Value vTmp; @@ -1454,36 +1586,40 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) since paths are copied when they are used in a derivation), and none of the strings are allowed to have contexts. */ if (first) { - firstType = vTmp.type; - first = false; + firstType = vTmp.type(); } - if (firstType == tInt) { - if (vTmp.type == tInt) { + if (firstType == nInt) { + if (vTmp.type() == nInt) { n += vTmp.integer; - } else if (vTmp.type == tFloat) { + } else if (vTmp.type() == nFloat) { // Upgrade the type from int to float; - firstType = tFloat; + firstType = nFloat; nf = n; nf += vTmp.fpoint; } else throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); - } else if (firstType == tFloat) { - if (vTmp.type == tInt) { + } else if (firstType == nFloat) { + if (vTmp.type() == nInt) { nf += vTmp.integer; - } else if (vTmp.type == tFloat) { + } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint; } else throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); } else - s << state.coerceToString(pos, vTmp, context, false, firstType == tString); + /* skip canonization of first path, which would only be not + canonized in the first place if it's coming from a ./${foo} type + path */ + s << state.coerceToString(pos, vTmp, context, false, firstType == nString, !first); + + first = false; } - if (firstType == tInt) + if (firstType == nInt) mkInt(v, n); - else if (firstType == tFloat) + else if (firstType == nFloat) mkFloat(v, nf); - else if (firstType == tPath) { + else if (firstType == nPath) { if (!context.empty()) throwEvalError(pos, "a string that refers to a store path cannot be appended to a path"); auto path = canonPath(s.str()); @@ -1495,7 +1631,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) void ExprPos::eval(EvalState & state, Env & env, Value & v) { - state.mkPos(v, &pos); + state.mkPos(v, ptr(&pos)); } @@ -1510,7 +1646,7 @@ void EvalState::forceValueDeep(Value & v) forceValue(v); - if (v.type == tAttrs) { + if (v.type() == nAttrs) { for (auto & i : *v.attrs) try { recurse(*i.value); @@ -1533,7 +1669,7 @@ void EvalState::forceValueDeep(Value & v) NixInt EvalState::forceInt(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tInt) + if (v.type() != nInt) throwTypeError(pos, "value is %1% while an integer was expected", v); return v.integer; } @@ -1542,9 +1678,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos) NixFloat EvalState::forceFloat(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type == tInt) + if (v.type() == nInt) return v.integer; - else if (v.type != tFloat) + else if (v.type() != nFloat) throwTypeError(pos, "value is %1% while a float was expected", v); return v.fpoint; } @@ -1553,7 +1689,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) bool EvalState::forceBool(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tBool) + if (v.type() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -1561,14 +1697,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos) bool EvalState::isFunctor(Value & fun) { - return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); + return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); } void EvalState::forceFunction(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) + if (v.type() != nFunction && !isFunctor(v)) throwTypeError(pos, "value is %1% while a function was expected", v); } @@ -1576,7 +1712,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos) string EvalState::forceString(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tString) { + if (v.type() != nString) { if (pos) throwTypeError(pos, "value is %1% while a string was expected", v); else @@ -1586,6 +1722,18 @@ string EvalState::forceString(Value & v, const Pos & pos) } +/* Decode a context string ‘!<name>!<path>’ into a pair <path, + name>. */ +std::pair<string, string> decodeContext(std::string_view s) +{ + if (s.at(0) == '!') { + size_t index = s.find("!", 1); + return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))}; + } else + return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""}; +} + + void copyContext(const Value & v, PathSet & context) { if (v.string.context) @@ -1594,6 +1742,17 @@ void copyContext(const Value & v, PathSet & context) } +std::vector<std::pair<Path, std::string>> Value::getContext() +{ + std::vector<std::pair<Path, std::string>> res; + assert(internalType == tString); + if (string.context) + for (const char * * p = string.context; *p; ++p) + res.push_back(decodeContext(*p)); + return res; +} + + string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) { string s = forceString(v, pos); @@ -1619,11 +1778,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) bool EvalState::isDerivation(Value & v) { - if (v.type != tAttrs) return false; + if (v.type() != nAttrs) return false; Bindings::iterator i = v.attrs->find(sType); if (i == v.attrs->end()) return false; forceValue(*i->value); - if (i->value->type != tString) return false; + if (i->value->type() != nString) return false; return strcmp(i->value->string.s, "derivation") == 0; } @@ -1642,23 +1801,23 @@ std::optional<string> EvalState::tryAttrsToString(const Pos & pos, Value & v, } string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, - bool coerceMore, bool copyToStore) + bool coerceMore, bool copyToStore, bool canonicalizePath) { forceValue(v, pos); string s; - if (v.type == tString) { + if (v.type() == nString) { copyContext(v, context); return v.string.s; } - if (v.type == tPath) { - Path path(canonPath(v.path)); + if (v.type() == nPath) { + Path path(canonicalizePath ? canonPath(v.path) : v.path); return copyToStore ? copyPathToStore(context, path) : path; } - if (v.type == tAttrs) { + if (v.type() == nAttrs) { auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); if (maybeString) { return *maybeString; @@ -1668,18 +1827,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } - if (v.type == tExternal) + if (v.type() == nExternal) return v.external->coerceToString(pos, context, coerceMore, copyToStore); if (coerceMore) { /* Note that `false' is represented as an empty string for shell scripting convenience, just like `null'. */ - if (v.type == tBool && v.boolean) return "1"; - if (v.type == tBool && !v.boolean) return ""; - if (v.type == tInt) return std::to_string(v.integer); - if (v.type == tFloat) return std::to_string(v.fpoint); - if (v.type == tNull) return ""; + if (v.type() == nBool && v.boolean) return "1"; + if (v.type() == nBool && !v.boolean) return ""; + if (v.type() == nInt) return std::to_string(v.integer); + if (v.type() == nFloat) return std::to_string(v.fpoint); + if (v.type() == nNull) return ""; if (v.isList()) { string result; @@ -1742,40 +1901,38 @@ bool EvalState::eqValues(Value & v1, Value & v2) if (&v1 == &v2) return true; // Special case type-compatibility between float and int - if (v1.type == tInt && v2.type == tFloat) + if (v1.type() == nInt && v2.type() == nFloat) return v1.integer == v2.fpoint; - if (v1.type == tFloat && v2.type == tInt) + if (v1.type() == nFloat && v2.type() == nInt) return v1.fpoint == v2.integer; // All other types are not compatible with each other. - if (v1.type != v2.type) return false; + if (v1.type() != v2.type()) return false; - switch (v1.type) { + switch (v1.type()) { - case tInt: + case nInt: return v1.integer == v2.integer; - case tBool: + case nBool: return v1.boolean == v2.boolean; - case tString: + case nString: return strcmp(v1.string.s, v2.string.s) == 0; - case tPath: + case nPath: return strcmp(v1.path, v2.path) == 0; - case tNull: + case nNull: return true; - case tList1: - case tList2: - case tListN: + case nList: if (v1.listSize() != v2.listSize()) return false; for (size_t n = 0; n < v1.listSize(); ++n) if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; return true; - case tAttrs: { + case nAttrs: { /* If both sets denote a derivation (type = "derivation"), then compare their outPaths. */ if (isDerivation(v1) && isDerivation(v2)) { @@ -1797,15 +1954,13 @@ bool EvalState::eqValues(Value & v1, Value & v2) } /* Functions are incomparable. */ - case tLambda: - case tPrimOp: - case tPrimOpApp: + case nFunction: return false; - case tExternal: + case nExternal: return *v1.external == *v2.external; - case tFloat: + case nFloat: return v1.fpoint == v2.fpoint; default: @@ -1934,7 +2089,7 @@ void EvalState::printStats() string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const { throw TypeError({ - .hint = hintfmt("cannot coerce %1% to a string", showType()), + .msg = hintfmt("cannot coerce %1% to a string", showType()), .errPos = pos }); } @@ -1960,16 +2115,28 @@ EvalSettings::EvalSettings() Strings EvalSettings::getDefaultNixPath() { Strings res; - auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } }; - add(getHome() + "/.nix-defexpr/channels"); - add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs"); - add(settings.nixStateDir + "/nix/profiles/per-user/root/channels"); + auto add = [&](const Path & p, const std::string & s = std::string()) { + if (pathExists(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); + } + } + }; + + if (!evalSettings.restrictEval && !evalSettings.pureEval) { + add(getHome() + "/.nix-defexpr/channels"); + add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs"); + add(settings.nixStateDir + "/profiles/per-user/root/channels"); + } + return res; } EvalSettings evalSettings; -static GlobalConfig::Register r1(&evalSettings); +static GlobalConfig::Register rEvalSettings(&evalSettings); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0d52a7f63..b29feb134 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -4,13 +4,12 @@ #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" -#include "hash.hh" #include "config.hh" -#include <regex> #include <map> #include <optional> #include <unordered_map> +#include <mutex> namespace nix { @@ -30,8 +29,8 @@ struct PrimOp PrimOpFun fun; size_t arity; Symbol name; - PrimOp(PrimOpFun fun, size_t arity, Symbol name) - : fun(fun), arity(arity), name(name) { } + std::vector<std::string> args; + const char * doc = nullptr; }; @@ -65,6 +64,11 @@ typedef std::list<SearchPathElem> SearchPath; void initGC(); +struct RegexCache; + +std::shared_ptr<RegexCache> makeRegexCache(); + + class EvalState { public: @@ -74,8 +78,10 @@ public: sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sContentAddressed, sOutputHash, sOutputHashAlgo, sOutputHashMode, - sRecurseForDerivations; + sRecurseForDerivations, + sDescription, sSelf, sEpsilon; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -88,8 +94,15 @@ public: Value vEmptySet; + /* Store used to materialise .drv files. */ const ref<Store> store; + /* Store used to build stuff. */ + const ref<Store> buildStore; + + RootValue vCallFlake = nullptr; + RootValue vImportedDrvToDerivation = nullptr; + private: SrcToStore srcToStore; @@ -117,11 +130,14 @@ private: std::unordered_map<Path, Path> resolvedPaths; /* Cache used by prim_match(). */ - std::unordered_map<std::string, std::regex> regexCache; + std::shared_ptr<RegexCache> regexCache; public: - EvalState(const Strings & _searchPath, ref<Store> store); + EvalState( + const Strings & _searchPath, + ref<Store> store, + std::shared_ptr<Store> buildStore = nullptr); ~EvalState(); void addToSearchPath(const string & s); @@ -152,8 +168,17 @@ public: Expr * parseStdin(); /* Evaluate an expression read from the given file to normal - form. */ - void evalFile(const Path & path, Value & v); + form. Optionally enforce that the top-level expression is + trivial (i.e. doesn't require arbitrary computation). */ + void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); + + /* Like `cacheFile`, but with an already parsed expression. */ + void cacheFile( + const Path & path, + const Path & resolvedPath, + Expr * e, + Value & v, + bool mustBeTrivial = false); void resetFileCache(); @@ -209,7 +234,8 @@ public: booleans and lists to a string. If `copyToStore' is set, referenced paths are copied to the Nix store as a side effect. */ string coerceToString(const Pos & pos, Value & v, PathSet & context, - bool coerceMore = false, bool copyToStore = true); + bool coerceMore = false, bool copyToStore = true, + bool canonicalizePath = true); string copyPathToStore(PathSet & context, const Path & path); @@ -238,10 +264,23 @@ private: Value * addPrimOp(const string & name, size_t arity, PrimOpFun primOp); + Value * addPrimOp(PrimOp && primOp); + public: Value & getBuiltin(const string & name); + struct Doc + { + Pos pos; + std::optional<Symbol> name; + size_t arity; + std::vector<std::string> args; + const char * doc; + }; + + std::optional<Doc> getDoc(Value & v); + private: inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval); @@ -280,7 +319,7 @@ public: void mkList(Value & v, size_t length); void mkAttrs(Value & v, size_t capacity); void mkThunk_(Value & v, Expr * expr); - void mkPos(Value & v, Pos * pos); + void mkPos(Value & v, ptr<Pos> pos); void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos); @@ -295,8 +334,10 @@ private: unsigned long nrValuesInEnvs = 0; unsigned long nrValues = 0; unsigned long nrListElems = 0; + unsigned long nrLookups = 0; unsigned long nrAttrsets = 0; unsigned long nrAttrsInAttrsets = 0; + unsigned long nrAvoided = 0; unsigned long nrOpUpdates = 0; unsigned long nrOpUpdateValuesCopied = 0; unsigned long nrListConcats = 0; @@ -318,6 +359,11 @@ private: friend struct ExprOpUpdate; friend struct ExprOpConcatLists; + friend struct ExprVar; + friend struct ExprString; + friend struct ExprInt; + friend struct ExprFloat; + friend struct ExprPath; friend struct ExprSelect; friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); friend void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -330,7 +376,7 @@ string showType(const Value & v); /* Decode a context string ‘!<name>!<path>’ into a pair <path, name>. */ -std::pair<string, string> decodeContext(const string & s); +std::pair<string, string> decodeContext(std::string_view s); /* If `path' refers to a directory, then append "/default.nix". */ Path resolveExprPath(Path path); @@ -353,26 +399,64 @@ struct EvalSettings : Config Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", "Whether builtin functions that allow executing native code should be enabled."}; - Setting<Strings> nixPath{this, getDefaultNixPath(), "nix-path", - "List of directories to be searched for <...> file references."}; + Setting<Strings> nixPath{ + this, getDefaultNixPath(), "nix-path", + "List of directories to be searched for `<...>` file references."}; - Setting<bool> restrictEval{this, false, "restrict-eval", - "Whether to restrict file system access to paths in $NIX_PATH, " - "and network access to the URI prefixes listed in 'allowed-uris'."}; + Setting<bool> restrictEval{ + this, false, "restrict-eval", + R"( + If set to `true`, the Nix evaluator will not allow access to any + files outside of the Nix search path (as set via the `NIX_PATH` + environment variable or the `-I` option), or to URIs outside of + `allowed-uri`. The default is `false`. + )"}; Setting<bool> pureEval{this, false, "pure-eval", "Whether to restrict file system and network access to files specified by cryptographic hash."}; - Setting<bool> enableImportFromDerivation{this, true, "allow-import-from-derivation", - "Whether the evaluator allows importing the result of a derivation."}; + Setting<bool> enableImportFromDerivation{ + this, true, "allow-import-from-derivation", + R"( + By default, Nix allows you to `import` from a derivation, allowing + building at evaluation time. With this option set to false, Nix will + throw an error when evaluating an expression that uses this feature, + allowing users to ensure their evaluation will not require any + builds to take place. + )"}; Setting<Strings> allowedUris{this, {}, "allowed-uris", - "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + R"( + A list of URI prefixes to which access is allowed in restricted + evaluation mode. For example, when set to + `https://github.com/NixOS`, builtin functions such as `fetchGit` are + allowed to access `https://github.com/NixOS/patchelf.git`. + )"}; Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", - "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)."}; + R"( + If set to `true`, the Nix evaluator will trace every function call. + Nix will print a log message at the "vomit" level for every function + entrance and function exit. + + function-trace entered undefined position at 1565795816999559622 + function-trace exited undefined position at 1565795816999581277 + function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 + function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + The `undefined position` means the function call is a builtin. + + Use the `contrib/stack-collapse.py` script distributed with the Nix + source code to convert the trace logs in to a format suitable for + `flamegraph.pl`. + )"}; + + Setting<bool> useEvalCache{this, true, "eval-cache", + "Whether to use the flake evaluation cache."}; }; extern EvalSettings evalSettings; +static const std::string corepkgsPrefix{"/__corepkgs__/"}; + } diff --git a/src/libexpr/fetchurl.nix b/src/libexpr/fetchurl.nix new file mode 100644 index 000000000..02531103b --- /dev/null +++ b/src/libexpr/fetchurl.nix @@ -0,0 +1,41 @@ +{ system ? "" # obsolete +, url +, hash ? "" # an SRI hash + +# Legacy hash specification +, md5 ? "", sha1 ? "", sha256 ? "", sha512 ? "" +, outputHash ? + if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256 +, outputHashAlgo ? + if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256" + +, executable ? false +, unpack ? false +, name ? baseNameOf (toString url) +}: + +derivation { + builder = "builtin:fetchurl"; + + # New-style output content requirements. + inherit outputHashAlgo outputHash; + outputHashMode = if unpack || executable then "recursive" else "flat"; + + inherit name url executable unpack; + + system = "builtin"; + + # No need to double the amount of network traffic + preferLocalBuild = true; + + impureEnvVars = [ + # We borrow these environment variables from the caller to allow + # easy proxy configuration. This is impure, but a fixed-output + # derivation like fetchurl is allowed to do so since its result is + # by definition pure. + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; + + # To make "nix-prefetch-url" work. + urls = [ url ]; +} diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix new file mode 100644 index 000000000..932ac5e90 --- /dev/null +++ b/src/libexpr/flake/call-flake.nix @@ -0,0 +1,56 @@ +lockFileStr: rootSrc: rootSubdir: + +let + + lockFile = builtins.fromJSON lockFileStr; + + allNodes = + builtins.mapAttrs + (key: node: + let + + sourceInfo = + if key == lockFile.root + then rootSrc + else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); + + subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; + + flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); + + inputs = builtins.mapAttrs + (inputName: inputSpec: allNodes.${resolveInput inputSpec}) + (node.inputs or {}); + + # Resolve a input spec into a node name. An input spec is + # either a node name, or a 'follows' path from the root + # node. + resolveInput = inputSpec: + if builtins.isList inputSpec + then getInputByPath lockFile.root inputSpec + else inputSpec; + + # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the + # root node, returning the final node. + getInputByPath = nodeName: path: + if path == [] + then nodeName + else + getInputByPath + # Since this could be a 'follows' input, call resolveInput. + (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) + (builtins.tail path); + + outputs = flake.outputs (inputs // { self = result; }); + + result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; }; + in + if node.flake or true then + assert builtins.isFunction flake.outputs; + result + else + sourceInfo + ) + lockFile.nodes; + +in allNodes.${lockFile.root} diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc new file mode 100644 index 000000000..41b6f78ed --- /dev/null +++ b/src/libexpr/flake/config.cc @@ -0,0 +1,83 @@ +#include "flake.hh" + +#include <nlohmann/json.hpp> + +namespace nix::flake { + +// setting name -> setting value -> allow or ignore. +typedef std::map<std::string, std::map<std::string, bool>> TrustedList; + +Path trustedListPath() +{ + return getDataDir() + "/nix/trusted-settings.json"; +} + +static TrustedList readTrustedList() +{ + auto path = trustedListPath(); + if (!pathExists(path)) return {}; + auto json = nlohmann::json::parse(readFile(path)); + return json; +} + +static void writeTrustedList(const TrustedList & trustedList) +{ + auto path = trustedListPath(); + createDirs(dirOf(path)); + writeFile(path, nlohmann::json(trustedList).dump()); +} + +void ConfigFile::apply() +{ + std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix", "flake-registry"}; + + for (auto & [name, value] : settings) { + + auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name; + + // FIXME: Move into libutil/config.cc. + std::string valueS; + if (auto s = std::get_if<std::string>(&value)) + valueS = *s; + else if (auto n = std::get_if<int64_t>(&value)) + valueS = fmt("%d", n); + else if (auto b = std::get_if<Explicit<bool>>(&value)) + valueS = b->t ? "true" : "false"; + else if (auto ss = std::get_if<std::vector<std::string>>(&value)) + valueS = concatStringsSep(" ", *ss); // FIXME: evil + else + assert(false); + + if (!whitelist.count(baseName)) { + auto trustedList = readTrustedList(); + + bool trusted = false; + + if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) { + trusted = *saved; + } else { + // FIXME: filter ANSI escapes, newlines, \r, etc. + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { + if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { + trustedList[name][valueS] = false; + writeTrustedList(trustedList); + } + } else { + if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { + trustedList[name][valueS] = trusted = true; + writeTrustedList(trustedList); + } + } + } + + if (!trusted) { + warn("ignoring untrusted flake configuration setting '%s'", name); + continue; + } + } + + globalConfig.set(name, valueS); + } +} + +} diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc new file mode 100644 index 000000000..010d97285 --- /dev/null +++ b/src/libexpr/flake/flake.cc @@ -0,0 +1,714 @@ +#include "flake.hh" +#include "eval.hh" +#include "lockfile.hh" +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "finally.hh" + +namespace nix { + +using namespace flake; + +namespace flake { + +typedef std::pair<fetchers::Tree, FlakeRef> FetchedFlake; +typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache; + +static std::optional<FetchedFlake> lookupInFlakeCache( + const FlakeCache & flakeCache, + const FlakeRef & flakeRef) +{ + // FIXME: inefficient. + for (auto & i : flakeCache) { + if (flakeRef == i.first) { + debug("mapping '%s' to previously seen input '%s' -> '%s", + flakeRef, i.first, i.second.second); + return i.second; + } + } + + return std::nullopt; +} + +static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( + EvalState & state, + const FlakeRef & originalRef, + bool allowLookup, + FlakeCache & flakeCache) +{ + auto fetched = lookupInFlakeCache(flakeCache, originalRef); + FlakeRef resolvedRef = originalRef; + + if (!fetched) { + if (originalRef.input.isDirect()) { + fetched.emplace(originalRef.fetchTree(state.store)); + } else { + if (allowLookup) { + resolvedRef = originalRef.resolve(state.store); + auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef); + if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store)); + flakeCache.push_back({resolvedRef, *fetchedResolved}); + fetched.emplace(*fetchedResolved); + } + else { + throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); + } + } + flakeCache.push_back({originalRef, *fetched}); + } + + auto [tree, lockedRef] = *fetched; + + debug("got tree '%s' from '%s'", + state.store->printStorePath(tree.storePath), lockedRef); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); + + return {std::move(tree), resolvedRef, lockedRef}; +} + +static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos) +{ + if (value.isThunk() && value.isTrivial()) + state.forceValue(value, pos); +} + + +static void expectType(EvalState & state, ValueType type, + Value & value, const Pos & pos) +{ + forceTrivialValue(state, value, pos); + if (value.type() != type) + throw Error("expected %s but got %s at %s", + showType(type), showType(value.type()), pos); +} + +static std::map<FlakeId, FlakeInput> parseFlakeInputs( + EvalState & state, Value * value, const Pos & pos); + +static FlakeInput parseFlakeInput(EvalState & state, + const std::string & inputName, Value * value, const Pos & pos) +{ + expectType(state, nAttrs, *value, pos); + + FlakeInput input; + + auto sInputs = state.symbols.create("inputs"); + auto sUrl = state.symbols.create("url"); + auto sFlake = state.symbols.create("flake"); + auto sFollows = state.symbols.create("follows"); + + fetchers::Attrs attrs; + std::optional<std::string> url; + + for (nix::Attr attr : *(value->attrs)) { + try { + if (attr.name == sUrl) { + expectType(state, nString, *attr.value, *attr.pos); + url = attr.value->string.s; + attrs.emplace("url", *url); + } else if (attr.name == sFlake) { + expectType(state, nBool, *attr.value, *attr.pos); + input.isFlake = attr.value->boolean; + } else if (attr.name == sInputs) { + input.overrides = parseFlakeInputs(state, attr.value, *attr.pos); + } else if (attr.name == sFollows) { + expectType(state, nString, *attr.value, *attr.pos); + input.follows = parseInputPath(attr.value->string.s); + } else { + switch (attr.value->type()) { + case nString: + attrs.emplace(attr.name, attr.value->string.s); + break; + case nBool: + attrs.emplace(attr.name, Explicit<bool> { attr.value->boolean }); + break; + case nInt: + attrs.emplace(attr.name, (long unsigned int)attr.value->integer); + break; + default: + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + attr.name, showType(*attr.value)); + } + } + } catch (Error & e) { + e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name)); + throw; + } + } + + if (attrs.count("type")) + try { + input.ref = FlakeRef::fromAttrs(attrs); + } catch (Error & e) { + e.addTrace(pos, hintfmt("in flake input")); + throw; + } + else { + attrs.erase("url"); + if (!attrs.empty()) + throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos); + if (url) + input.ref = parseFlakeRef(*url, {}, true); + } + + if (!input.follows && !input.ref) + input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); + + return input; +} + +static std::map<FlakeId, FlakeInput> parseFlakeInputs( + EvalState & state, Value * value, const Pos & pos) +{ + std::map<FlakeId, FlakeInput> inputs; + + expectType(state, nAttrs, *value, pos); + + for (nix::Attr & inputAttr : *(*value).attrs) { + inputs.emplace(inputAttr.name, + parseFlakeInput(state, + inputAttr.name, + inputAttr.value, + *inputAttr.pos)); + } + + return inputs; +} + +static Flake getFlake( + EvalState & state, + const FlakeRef & originalRef, + bool allowLookup, + FlakeCache & flakeCache) +{ + auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, originalRef, allowLookup, flakeCache); + + // Guard against symlink attacks. + auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix"); + if (!isInDir(flakeFile, sourceInfo.actualPath)) + throw Error("'flake.nix' file of flake '%s' escapes from '%s'", + lockedRef, state.store->printStorePath(sourceInfo.storePath)); + + Flake flake { + .originalRef = originalRef, + .resolvedRef = resolvedRef, + .lockedRef = lockedRef, + .sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo)) + }; + + if (!pathExists(flakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); + + Value vInfo; + state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack + + expectType(state, nAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); + + if (auto description = vInfo.attrs->get(state.sDescription)) { + expectType(state, nString, *description->value, *description->pos); + flake.description = description->value->string.s; + } + + auto sInputs = state.symbols.create("inputs"); + + if (auto inputs = vInfo.attrs->get(sInputs)) + flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos); + + auto sOutputs = state.symbols.create("outputs"); + + if (auto outputs = vInfo.attrs->get(sOutputs)) { + expectType(state, nFunction, *outputs->value, *outputs->pos); + + if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) { + for (auto & formal : outputs->value->lambda.fun->formals->formals) { + if (formal.name != state.sSelf) + flake.inputs.emplace(formal.name, FlakeInput { + .ref = parseFlakeRef(formal.name) + }); + } + } + + } else + throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); + + auto sNixConfig = state.symbols.create("nixConfig"); + + if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { + expectType(state, nAttrs, *nixConfig->value, *nixConfig->pos); + + for (auto & setting : *nixConfig->value->attrs) { + forceTrivialValue(state, *setting.value, *setting.pos); + if (setting.value->type() == nString) + flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)}); + else if (setting.value->type() == nInt) + flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); + else if (setting.value->type() == nBool) + flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); + else if (setting.value->type() == nList) { + std::vector<std::string> ss; + for (unsigned int n = 0; n < setting.value->listSize(); ++n) { + auto elem = setting.value->listElems()[n]; + if (elem->type() != nString) + throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", + setting.name, showType(*setting.value)); + ss.push_back(state.forceStringNoCtx(*elem, *setting.pos)); + } + flake.config.settings.insert({setting.name, ss}); + } + else + throw TypeError("flake configuration setting '%s' is %s", + setting.name, showType(*setting.value)); + } + } + + for (auto & attr : *vInfo.attrs) { + if (attr.name != state.sDescription && + attr.name != sInputs && + attr.name != sOutputs && + attr.name != sNixConfig) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + lockedRef, attr.name, *attr.pos); + } + + return flake; +} + +Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) +{ + FlakeCache flakeCache; + return getFlake(state, originalRef, allowLookup, flakeCache); +} + +/* Compute an in-memory lock file for the specified top-level flake, + and optionally write it to file, if the flake is writable. */ +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & topRef, + const LockFlags & lockFlags) +{ + settings.requireExperimentalFeature("flakes"); + + FlakeCache flakeCache; + + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); + + auto flake = getFlake(state, topRef, useRegistries, flakeCache); + + if (lockFlags.applyNixConfig) { + flake.config.apply(); + // FIXME: send new config to the daemon. + } + + try { + + // FIXME: symlink attack + auto oldLockFile = LockFile::read( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + + debug("old lock file: %s", oldLockFile); + + // FIXME: check whether all overrides are used. + std::map<InputPath, FlakeInput> overrides; + std::set<InputPath> overridesUsed, updatesUsed; + + for (auto & i : lockFlags.inputOverrides) + overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + + LockFile newLockFile; + + std::vector<FlakeRef> parents; + + struct LockParent { + /* The path to this parent. */ + InputPath path; + + /* Whether we are currently inside a top-level lockfile + (inputs absolute) or subordinate lockfile (inputs + relative). */ + bool absolute; + }; + + std::function<void( + const FlakeInputs & flakeInputs, + std::shared_ptr<Node> node, + const InputPath & inputPathPrefix, + std::shared_ptr<const Node> oldNode, + const LockParent & parent, + const Path & parentPath)> + computeLocks; + + computeLocks = [&]( + const FlakeInputs & flakeInputs, + std::shared_ptr<Node> node, + const InputPath & inputPathPrefix, + std::shared_ptr<const Node> oldNode, + const LockParent & parent, + const Path & parentPath) + { + debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); + + /* Get the overrides (i.e. attributes of the form + 'inputs.nixops.inputs.nixpkgs.url = ...'). */ + for (auto & [id, input] : flakeInputs) { + for (auto & [idOverride, inputOverride] : input.overrides) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + inputPath.push_back(idOverride); + overrides.insert_or_assign(inputPath, inputOverride); + } + } + + /* Go over the flake inputs, resolve/fetch them if + necessary (i.e. if they're new or the flakeref changed + from what's in the lock file). */ + for (auto & [id, input2] : flakeInputs) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + auto inputPathS = printInputPath(inputPath); + debug("computing input '%s'", inputPathS); + + try { + + /* Do we have an override for this input from one of the + ancestors? */ + auto i = overrides.find(inputPath); + bool hasOverride = i != overrides.end(); + if (hasOverride) { + overridesUsed.insert(inputPath); + // Respect the “flakeness” of the input even if we + // override it + i->second.isFlake = input2.isFlake; + } + auto & input = hasOverride ? i->second : input2; + + /* Resolve 'follows' later (since it may refer to an input + path we haven't processed yet. */ + if (input.follows) { + InputPath target; + + if (parent.absolute && !hasOverride) { + target = *input.follows; + } else { + if (hasOverride) { + target = inputPathPrefix; + target.pop_back(); + } else + target = parent.path; + + for (auto & i : *input.follows) target.push_back(i); + } + + debug("input '%s' follows '%s'", inputPathS, printInputPath(target)); + node->inputs.insert_or_assign(id, target); + continue; + } + + assert(input.ref); + + /* Do we have an entry in the existing lock file? And we + don't have a --update-input flag for this input? */ + std::shared_ptr<LockedNode> oldLock; + + updatesUsed.insert(inputPath); + + if (oldNode && !lockFlags.inputUpdates.count(inputPath)) + if (auto oldLock2 = get(oldNode->inputs, id)) + if (auto oldLock3 = std::get_if<0>(&*oldLock2)) + oldLock = *oldLock3; + + if (oldLock + && oldLock->originalRef == *input.ref + && !hasOverride) + { + debug("keeping existing input '%s'", inputPathS); + + /* Copy the input from the old lock since its flakeref + didn't change and there is no override from a + higher level flake. */ + auto childNode = std::make_shared<LockedNode>( + oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake); + + node->inputs.insert_or_assign(id, childNode); + + /* If we have an --update-input flag for an input + of this input, then we must fetch the flake to + update it. */ + auto lb = lockFlags.inputUpdates.lower_bound(inputPath); + + auto hasChildUpdate = + lb != lockFlags.inputUpdates.end() + && lb->size() > inputPath.size() + && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); + + if (hasChildUpdate) { + auto inputFlake = getFlake( + state, oldLock->lockedRef, false, flakeCache); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, parent, parentPath); + } else { + /* No need to fetch this flake, we can be + lazy. However there may be new overrides on the + inputs of this flake, so we need to check + those. */ + FlakeInputs fakeInputs; + + for (auto & i : oldLock->inputs) { + if (auto lockedNode = std::get_if<0>(&i.second)) { + fakeInputs.emplace(i.first, FlakeInput { + .ref = (*lockedNode)->originalRef, + .isFlake = (*lockedNode)->isFlake, + }); + } else if (auto follows = std::get_if<1>(&i.second)) { + fakeInputs.emplace(i.first, FlakeInput { + .follows = *follows, + }); + } + } + + computeLocks(fakeInputs, childNode, inputPath, oldLock, parent, parentPath); + } + + } else { + /* We need to create a new lock file entry. So fetch + this input. */ + debug("creating new input '%s'", inputPathS); + + if (!lockFlags.allowMutable && !input.ref->input.isImmutable()) + throw Error("cannot update flake input '%s' in pure mode", inputPathS); + + if (input.isFlake) { + Path localPath = parentPath; + FlakeRef localRef = *input.ref; + + // If this input is a path, recurse it down. + // This allows us to resolve path inputs relative to the current flake. + if (localRef.input.getType() == "path") { + localRef.input.parent = parentPath; + localPath = canonPath(parentPath + "/" + *input.ref->input.getSourcePath()); + } + + auto inputFlake = getFlake(state, localRef, useRegistries, flakeCache); + + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto childNode = std::make_shared<LockedNode>( + inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref); + + node->inputs.insert_or_assign(id, childNode); + + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == *input.ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(*input.ref); + Finally cleanup([&]() { parents.pop_back(); }); + + // Follows paths from existing inputs in the top-level lockfile are absolute, + // whereas paths in subordinate lockfiles are relative to those lockfiles. + LockParent newParent { + .path = inputPath, + .absolute = oldLock ? true : false + }; + + /* Recursively process the inputs of this + flake. Also, unless we already have this flake + in the top-level lock file, use this flake's + own lock file. */ + computeLocks( + inputFlake.inputs, childNode, inputPath, + oldLock + ? std::dynamic_pointer_cast<const Node>(oldLock) + : LockFile::read( + inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root, + newParent, localPath); + } + + else { + auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, *input.ref, useRegistries, flakeCache); + node->inputs.insert_or_assign(id, + std::make_shared<LockedNode>(lockedRef, *input.ref, false)); + } + } + + } catch (Error & e) { + e.addTrace({}, "while updating the flake input '%s'", inputPathS); + throw; + } + } + }; + + LockParent parent { + .path = {}, + .absolute = true + }; + + // Bring in the current ref for relative path resolution if we have it + auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir); + + computeLocks( + flake.inputs, newLockFile.root, {}, + lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath); + + for (auto & i : lockFlags.inputOverrides) + if (!overridesUsed.count(i.first)) + warn("the flag '--override-input %s %s' does not match any input", + printInputPath(i.first), i.second); + + for (auto & i : lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + + /* Check 'follows' inputs. */ + newLockFile.check(); + + debug("new lock file: %s", newLockFile); + + /* Check whether we need to / can write the new lock file. */ + if (!(newLockFile == oldLockFile)) { + + auto diff = LockFile::diff(oldLockFile, newLockFile); + + if (lockFlags.writeLockFile) { + if (auto sourcePath = topRef.input.getSourcePath()) { + if (!newLockFile.isImmutable()) { + if (settings.warnDirty) + warn("will not write lock file of flake '%s' because it has a mutable input", topRef); + } else { + if (!lockFlags.updateLockFile) + throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); + + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + + auto path = *sourcePath + "/" + relPath; + + bool lockFileExists = pathExists(path); + + if (lockFileExists) { + auto s = chomp(diff); + if (s.empty()) + warn("updating lock file '%s'", path); + else + warn("updating lock file '%s':\n%s", path, s); + } else + warn("creating lock file '%s'", path); + + newLockFile.write(path); + + topRef.input.markChangedFile( + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", + lockFlags.commitLockFile + ? std::optional<std::string>(fmt("%s: %s\n\nFlake lock file changes:\n\n%s", + relPath, lockFileExists ? "Update" : "Add", filterANSIEscapes(diff, true))) + : std::nullopt); + + /* Rewriting the lockfile changed the top-level + repo, so we should re-read it. FIXME: we could + also just clear the 'rev' field... */ + auto prevLockedRef = flake.lockedRef; + FlakeCache dummyCache; + flake = getFlake(state, topRef, useRegistries, dummyCache); + + if (lockFlags.commitLockFile && + flake.lockedRef.input.getRev() && + prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) + warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); + + /* Make sure that we picked up the change, + i.e. the tree should usually be dirty + now. Corner case: we could have reverted from a + dirty to a clean tree! */ + if (flake.lockedRef.input == prevLockedRef.input + && !flake.lockedRef.input.isImmutable()) + throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); + } + } else + throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); + } else + warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff)); + } + + return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; + + } catch (Error & e) { + e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string()); + throw; + } +} + +void callFlake(EvalState & state, + const LockedFlake & lockedFlake, + Value & vRes) +{ + auto vLocks = state.allocValue(); + auto vRootSrc = state.allocValue(); + auto vRootSubdir = state.allocValue(); + auto vTmp1 = state.allocValue(); + auto vTmp2 = state.allocValue(); + + mkString(*vLocks, lockedFlake.lockFile.to_string()); + + emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc); + + mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); + + if (!state.vCallFlake) { + state.vCallFlake = allocRootValue(state.allocValue()); + state.eval(state.parseExprFromString( + #include "call-flake.nix.gen.hh" + , "/"), **state.vCallFlake); + } + + state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); + state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); +} + +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto flakeRefS = state.forceStringNoCtx(*args[0], pos); + auto flakeRef = parseFlakeRef(flakeRefS, {}, true); + if (evalSettings.pureEval && !flakeRef.input.isImmutable()) + throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); + + callFlake(state, + lockFlake(state, flakeRef, + LockFlags { + .updateLockFile = false, + .useRegistries = !evalSettings.pureEval && settings.useRegistries, + .allowMutable = !evalSettings.pureEval, + }), + v); +} + +static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes"); + +} + +Fingerprint LockedFlake::getFingerprint() const +{ + // FIXME: as an optimization, if the flake contains a lock file + // and we haven't changed it, then it's sufficient to use + // flake.sourceInfo.storePath for the fingerprint. + return hashString(htSHA256, + fmt("%s;%d;%d;%s", + flake.sourceInfo->storePath.to_string(), + flake.lockedRef.input.getRevCount().value_or(0), + flake.lockedRef.input.getLastModified().value_or(0), + lockFile)); +} + +Flake::~Flake() { } + +} diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh new file mode 100644 index 000000000..d46da9d68 --- /dev/null +++ b/src/libexpr/flake/flake.hh @@ -0,0 +1,145 @@ +#pragma once + +#include "types.hh" +#include "flakeref.hh" +#include "lockfile.hh" +#include "value.hh" + +namespace nix { + +class EvalState; + +namespace fetchers { struct Tree; } + +namespace flake { + +struct FlakeInput; + +typedef std::map<FlakeId, FlakeInput> FlakeInputs; + +/* FlakeInput is the 'Flake'-level parsed form of the "input" entries + * in the flake file. + * + * A FlakeInput is normally constructed by the 'parseFlakeInput' + * function which parses the input specification in the '.flake' file + * to create a 'FlakeRef' (a fetcher, the fetcher-specific + * representation of the input specification, and possibly the fetched + * local store path result) and then creating this FlakeInput to hold + * that FlakeRef, along with anything that might override that + * FlakeRef (like command-line overrides or "follows" specifications). + * + * A FlakeInput is also sometimes constructed directly from a FlakeRef + * instead of starting at the flake-file input specification + * (e.g. overrides, follows, and implicit inputs). + * + * A FlakeInput will usually have one of either "ref" or "follows" + * set. If not otherwise specified, a "ref" will be generated to a + * 'type="indirect"' flake, which is treated as simply the name of a + * flake to be resolved in the registry. + */ + +struct FlakeInput +{ + std::optional<FlakeRef> ref; + bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path + std::optional<InputPath> follows; + FlakeInputs overrides; +}; + +struct ConfigFile +{ + using ConfigValue = std::variant<std::string, int64_t, Explicit<bool>, std::vector<std::string>>; + + std::map<std::string, ConfigValue> settings; + + void apply(); +}; + +/* The contents of a flake.nix file. */ +struct Flake +{ + FlakeRef originalRef; // the original flake specification (by the user) + FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake + FlakeRef lockedRef; // the specific local store result of invoking the fetcher + std::optional<std::string> description; + std::shared_ptr<const fetchers::Tree> sourceInfo; + FlakeInputs inputs; + ConfigFile config; // 'nixConfig' attribute + ~Flake(); +}; + +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); + +/* Fingerprint of a locked flake; used as a cache key. */ +typedef Hash Fingerprint; + +struct LockedFlake +{ + Flake flake; + LockFile lockFile; + + Fingerprint getFingerprint() const; +}; + +struct LockFlags +{ + /* Whether to ignore the existing lock file, creating a new one + from scratch. */ + bool recreateLockFile = false; + + /* Whether to update the lock file at all. If set to false, if any + change to the lock file is needed (e.g. when an input has been + added to flake.nix), you get a fatal error. */ + bool updateLockFile = true; + + /* Whether to write the lock file to disk. If set to true, if the + any changes to the lock file are needed and the flake is not + writable (i.e. is not a local Git working tree or similar), you + get a fatal error. If set to false, Nix will use the modified + lock file in memory only, without writing it to disk. */ + bool writeLockFile = true; + + /* Whether to use the registries to lookup indirect flake + references like 'nixpkgs'. */ + std::optional<bool> useRegistries = std::nullopt; + + /* Whether to apply flake's nixConfig attribute to the configuration */ + + bool applyNixConfig = false; + + /* Whether mutable flake references (i.e. those without a Git + revision or similar) without a corresponding lock are + allowed. Mutable flake references with a lock are always + allowed. */ + bool allowMutable = true; + + /* Whether to commit changes to flake.lock. */ + bool commitLockFile = false; + + /* Flake inputs to be overridden. */ + std::map<InputPath, FlakeRef> inputOverrides; + + /* Flake inputs to be updated. This means that any existing lock + for those inputs will be ignored. */ + std::set<InputPath> inputUpdates; +}; + +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & flakeRef, + const LockFlags & lockFlags); + +void callFlake( + EvalState & state, + const LockedFlake & lockedFlake, + Value & v); + +} + +void emitTreeAttrs( + EvalState & state, + const fetchers::Tree & tree, + const fetchers::Input & input, + Value & v, bool emptyRevFallback = false); + +} diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc new file mode 100644 index 000000000..833e8a776 --- /dev/null +++ b/src/libexpr/flake/flakeref.cc @@ -0,0 +1,206 @@ +#include "flakeref.hh" +#include "store-api.hh" +#include "url.hh" +#include "url-parts.hh" +#include "fetchers.hh" +#include "registry.hh" + +namespace nix { + +#if 0 +// 'dir' path elements cannot start with a '.'. We also reject +// potentially dangerous characters like ';'. +const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; +const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; +#endif + +std::string FlakeRef::to_string() const +{ + std::map<std::string, std::string> extraQuery; + if (subdir != "") + extraQuery.insert_or_assign("dir", subdir); + return input.toURLString(extraQuery); +} + +fetchers::Attrs FlakeRef::toAttrs() const +{ + auto attrs = input.toAttrs(); + if (subdir != "") + attrs.emplace("dir", subdir); + return attrs; +} + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) +{ + str << flakeRef.to_string(); + return str; +} + +bool FlakeRef::operator ==(const FlakeRef & other) const +{ + return input == other.input && subdir == other.subdir; +} + +FlakeRef FlakeRef::resolve(ref<Store> store) const +{ + auto [input2, extraAttrs] = lookupInRegistries(store, input); + return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); +} + +FlakeRef parseFlakeRef( + const std::string & url, const std::optional<Path> & baseDir, bool allowMissing) +{ + auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing); + if (fragment != "") + throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); + return flakeRef; +} + +std::optional<FlakeRef> maybeParseFlakeRef( + const std::string & url, const std::optional<Path> & baseDir) +{ + try { + return parseFlakeRef(url, baseDir); + } catch (Error &) { + return {}; + } +} + +std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir, bool allowMissing) +{ + using namespace fetchers; + + static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+"; + + static std::regex pathUrlRegex( + "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)" + + "(?:\\?(" + queryRegex + "))?" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + static std::regex flakeRegex( + "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + std::smatch match; + + /* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:<flake-id>?ref=<ref>&rev=<rev>'. */ + + if (std::regex_match(url, match, flakeRegex)) { + auto parsedURL = ParsedURL{ + .url = url, + .base = "flake:" + std::string(match[1]), + .scheme = "flake", + .authority = "", + .path = match[1], + }; + + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), ""), + percentDecode(std::string(match[6]))); + } + + else if (std::regex_match(url, match, pathUrlRegex)) { + std::string path = match[1]; + std::string fragment = percentDecode(std::string(match[3])); + + if (baseDir) { + /* Check if 'url' is a path (either absolute or relative + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ + + path = absPath(path, baseDir, true); + + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + + auto flakeRoot = path; + std::string subdir; + + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; + + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = decodeQuery(match[2]), + }; + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); + + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + fragment); + } + + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + } + + } else { + if (!hasPrefix(path, "/")) + throw BadURL("flake reference '%s' is not an absolute path", url); + auto query = decodeQuery(match[2]); + path = canonPath(path + "/" + get(query, "dir").value_or("")); + } + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "path"); + attrs.insert_or_assign("path", path); + + return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); + } + + else { + auto parsedURL = parseURL(url); + std::string fragment; + std::swap(fragment, parsedURL.fragment); + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + fragment); + } +} + +std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir) +{ + try { + return parseFlakeRefWithFragment(url, baseDir); + } catch (Error & e) { + return {}; + } +} + +FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) +{ + auto attrs2(attrs); + attrs2.erase("dir"); + return FlakeRef( + fetchers::Input::fromAttrs(std::move(attrs2)), + fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); +} + +std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const +{ + auto [tree, lockedInput] = input.fetch(store); + return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; +} + +} diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh new file mode 100644 index 000000000..0292eb210 --- /dev/null +++ b/src/libexpr/flake/flakeref.hh @@ -0,0 +1,76 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "fetchers.hh" + +#include <variant> + +namespace nix { + +class Store; + +typedef std::string FlakeId; + +/* A flake reference specifies how to fetch a flake or raw source + * (e.g. from a Git repository). It is created from a URL-like syntax + * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{ + * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local + * path. + * + * Each flake will have a number of FlakeRef objects: one for each + * input to the flake. + * + * The normal method of constructing a FlakeRef is by starting with an + * input description (usually the attrs or a url from the flake file), + * locating a fetcher for that input, and then capturing the Input + * object that fetcher generates (usually via + * FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls). + * + * The actual fetch not have been performed yet (i.e. a FlakeRef may + * be lazy), but the fetcher can be invoked at any time via the + * FlakeRef to ensure the store is populated with this input. + */ + +struct FlakeRef +{ + /* fetcher-specific representation of the input, sufficient to + perform the fetch operation. */ + fetchers::Input input; + + /* sub-path within the fetched input that represents this input */ + Path subdir; + + bool operator==(const FlakeRef & other) const; + + FlakeRef(fetchers::Input && input, const Path & subdir) + : input(std::move(input)), subdir(subdir) + { } + + // FIXME: change to operator <<. + std::string to_string() const; + + fetchers::Attrs toAttrs() const; + + FlakeRef resolve(ref<Store> store) const; + + static FlakeRef fromAttrs(const fetchers::Attrs & attrs); + + std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const; +}; + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); + +FlakeRef parseFlakeRef( + const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false); + +std::optional<FlakeRef> maybeParseFlake( + const std::string & url, const std::optional<Path> & baseDir = {}); + +std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false); + +std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir = {}); + +} diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc new file mode 100644 index 000000000..fda340789 --- /dev/null +++ b/src/libexpr/flake/lockfile.cc @@ -0,0 +1,356 @@ +#include "lockfile.hh" +#include "store-api.hh" +#include "url-parts.hh" + +#include <iomanip> + +#include <nlohmann/json.hpp> + +namespace nix::flake { + +FlakeRef getFlakeRef( + const nlohmann::json & json, + const char * attr, + const char * info) +{ + auto i = json.find(attr); + if (i != json.end()) { + auto attrs = fetchers::jsonToAttrs(*i); + // FIXME: remove when we drop support for version 5. + if (info) { + auto j = json.find(info); + if (j != json.end()) { + for (auto k : fetchers::jsonToAttrs(*j)) + attrs.insert_or_assign(k.first, k.second); + } + } + return FlakeRef::fromAttrs(attrs); + } + + throw Error("attribute '%s' missing in lock file", attr); +} + +LockedNode::LockedNode(const nlohmann::json & json) + : lockedRef(getFlakeRef(json, "locked", "info")) + , originalRef(getFlakeRef(json, "original", nullptr)) + , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) +{ + if (!lockedRef.input.isImmutable()) + throw Error("lockfile contains mutable lock '%s'", + fetchers::attrsToJSON(lockedRef.input.toAttrs())); +} + +StorePath LockedNode::computeStorePath(Store & store) const +{ + return lockedRef.input.computeStorePath(store); +} + +std::shared_ptr<Node> LockFile::findInput(const InputPath & path) +{ + auto pos = root; + + if (!pos) return {}; + + for (auto & elem : path) { + if (auto i = get(pos->inputs, elem)) { + if (auto node = std::get_if<0>(&*i)) + pos = *node; + else if (auto follows = std::get_if<1>(&*i)) { + pos = findInput(*follows); + if (!pos) return {}; + } + } else + return {}; + } + + return pos; +} + +LockFile::LockFile(const nlohmann::json & json, const Path & path) +{ + auto version = json.value("version", 0); + if (version < 5 || version > 7) + throw Error("lock file '%s' has unsupported version %d", path, version); + + std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap; + + std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs; + + getInputs = [&](Node & node, const nlohmann::json & jsonNode) + { + if (jsonNode.find("inputs") == jsonNode.end()) return; + for (auto & i : jsonNode["inputs"].items()) { + if (i.value().is_array()) { // FIXME: remove, obsolete + InputPath path; + for (auto & j : i.value()) + path.push_back(j); + node.inputs.insert_or_assign(i.key(), path); + } else { + std::string inputKey = i.value(); + auto k = nodeMap.find(inputKey); + if (k == nodeMap.end()) { + auto nodes = json["nodes"]; + auto jsonNode2 = nodes.find(inputKey); + if (jsonNode2 == nodes.end()) + throw Error("lock file references missing node '%s'", inputKey); + auto input = std::make_shared<LockedNode>(*jsonNode2); + k = nodeMap.insert_or_assign(inputKey, input).first; + getInputs(*input, *jsonNode2); + } + if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second)) + node.inputs.insert_or_assign(i.key(), child); + else + // FIXME: replace by follows node + throw Error("lock file contains cycle to root node"); + } + } + }; + + std::string rootKey = json["root"]; + nodeMap.insert_or_assign(rootKey, root); + getInputs(*root, json["nodes"][rootKey]); + + // FIXME: check that there are no cycles in version >= 7. Cycles + // between inputs are only possible using 'follows' indirections. + // Once we drop support for version <= 6, we can simplify the code + // a bit since we don't need to worry about cycles. +} + +nlohmann::json LockFile::toJSON() const +{ + nlohmann::json nodes; + std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys; + std::unordered_set<std::string> keys; + + std::function<std::string(const std::string & key, std::shared_ptr<const Node> node)> dumpNode; + + dumpNode = [&](std::string key, std::shared_ptr<const Node> node) -> std::string + { + auto k = nodeKeys.find(node); + if (k != nodeKeys.end()) + return k->second; + + if (!keys.insert(key).second) { + for (int n = 2; ; ++n) { + auto k = fmt("%s_%d", key, n); + if (keys.insert(k).second) { + key = k; + break; + } + } + } + + nodeKeys.insert_or_assign(node, key); + + auto n = nlohmann::json::object(); + + if (!node->inputs.empty()) { + auto inputs = nlohmann::json::object(); + for (auto & i : node->inputs) { + if (auto child = std::get_if<0>(&i.second)) { + inputs[i.first] = dumpNode(i.first, *child); + } else if (auto follows = std::get_if<1>(&i.second)) { + auto arr = nlohmann::json::array(); + for (auto & x : *follows) + arr.push_back(x); + inputs[i.first] = std::move(arr); + } + } + n["inputs"] = std::move(inputs); + } + + if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) { + n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs()); + if (!lockedNode->isFlake) n["flake"] = false; + } + + nodes[key] = std::move(n); + + return key; + }; + + nlohmann::json json; + json["version"] = 7; + json["root"] = dumpNode("root", root); + json["nodes"] = std::move(nodes); + + return json; +} + +std::string LockFile::to_string() const +{ + return toJSON().dump(2); +} + +LockFile LockFile::read(const Path & path) +{ + if (!pathExists(path)) return LockFile(); + return LockFile(nlohmann::json::parse(readFile(path)), path); +} + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) +{ + stream << lockFile.toJSON().dump(2); + return stream; +} + +void LockFile::write(const Path & path) const +{ + createDirs(dirOf(path)); + writeFile(path, fmt("%s\n", *this)); +} + +bool LockFile::isImmutable() const +{ + std::unordered_set<std::shared_ptr<const Node>> nodes; + + std::function<void(std::shared_ptr<const Node> node)> visit; + + visit = [&](std::shared_ptr<const Node> node) + { + if (!nodes.insert(node).second) return; + for (auto & i : node->inputs) + if (auto child = std::get_if<0>(&i.second)) + visit(*child); + }; + + visit(root); + + for (auto & i : nodes) { + if (i == root) continue; + auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i); + if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false; + } + + return true; +} + +bool LockFile::operator ==(const LockFile & other) const +{ + // FIXME: slow + return toJSON() == other.toJSON(); +} + +InputPath parseInputPath(std::string_view s) +{ + InputPath path; + + for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) { + if (!std::regex_match(elem, flakeIdRegex)) + throw UsageError("invalid flake input path element '%s'", elem); + path.push_back(elem); + } + + return path; +} + +std::map<InputPath, Node::Edge> LockFile::getAllInputs() const +{ + std::unordered_set<std::shared_ptr<Node>> done; + std::map<InputPath, Node::Edge> res; + + std::function<void(const InputPath & prefix, std::shared_ptr<Node> node)> recurse; + + recurse = [&](const InputPath & prefix, std::shared_ptr<Node> node) + { + if (!done.insert(node).second) return; + + for (auto &[id, input] : node->inputs) { + auto inputPath(prefix); + inputPath.push_back(id); + res.emplace(inputPath, input); + if (auto child = std::get_if<0>(&input)) + recurse(inputPath, *child); + } + }; + + recurse({}, root); + + return res; +} + +static std::string describe(const FlakeRef & flakeRef) +{ + auto s = fmt("'%s'", flakeRef.to_string()); + + if (auto lastModified = flakeRef.input.getLastModified()) + s += fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%Y-%m-%d")); + + return s; +} + +std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge) +{ + if (auto node = std::get_if<0>(&edge)) + stream << describe((*node)->lockedRef); + else if (auto follows = std::get_if<1>(&edge)) + stream << fmt("follows '%s'", printInputPath(*follows)); + return stream; +} + +static bool equals(const Node::Edge & e1, const Node::Edge & e2) +{ + if (auto n1 = std::get_if<0>(&e1)) + if (auto n2 = std::get_if<0>(&e2)) + return (*n1)->lockedRef == (*n2)->lockedRef; + if (auto f1 = std::get_if<1>(&e1)) + if (auto f2 = std::get_if<1>(&e2)) + return *f1 == *f2; + return false; +} + +std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks) +{ + auto oldFlat = oldLocks.getAllInputs(); + auto newFlat = newLocks.getAllInputs(); + + auto i = oldFlat.begin(); + auto j = newFlat.begin(); + std::string res; + + while (i != oldFlat.end() || j != newFlat.end()) { + if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { + res += fmt("• " ANSI_GREEN "Added input '%s':" ANSI_NORMAL "\n %s\n", + printInputPath(j->first), j->second); + ++j; + } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { + res += fmt("• " ANSI_RED "Removed input '%s'" ANSI_NORMAL "\n", printInputPath(i->first)); + ++i; + } else { + if (!equals(i->second, j->second)) { + res += fmt("• " ANSI_BOLD "Updated input '%s':" ANSI_NORMAL "\n %s\n → %s\n", + printInputPath(i->first), + i->second, + j->second); + } + ++i; + ++j; + } + } + + return res; +} + +void LockFile::check() +{ + auto inputs = getAllInputs(); + + for (auto & [inputPath, input] : inputs) { + if (auto follows = std::get_if<1>(&input)) { + if (!follows->empty() && !get(inputs, *follows)) + throw Error("input '%s' follows a non-existent input '%s'", + printInputPath(inputPath), + printInputPath(*follows)); + } + } +} + +void check(); + +std::string printInputPath(const InputPath & path) +{ + return concatStringsSep("/", path); +} + +} diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh new file mode 100644 index 000000000..96f1edc76 --- /dev/null +++ b/src/libexpr/flake/lockfile.hh @@ -0,0 +1,83 @@ +#pragma once + +#include "flakeref.hh" + +#include <nlohmann/json_fwd.hpp> + +namespace nix { +class Store; +class StorePath; +} + +namespace nix::flake { + +typedef std::vector<FlakeId> InputPath; + +struct LockedNode; + +/* A node in the lock file. It has outgoing edges to other nodes (its + inputs). Only the root node has this type; all other nodes have + type LockedNode. */ +struct Node : std::enable_shared_from_this<Node> +{ + typedef std::variant<std::shared_ptr<LockedNode>, InputPath> Edge; + + std::map<FlakeId, Edge> inputs; + + virtual ~Node() { } +}; + +/* A non-root node in the lock file. */ +struct LockedNode : Node +{ + FlakeRef lockedRef, originalRef; + bool isFlake = true; + + LockedNode( + const FlakeRef & lockedRef, + const FlakeRef & originalRef, + bool isFlake = true) + : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) + { } + + LockedNode(const nlohmann::json & json); + + StorePath computeStorePath(Store & store) const; +}; + +struct LockFile +{ + std::shared_ptr<Node> root = std::make_shared<Node>(); + + LockFile() {}; + LockFile(const nlohmann::json & json, const Path & path); + + nlohmann::json toJSON() const; + + std::string to_string() const; + + static LockFile read(const Path & path); + + void write(const Path & path) const; + + bool isImmutable() const; + + bool operator ==(const LockFile & other) const; + + std::shared_ptr<Node> findInput(const InputPath & path); + + std::map<InputPath, Node::Edge> getAllInputs() const; + + static std::string diff(const LockFile & oldLocks, const LockFile & newLocks); + + /* Check that every 'follows' input target exists. */ + void check(); +}; + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile); + +InputPath parseInputPath(std::string_view s); + +std::string printInputPath(const InputPath & path); + +} diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 9055f59a1..f774e6493 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -2,6 +2,7 @@ #include "util.hh" #include "eval-inline.hh" #include "store-api.hh" +#include "path-with-outputs.hh" #include <cstring> #include <regex> @@ -19,7 +20,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs) : state(&state), attrs(nullptr), attrPath("") { - auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs); + auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs); this->drvPath = store->printStorePath(drvPath); @@ -38,8 +39,11 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat auto i = drv.outputs.find(outputName); if (i == drv.outputs.end()) throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName); + auto & [outputName, output] = *i; - outPath = store->printStorePath(i->second.path); + auto optStorePath = output.path(*store, drv.name, outputName); + if (optStorePath) + outPath = store->printStorePath(*optStorePath); } @@ -77,12 +81,15 @@ string DrvInfo::queryDrvPath() const string DrvInfo::queryOutPath() const { - if (outPath == "" && attrs) { + if (!outPath && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); PathSet context; - outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; + if (i != attrs->end()) + outPath = state->coerceToPath(*i->pos, *i->value, context); } - return outPath; + if (!outPath) + throw UnimplementedError("CA derivations are not yet supported"); + return *outPath; } @@ -122,7 +129,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) if (!outTI->isList()) throw errMsg; Outputs result; for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->type != tString) throw errMsg; + if ((*i)->type() != nString) throw errMsg; auto out = outputs.find((*i)->string.s); if (out == outputs.end()) throw errMsg; result.insert(*out); @@ -166,20 +173,20 @@ StringSet DrvInfo::queryMetaNames() bool DrvInfo::checkMeta(Value & v) { state->forceValue(v); - if (v.isList()) { + if (v.type() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) if (!checkMeta(*v.listElems()[n])) return false; return true; } - else if (v.type == tAttrs) { + else if (v.type() == nAttrs) { Bindings::iterator i = v.attrs->find(state->sOutPath); if (i != v.attrs->end()) return false; for (auto & i : *v.attrs) if (!checkMeta(*i.value)) return false; return true; } - else return v.type == tInt || v.type == tBool || v.type == tString || - v.type == tFloat; + else return v.type() == nInt || v.type() == nBool || v.type() == nString || + v.type() == nFloat; } @@ -195,7 +202,7 @@ Value * DrvInfo::queryMeta(const string & name) string DrvInfo::queryMetaString(const string & name) { Value * v = queryMeta(name); - if (!v || v->type != tString) return ""; + if (!v || v->type() != nString) return ""; return v->string.s; } @@ -204,12 +211,12 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tInt) return v->integer; - if (v->type == tString) { + if (v->type() == nInt) return v->integer; + if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - NixInt n; - if (string2Int(v->string.s, n)) return n; + if (auto n = string2Int<NixInt>(v->string.s)) + return *n; } return def; } @@ -218,12 +225,12 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tFloat) return v->fpoint; - if (v->type == tString) { + if (v->type() == nFloat) return v->fpoint; + if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - NixFloat n; - if (string2Float(v->string.s, n)) return n; + if (auto n = string2Float<NixFloat>(v->string.s)) + return *n; } return def; } @@ -233,8 +240,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tBool) return v->boolean; - if (v->type == tString) { + if (v->type() == nBool) return v->boolean; + if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ if (strcmp(v->string.s, "true") == 0) return true; @@ -325,7 +332,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* Process the expression. */ if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; - else if (v.type == tAttrs) { + else if (v.type() == nAttrs) { /* !!! undocumented hackery to support combining channels in nix-env.cc. */ @@ -347,7 +354,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* If the value of this attribute is itself a set, should we recurse into it? => Only if it has a `recurseForDerivations = true' attribute. */ - if (i->value->type == tAttrs) { + if (i->value->type() == nAttrs) { Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); @@ -356,7 +363,7 @@ static void getDerivations(EvalState & state, Value & vIn, } } - else if (v.isList()) { + else if (v.type() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) { string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index d7860fc6a..29bb6a660 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -20,7 +20,7 @@ private: mutable string name; mutable string system; mutable string drvPath; - mutable string outPath; + mutable std::optional<string> outPath; mutable string outputName; Outputs outputs; diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 76e1a26bf..9ca5ac86d 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -115,6 +115,14 @@ public: { return handle_value<void(Value&, const char*)>(mkString, val.c_str()); } +#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 + bool binary(binary_t&) + { + // This function ought to be unreachable + assert(false); + return true; + } +#endif bool start_object(std::size_t len) { diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index f6e83926b..8ad6a1957 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -9,9 +9,16 @@ %s DEFAULT %x STRING %x IND_STRING +%x INPATH +%x INPATH_SLASH +%x PATH_START %{ +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wunneeded-internal-declaration" +#endif + #include <boost/lexical_cast.hpp> #include "nixexpr.hh" @@ -34,11 +41,13 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) loc->first_line = loc->last_line; loc->first_column = loc->last_column; - while (len--) { + for (size_t i = 0; i < len; i++) { switch (*s++) { case '\r': - if (*s == '\n') /* cr/lf */ + if (*s == '\n') { /* cr/lf */ + i++; s++; + } /* fall through */ case '\n': ++loc->last_line; @@ -91,9 +100,12 @@ ANY .|\n ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* INT [0-9]+ FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)? -PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/? -HPATH \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/? -SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\> +PATH_CHAR [a-zA-Z0-9\.\_\-\+] +PATH {PATH_CHAR}*(\/{PATH_CHAR}+)+\/? +PATH_SEG {PATH_CHAR}*\/ +HPATH \~(\/{PATH_CHAR}+)+\/? +HPATH_START \~\/ +SPATH \<{PATH_CHAR}+(\/{PATH_CHAR}+)*\> URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ @@ -194,17 +206,73 @@ or { return OR_KW; } return IND_STR; } +{PATH_SEG}\$\{ | +{HPATH_START}\$\{ { + PUSH_STATE(PATH_START); + yyless(0); +} + +<PATH_START>{PATH_SEG} { + POP_STATE(); + PUSH_STATE(INPATH_SLASH); + yylval->path = strdup(yytext); + return PATH; +} + +<PATH_START>{HPATH_START} { + POP_STATE(); + PUSH_STATE(INPATH_SLASH); + yylval->path = strdup(yytext); + return HPATH; +} + +{PATH} { + if (yytext[yyleng-1] == '/') + PUSH_STATE(INPATH_SLASH); + else + PUSH_STATE(INPATH); + yylval->path = strdup(yytext); + return PATH; +} +{HPATH} { + if (yytext[yyleng-1] == '/') + PUSH_STATE(INPATH_SLASH); + else + PUSH_STATE(INPATH); + yylval->path = strdup(yytext); + return HPATH; +} + +<INPATH,INPATH_SLASH>\$\{ { + POP_STATE(); + PUSH_STATE(INPATH); + PUSH_STATE(DEFAULT); + return DOLLAR_CURLY; +} +<INPATH,INPATH_SLASH>{PATH}|{PATH_SEG}|{PATH_CHAR}+ { + POP_STATE(); + if (yytext[yyleng-1] == '/') + PUSH_STATE(INPATH_SLASH); + else + PUSH_STATE(INPATH); + yylval->e = new ExprString(data->symbols.create(string(yytext))); + return STR; +} +<INPATH>{ANY} | +<INPATH><<EOF>> { + /* if we encounter a non-path character we inform the parser that the path has + ended with a PATH_END token and re-parse this character in the default + context (it may be ')', ';', or something of that sort) */ + POP_STATE(); + yyless(0); + return PATH_END; +} + +<INPATH_SLASH>{ANY} | +<INPATH_SLASH><<EOF>> { + throw ParseError("path has a trailing slash"); +} -{PATH} { if (yytext[yyleng-1] == '/') - throw ParseError("path '%s' has a trailing slash", yytext); - yylval->path = strdup(yytext); - return PATH; - } -{HPATH} { if (yytext[yyleng-1] == '/') - throw ParseError("path '%s' has a trailing slash", yytext); - yylval->path = strdup(yytext); - return HPATH; - } {SPATH} { yylval->path = strdup(yytext); return SPATH; } {URI} { yylval->uri = strdup(yytext); return URI; } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 9ed39e745..016631647 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,14 +4,19 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/primops/*.cc) \ + $(wildcard $(d)/flake/*.cc) \ + $(d)/lexer-tab.cc \ + $(d)/parser-tab.cc libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS = -ifneq ($(OS), FreeBSD) +libexpr_LDFLAGS += -lboost_context -pthread +ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif @@ -30,8 +35,11 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh -dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh +$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) -$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) +$(foreach i, $(wildcard src/libexpr/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh + +$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index d5698011f..492b819e7 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -284,7 +284,7 @@ void ExprVar::bindVars(const StaticEnv & env) "undefined variable" error now. */ if (withLevel == -1) throw UndefinedVarError({ - .hint = hintfmt("undefined variable '%1%'", name), + .msg = hintfmt("undefined variable '%1%'", name), .errPos = pos }); fromWith = true; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e4cbc660f..51a14cd59 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -17,6 +17,7 @@ MakeError(ThrownError, AssertionError); MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); +MakeError(MissingArgumentError, EvalError); MakeError(RestrictedPathError, Error); @@ -129,7 +130,7 @@ struct ExprPath : Expr { string s; Value v; - ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; + ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; @@ -179,6 +180,7 @@ struct ExprOpHasAttr : Expr struct ExprAttrs : Expr { bool recursive; + Pos pos; struct AttrDef { bool inherited; Expr * e; @@ -198,7 +200,8 @@ struct ExprAttrs : Expr }; typedef std::vector<DynamicAttrDef> DynamicAttrDefs; DynamicAttrDefs dynamicAttrs; - ExprAttrs() : recursive(false) { }; + ExprAttrs(const Pos &pos) : recursive(false), pos(pos) { }; + ExprAttrs() : recursive(false), pos(noPos) { }; COMMON_METHODS }; @@ -238,7 +241,7 @@ struct ExprLambda : Expr { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) throw ParseError({ - .hint = hintfmt("duplicate formal function argument '%1%'", arg), + .msg = hintfmt("duplicate formal function argument '%1%'", arg), .errPos = pos }); }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 878f06c96..e3749783a 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -32,7 +32,7 @@ namespace nix { Path basePath; Symbol file; FileOrigin origin; - ErrorInfo error; + std::optional<ErrorInfo> error; Symbol sLetBody; ParseData(EvalState & state) : state(state) @@ -66,8 +66,8 @@ namespace nix { static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) { throw ParseError({ - .hint = hintfmt("attribute '%1%' already defined at %2%", - showAttrPath(attrPath), prevPos), + .msg = hintfmt("attribute '%1%' already defined at %2%", + showAttrPath(attrPath), prevPos), .errPos = pos }); } @@ -75,7 +75,7 @@ static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prev static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) { throw ParseError({ - .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), + .msg = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos), .errPos = pos }); } @@ -146,7 +146,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) { if (!formals->argNames.insert(formal.name).second) throw ParseError({ - .hint = hintfmt("duplicate formal function argument '%1%'", + .msg = hintfmt("duplicate formal function argument '%1%'", formal.name), .errPos = pos }); @@ -258,7 +258,7 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) { data->error = { - .hint = hintfmt(error), + .msg = hintfmt(error), .errPos = makeCurPos(*loc, data) }; } @@ -290,13 +290,13 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err %type <formal> formal %type <attrNames> attrs attrpath %type <string_parts> string_parts_interpolated ind_string_parts -%type <e> string_parts string_attr +%type <e> path_start string_parts string_attr %type <id> attr %token <id> ID ATTRPATH %token <e> STR IND_STR %token <n> INT %token <nf> FLOAT -%token <path> PATH HPATH SPATH +%token <path> PATH HPATH SPATH PATH_END %token <uri> URI %token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW %token DOLLAR_CURLY /* == ${ */ @@ -338,7 +338,7 @@ expr_function | LET binds IN expr_function { if (!$2->dynamicAttrs.empty()) throw ParseError({ - .hint = hintfmt("dynamic attributes not allowed in let"), + .msg = hintfmt("dynamic attributes not allowed in let"), .errPos = CUR_POS }); $$ = new ExprLet($2, $4); @@ -405,8 +405,11 @@ expr_simple | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { $$ = stripIndentation(CUR_POS, data->symbols, *$2); } - | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } - | HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); } + | path_start PATH_END { $$ = $1; } + | path_start string_parts_interpolated PATH_END { + $2->insert($2->begin(), $1); + $$ = new ExprConcatStrings(CUR_POS, false, $2); + } | SPATH { string path($1 + 1, strlen($1) - 2); $$ = new ExprApp(CUR_POS, @@ -418,7 +421,7 @@ expr_simple static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals"); if (noURLLiterals) throw ParseError({ - .hint = hintfmt("URL literals are disabled"), + .msg = hintfmt("URL literals are disabled"), .errPos = CUR_POS }); $$ = new ExprString(data->symbols.create($1)); @@ -452,6 +455,20 @@ string_parts_interpolated } ; +path_start + : PATH { + Path path(absPath($1, data->basePath)); + /* add back in the trailing '/' to the first segment */ + if ($1[strlen($1)-1] == '/' && strlen($1) > 1) + path += "/"; + $$ = new ExprPath(path); + } + | HPATH { + Path path(getHome() + string($1 + 1)); + $$ = new ExprPath(path); + } + ; + ind_string_parts : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); } | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } @@ -478,7 +495,7 @@ binds $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)); } } - | { $$ = new ExprAttrs; } + | { $$ = new ExprAttrs(makeCurPos(@0, data)); } ; attrs @@ -491,7 +508,7 @@ attrs delete str; } else throw ParseError({ - .hint = hintfmt("dynamic attributes not allowed in inherit"), + .msg = hintfmt("dynamic attributes not allowed in inherit"), .errPos = makeCurPos(@2, data) }); } @@ -576,7 +593,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin, ParseData data(*this); data.origin = origin; switch (origin) { - case foFile: + case foFile: data.file = data.symbols.create(path); break; case foStdin: @@ -593,7 +610,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin, int res = yyparse(scanner, &data); yylex_destroy(scanner); - if (res) throw ParseError(data.error); + if (res) throw ParseError(data.error.value()); data.result->bindVars(staticEnv); @@ -614,8 +631,7 @@ Path resolveExprPath(Path path) // Basic cycle/depth limit to avoid infinite loops. if (++followCount >= maxFollow) throw Error("too many symbolic links encountered while traversing the path '%s'", path); - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%s'", path); + st = lstat(path); if (!S_ISLNK(st.st_mode)) break; path = absPath(readLink(path), dirOf(path)); } @@ -699,8 +715,12 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } + + if (hasPrefix(path, "nix/")) + return corepkgsPrefix + path.substr(4); + throw ThrownError({ - .hint = hintfmt(evalSettings.pureEval + .msg = hintfmt(evalSettings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", path), @@ -719,11 +739,10 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl if (isUri(elem.second)) { try { res = { true, store->toRealPath(fetchers::downloadTarball( - store, resolveUri(elem.second), "source", false).storePath) }; + store, resolveUri(elem.second), "source", false).first.storePath) }; } catch (FileTransferError & e) { logWarning({ - .name = "Entry download", - .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second) }); res = { false, "" }; } @@ -733,8 +752,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl res = { true, path }; else { logWarning({ - .name = "Entry not found", - .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) + .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second) }); res = { false, "" }; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index dec917b38..1de86ad04 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -21,6 +21,8 @@ #include <regex> #include <dlfcn.h> +#include <cmath> + namespace nix { @@ -30,24 +32,12 @@ namespace nix { *************************************************************/ -/* Decode a context string ‘!<name>!<path>’ into a pair <path, - name>. */ -std::pair<string, string> decodeContext(const string & s) -{ - if (s.at(0) == '!') { - size_t index = s.find("!", 1); - return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1)); - } else - return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), ""); -} - - InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} void EvalState::realiseContext(const PathSet & context) { - std::vector<StorePathWithOutputs> drvs; + std::vector<DerivedPath::Built> drvs; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -55,17 +45,7 @@ void EvalState::realiseContext(const PathSet & context) if (!store->isValidPath(ctx)) throw InvalidPathError(store->printStorePath(ctx)); if (!outputName.empty() && ctx.isDerivation()) { - drvs.push_back(StorePathWithOutputs{ctx, {outputName}}); - - /* Add the output of this derivation to the allowed - paths. */ - if (allowedPaths) { - auto drv = store->derivationFromPath(ctx); - DerivationOutputs::iterator i = drv.outputs.find(outputName); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName); - allowedPaths->insert(store->printStorePath(i->second.path)); - } + drvs.push_back({ctx, {outputName}}); } } @@ -73,38 +53,95 @@ void EvalState::realiseContext(const PathSet & context) if (!evalSettings.enableImportFromDerivation) throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false", - store->printStorePath(drvs.begin()->path)); + store->printStorePath(drvs.begin()->drvPath)); /* For performance, prefetch all substitute info. */ StorePathSet willBuild, willSubstitute, unknown; - unsigned long long downloadSize, narSize; - store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); - - store->buildPaths(drvs); + uint64_t downloadSize, narSize; + std::vector<DerivedPath> buildReqs; + for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); + store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); + + store->buildPaths(buildReqs); + + /* Add the output of this derivations to the allowed + paths. */ + if (allowedPaths) { + for (auto & [drvPath, outputs] : drvs) { + auto outputPaths = store->queryDerivationOutputMap(drvPath); + for (auto & outputName : outputs) { + if (outputPaths.count(outputName) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store->printStorePath(drvPath), outputName); + allowedPaths->insert(store->printStorePath(outputPaths.at(outputName))); + } + } + } } +/* Add and attribute to the given attribute map from the output name to + the output path, or a placeholder. + + Where possible the path is used, but for floating CA derivations we + may not know it. For sake of determinism we always assume we don't + and instead put in a place holder. In either case, however, the + string context will contain the drv path and output name, so + downstream derivations will have the proper dependency, and in + addition, before building, the placeholder will be rewritten to be + the actual path. + + The 'drv' and 'drvPath' outputs must correspond. */ +static void mkOutputString(EvalState & state, Value & v, + const StorePath & drvPath, const BasicDerivation & drv, + std::pair<string, DerivationOutput> o) +{ + auto optOutputPath = o.second.path(*state.store, drv.name, o.first); + mkString( + *state.allocAttr(v, state.symbols.create(o.first)), + optOutputPath + ? state.store->printStorePath(*optOutputPath) + /* Downstream we would substitute this for an actual path once + we build the floating CA derivation */ + /* FIXME: we need to depend on the basic derivation, not + derivation */ + : downstreamPlaceholder(*state.store, drvPath, o.first), + {"!" + o.first + "!" + state.store->printStorePath(drvPath)}); +} /* Load and evaluate an expression from path specified by the argument. */ -static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vScope, Value & v) { PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); + Path path = state.coerceToPath(pos, vPath, context); try { state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); + } catch (Error & e) { + e.addTrace(pos, "while importing '%s'", path); + throw e; } Path realPath = state.checkSourcePath(state.toRealPath(path, context)); // FIXME - if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) { - Derivation drv = readDerivation(*state.store, realPath); + auto isValidDerivationInStore = [&]() -> std::optional<StorePath> { + if (!state.store->isStorePath(path)) + return std::nullopt; + auto storePath = state.store->parseStorePath(path); + if (!(state.store->isValidPath(storePath) && isDerivation(path))) + return std::nullopt; + return storePath; + }; + + if (auto optStorePath = isValidDerivationInStore()) { + auto storePath = *optStorePath; + Derivation drv = state.store->readDerivation(storePath); Value & w = *state.allocValue(); state.mkAttrs(w, 3 + drv.outputs.size()); Value * v2 = state.allocAttr(w, state.sDrvPath); @@ -117,36 +154,43 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args unsigned int outputs_index = 0; for (const auto & o : drv.outputs) { - v2 = state.allocAttr(w, state.symbols.create(o.first)); - mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path}); + mkOutputString(state, w, storePath, drv, o); outputsVal->listElems()[outputs_index] = state.allocValue(); mkString(*(outputsVal->listElems()[outputs_index++]), o.first); } w.attrs->sort(); - static RootValue fun; - if (!fun) { - fun = allocRootValue(state.allocValue()); + if (!state.vImportedDrvToDerivation) { + state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , "/"), **fun); + , "/"), **state.vImportedDrvToDerivation); } - state.forceFunction(**fun, pos); - mkApp(v, **fun, w); + state.forceFunction(**state.vImportedDrvToDerivation, pos); + mkApp(v, **state.vImportedDrvToDerivation, w); state.forceAttrs(v, pos); - } else { - state.forceAttrs(*args[0]); - if (args[0]->attrs->empty()) + } + + else if (path == corepkgsPrefix + "fetchurl.nix") { + state.eval(state.parseExprFromString( + #include "fetchurl.nix.gen.hh" + , "/"), v); + } + + else { + if (!vScope) state.evalFile(realPath, v); else { - Env * env = &state.allocEnv(args[0]->attrs->size()); + state.forceAttrs(*vScope); + + Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; StaticEnv staticEnv(false, &state.staticBaseEnv); unsigned int displ = 0; - for (auto & attr : *args[0]->attrs) { + for (auto & attr : *vScope->attrs) { staticEnv.vars[attr.name] = displ; env->values[displ++] = attr.value; } @@ -159,6 +203,76 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args } } +static RegisterPrimOp primop_scopedImport(RegisterPrimOp::Info { + .name = "scopedImport", + .arity = 2, + .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + { + import(state, pos, *args[1], args[0], v); + } +}); + +static RegisterPrimOp primop_import({ + .name = "import", + .args = {"path"}, + .doc = R"( + Load, parse and return the Nix expression in the file *path*. If + *path* is a directory, the file ` default.nix ` in that directory + is loaded. Evaluation aborts if the file doesn’t exist or contains + an incorrect Nix expression. `import` implements Nix’s module + system: you can put any Nix expression (such as a set or a + function) in a separate file, and use it from Nix expressions in + other files. + + > **Note** + > + > Unlike some languages, `import` is a regular function in Nix. + > Paths using the angle bracket syntax (e.g., `import` *\<foo\>*) + > are [normal path values](language-values.md). + + A Nix expression loaded by `import` must not contain any *free + variables* (identifiers that are not defined in the Nix expression + itself and are not built-in). Therefore, it cannot refer to + variables that are in scope at the call site. For instance, if you + have a calling expression + + ```nix + rec { + x = 123; + y = import ./foo.nix; + } + ``` + + then the following `foo.nix` will give an error: + + ```nix + x + 456 + ``` + + since `x` is not in scope in `foo.nix`. If you want `x` to be + available in `foo.nix`, you should pass it as a function argument: + + ```nix + rec { + x = 123; + y = import ./foo.nix x; + } + ``` + + and + + ```nix + x: x + 456 + ``` + + (The function argument doesn’t have to be called `x` in `foo.nix`; + any name would work.) + )", + .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + { + import(state, pos, *args[0], nullptr, v); + } +}); /* Want reasonable symbol names, so extern C */ /* !!! Should we pass the Pos or the file name too? */ @@ -174,7 +288,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt( + .msg = hintfmt( "cannot import '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos @@ -214,7 +328,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) auto count = args[0]->listSize(); if (count == 0) { throw EvalError({ - .hint = hintfmt("at least one argument to 'exec' required"), + .msg = hintfmt("at least one argument to 'exec' required"), .errPos = pos }); } @@ -228,7 +342,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid", + .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid", program, e.path), .errPos = pos }); @@ -256,110 +370,169 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu { state.forceValue(*args[0], pos); string t; - switch (args[0]->type) { - case tInt: t = "int"; break; - case tBool: t = "bool"; break; - case tString: t = "string"; break; - case tPath: t = "path"; break; - case tNull: t = "null"; break; - case tAttrs: t = "set"; break; - case tList1: case tList2: case tListN: t = "list"; break; - case tLambda: - case tPrimOp: - case tPrimOpApp: - t = "lambda"; - break; - case tExternal: + switch (args[0]->type()) { + case nInt: t = "int"; break; + case nBool: t = "bool"; break; + case nString: t = "string"; break; + case nPath: t = "path"; break; + case nNull: t = "null"; break; + case nAttrs: t = "set"; break; + case nList: t = "list"; break; + case nFunction: t = "lambda"; break; + case nExternal: t = args[0]->external->typeOf(); break; - case tFloat: t = "float"; break; - default: abort(); + case nFloat: t = "float"; break; + case nThunk: abort(); } mkString(v, state.symbols.create(t)); } +static RegisterPrimOp primop_typeOf({ + .name = "__typeOf", + .args = {"e"}, + .doc = R"( + Return a string representing the type of the value *e*, namely + `"int"`, `"bool"`, `"string"`, `"path"`, `"null"`, `"set"`, + `"list"`, `"lambda"` or `"float"`. + )", + .fun = prim_typeOf, +}); /* Determine whether the argument is the null value. */ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tNull); + mkBool(v, args[0]->type() == nNull); } +static RegisterPrimOp primop_isNull({ + .name = "isNull", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to `null`, and `false` otherwise. + + > **Warning** + > + > This function is *deprecated*; just write `e == null` instead. + )", + .fun = prim_isNull, +}); /* Determine whether the argument is a function. */ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - bool res; - switch (args[0]->type) { - case tLambda: - case tPrimOp: - case tPrimOpApp: - res = true; - break; - default: - res = false; - break; - } - mkBool(v, res); + mkBool(v, args[0]->type() == nFunction); } +static RegisterPrimOp primop_isFunction({ + .name = "__isFunction", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a function, and `false` otherwise. + )", + .fun = prim_isFunction, +}); /* Determine whether the argument is an integer. */ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tInt); + mkBool(v, args[0]->type() == nInt); } +static RegisterPrimOp primop_isInt({ + .name = "__isInt", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to an integer, and `false` otherwise. + )", + .fun = prim_isInt, +}); + /* Determine whether the argument is a float. */ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tFloat); + mkBool(v, args[0]->type() == nFloat); } +static RegisterPrimOp primop_isFloat({ + .name = "__isFloat", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a float, and `false` otherwise. + )", + .fun = prim_isFloat, +}); + /* Determine whether the argument is a string. */ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tString); + mkBool(v, args[0]->type() == nString); } +static RegisterPrimOp primop_isString({ + .name = "__isString", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a string, and `false` otherwise. + )", + .fun = prim_isString, +}); /* Determine whether the argument is a Boolean. */ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tBool); + mkBool(v, args[0]->type() == nBool); } +static RegisterPrimOp primop_isBool({ + .name = "__isBool", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a bool, and `false` otherwise. + )", + .fun = prim_isBool, +}); + /* Determine whether the argument is a path. */ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tPath); + mkBool(v, args[0]->type() == nPath); } +static RegisterPrimOp primop_isPath({ + .name = "__isPath", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a path, and `false` otherwise. + )", + .fun = prim_isPath, +}); + struct CompareValues { bool operator () (const Value * v1, const Value * v2) const { - if (v1->type == tFloat && v2->type == tInt) + if (v1->type() == nFloat && v2->type() == nInt) return v1->fpoint < v2->integer; - if (v1->type == tInt && v2->type == tFloat) + if (v1->type() == nInt && v2->type() == nFloat) return v1->integer < v2->fpoint; - if (v1->type != v2->type) + if (v1->type() != v2->type()) throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); - switch (v1->type) { - case tInt: + switch (v1->type()) { + case nInt: return v1->integer < v2->integer; - case tFloat: + case nFloat: return v1->fpoint < v2->fpoint; - case tString: + case nString: return strcmp(v1->string.s, v2->string.s) < 0; - case tPath: + case nPath: return strcmp(v1->path, v2->path) < 0; default: throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); @@ -375,18 +548,56 @@ typedef list<Value *> ValueList; #endif +static Bindings::iterator getAttr( + EvalState & state, + string funcName, + string attrName, + Bindings * attrSet, + const Pos & pos) +{ + Bindings::iterator value = attrSet->find(state.symbols.create(attrName)); + if (value == attrSet->end()) { + hintformat errorMsg = hintfmt( + "attribute '%s' missing for call to '%s'", + attrName, + funcName + ); + + Pos aPos = *attrSet->pos; + if (aPos == noPos) { + throw TypeError({ + .msg = errorMsg, + .errPos = pos, + }); + } else { + auto e = TypeError({ + .msg = errorMsg, + .errPos = aPos, + }); + + // Adding another trace for the function name to make it clear + // which call received wrong arguments. + e.addTrace(pos, hintfmt("while invoking '%s'", funcName)); + throw e; + } + } + + return value; +} + static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceAttrs(*args[0], pos); /* Get the start set. */ - Bindings::iterator startSet = - args[0]->attrs->find(state.symbols.create("startSet")); - if (startSet == args[0]->attrs->end()) - throw EvalError({ - .hint = hintfmt("attribute 'startSet' required"), - .errPos = pos - }); + Bindings::iterator startSet = getAttr( + state, + "genericClosure", + "startSet", + args[0]->attrs, + pos + ); + state.forceList(*startSet->value, pos); ValueList workSet; @@ -394,13 +605,14 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar workSet.push_back(startSet->value->listElems()[n]); /* Get the operator. */ - Bindings::iterator op = - args[0]->attrs->find(state.symbols.create("operator")); - if (op == args[0]->attrs->end()) - throw EvalError({ - .hint = hintfmt("attribute 'operator' required"), - .errPos = pos - }); + Bindings::iterator op = getAttr( + state, + "genericClosure", + "operator", + args[0]->attrs, + pos + ); + state.forceValue(*op->value, pos); /* Construct the closure by applying the operator to element of @@ -420,7 +632,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar e->attrs->find(state.symbols.create("key")); if (key == e->attrs->end()) throw EvalError({ - .hint = hintfmt("attribute 'key' required"), + .msg = hintfmt("attribute 'key' required"), .errPos = pos }); state.forceValue(*key->value, pos); @@ -447,22 +659,43 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar v.listElems()[n++] = i; } +static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info { + .name = "__genericClosure", + .arity = 1, + .fun = prim_genericClosure, +}); -static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - throw Abort("evaluation aborted with the following error message: '%1%'", s); -} - - -static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - PathSet context; - string s = state.coerceToString(pos, *args[0], context); - throw ThrownError(s); -} - +static RegisterPrimOp primop_abort({ + .name = "abort", + .args = {"s"}, + .doc = R"( + Abort Nix expression evaluation and print the error message *s*. + )", + .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + { + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + throw Abort("evaluation aborted with the following error message: '%1%'", s); + } +}); + +static RegisterPrimOp primop_throw({ + .name = "throw", + .args = {"s"}, + .doc = R"( + Throw an error message *s*. This usually aborts Nix expression + evaluation, but in `nix-env -qa` and other commands that try to + evaluate a set of derivations to get information about those + derivations, a derivation that throws an error is silently skipped + (which is not the case for `abort`). + )", + .fun = [](EvalState & state, const Pos & pos, Value * * args, Value & v) + { + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + throw ThrownError(s); + } +}); static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -476,6 +709,49 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a } } +static RegisterPrimOp primop_addErrorContext(RegisterPrimOp::Info { + .name = "__addErrorContext", + .arity = 2, + .fun = prim_addErrorContext, +}); + +static void prim_ceil(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + mkInt(v, ceil(value)); +} + +static RegisterPrimOp primop_ceil({ + .name = "__ceil", + .args = {"double"}, + .doc = R"( + Converts an IEEE-754 double-precision floating-point number (*double*) to + the next higher integer. + + If the datatype is neither an integer nor a "float", an evaluation error will be + thrown. + )", + .fun = prim_ceil, +}); + +static void prim_floor(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto value = state.forceFloat(*args[0], args[0]->determinePos(pos)); + mkInt(v, floor(value)); +} + +static RegisterPrimOp primop_floor({ + .name = "__floor", + .args = {"double"}, + .doc = R"( + Converts an IEEE-754 double-precision floating-point number (*double*) to + the next lower integer. + + If the datatype is neither an integer nor a "float", an evaluation error will be + thrown. + )", + .fun = prim_floor, +}); /* Try evaluating the argument. Success => {success=true; value=something;}, * else => {success=false; value=false;} */ @@ -493,6 +769,26 @@ static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Val v.attrs->sort(); } +static RegisterPrimOp primop_tryEval({ + .name = "__tryEval", + .args = {"e"}, + .doc = R"( + Try to shallowly evaluate *e*. Return a set containing the + attributes `success` (`true` if *e* evaluated successfully, + `false` if an error was thrown) and `value`, equalling *e* if + successful and `false` otherwise. `tryEval` will only prevent + errors created by `throw` or `assert` from being thrown. + Errors `tryEval` will not catch are for example those created + by `abort` and type errors generated by builtins. Also note that + this doesn't evaluate *e* deeply, so `let e = { x = throw ""; }; + in (builtins.tryEval e).success` will be `true`. Using + `builtins.deepSeq` one can get the expected result: + `let e = { x = throw ""; }; in + (builtins.tryEval (builtins.deepSeq e e)).success` will be + `false`. + )", + .fun = prim_tryEval, +}); /* Return an environment variable. Use with care. */ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -501,6 +797,22 @@ static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); } +static RegisterPrimOp primop_getEnv({ + .name = "__getEnv", + .args = {"s"}, + .doc = R"( + `getEnv` returns the value of the environment variable *s*, or an + empty string if the variable doesn’t exist. This function should be + used with care, as it can introduce all sorts of nasty environment + dependencies in your Nix expression. + + `getEnv` is used in Nix Packages to locate the file + `~/.nixpkgs/config.nix`, which contains user-local settings for Nix + Packages. (That is, it does a `getEnv "HOME"` to locate the user’s + home directory.) + )", + .fun = prim_getEnv, +}); /* Evaluate the first argument, then return the second argument. */ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -510,6 +822,15 @@ static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v = *args[1]; } +static RegisterPrimOp primop_seq({ + .name = "__seq", + .args = {"e1", "e2"}, + .doc = R"( + Evaluate *e1*, then evaluate and return *e2*. This ensures that a + computation is strict in the value of *e1*. + )", + .fun = prim_seq, +}); /* Evaluate the first argument deeply (i.e. recursing into lists and attrsets), then return the second argument. */ @@ -520,13 +841,23 @@ static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Val v = *args[1]; } +static RegisterPrimOp primop_deepSeq({ + .name = "__deepSeq", + .args = {"e1", "e2"}, + .doc = R"( + This is like `seq e1 e2`, except that *e1* is evaluated *deeply*: + if it’s a list or set, its elements or attributes are also + evaluated recursively. + )", + .fun = prim_deepSeq, +}); /* Evaluate the first expression and print it on standard error. Then return the second expression. Useful for debugging. */ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - if (args[0]->type == tString) + if (args[0]->type() == nString) printError("trace: %1%", args[0]->string.s); else printError("trace: %1%", *args[0]); @@ -534,6 +865,17 @@ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value v = *args[1]; } +static RegisterPrimOp primop_trace({ + .name = "__trace", + .args = {"e1", "e2"}, + .doc = R"( + Evaluate *e1* and print its abstract syntax representation on + standard error. Then return *e2*. This function is useful for + debugging. + )", + .fun = prim_trace, +}); + /************************************************************* * Derivations @@ -552,12 +894,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * state.forceAttrs(*args[0], pos); /* Figure out the name first (for stack backtraces). */ - Bindings::iterator attr = args[0]->attrs->find(state.sName); - if (attr == args[0]->attrs->end()) - throw EvalError({ - .hint = hintfmt("required attribute 'name' missing"), - .errPos = pos - }); + Bindings::iterator attr = getAttr( + state, + "derivationStrict", + state.sName, + args[0]->attrs, + pos + ); + string drvName; Pos & posDrvName(*attr->pos); try { @@ -582,9 +926,11 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Build the derivation expression by processing the attributes. */ Derivation drv; + drv.name = drvName; PathSet context; + bool contentAddressed = false; std::optional<std::string> outputHash; std::string outputHashAlgo; auto ingestionMethod = FileIngestionMethod::Flat; @@ -602,7 +948,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; else throw EvalError({ - .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), + .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), .errPos = posDrvName }); }; @@ -612,7 +958,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * for (auto & j : ss) { if (outputs.find(j) != outputs.end()) throw EvalError({ - .hint = hintfmt("duplicate derivation output '%1%'", j), + .msg = hintfmt("duplicate derivation output '%1%'", j), .errPos = posDrvName }); /* !!! Check whether j is a valid attribute @@ -622,14 +968,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * the resulting set. */ if (j == "drv") throw EvalError({ - .hint = hintfmt("invalid derivation output name 'drv'" ), + .msg = hintfmt("invalid derivation output name 'drv'" ), .errPos = posDrvName }); outputs.insert(j); } if (outputs.empty()) throw EvalError({ - .hint = hintfmt("derivation cannot have an empty set of outputs"), + .msg = hintfmt("derivation cannot have an empty set of outputs"), .errPos = posDrvName }); }; @@ -638,12 +984,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (ignoreNulls) { state.forceValue(*i->value, pos); - if (i->value->type == tNull) continue; + if (i->value->type() == nNull) continue; + } + + if (i->name == state.sContentAddressed) { + settings.requireExperimentalFeature("ca-derivations"); + contentAddressed = state.forceBool(*i->value, pos); } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ - if (i->name == state.sArgs) { + else if (i->name == state.sArgs) { state.forceList(*i->value, pos); for (unsigned int n = 0; n < i->value->listSize(); ++n) { string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); @@ -682,7 +1033,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } else { - auto s = state.coerceToString(posDrvName, *i->value, context, true); + auto s = state.coerceToString(*i->pos, *i->value, context, true); drv.env.emplace(key, s); if (i->name == state.sBuilder) drv.builder = s; else if (i->name == state.sSystem) drv.platform = s; @@ -696,7 +1047,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } } catch (Error & e) { - e.addTrace(posDrvName, + e.addTrace(posDrvName, "while evaluating the attribute '%1%' of the derivation '%2%'", key, drvName); throw; @@ -745,28 +1096,31 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Do we have all required attributes? */ if (drv.builder == "") throw EvalError({ - .hint = hintfmt("required attribute 'builder' missing"), + .msg = hintfmt("required attribute 'builder' missing"), .errPos = posDrvName }); if (drv.platform == "") throw EvalError({ - .hint = hintfmt("required attribute 'system' missing"), + .msg = hintfmt("required attribute 'system' missing"), .errPos = posDrvName }); /* Check whether the derivation name is valid. */ if (isDerivation(drvName)) throw EvalError({ - .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), + .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), .errPos = posDrvName }); if (outputHash) { - /* Handle fixed-output derivations. */ + /* Handle fixed-output derivations. + + Ignore `__contentAddressed` because fixed output derivations are + already content addressed. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") throw Error({ - .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"), + .msg = hintfmt("multiple outputs are not supported in fixed-output derivations"), .errPos = posDrvName }); @@ -774,16 +1128,30 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * Hash h = newHashAllowEmpty(*outputHash, ht); auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName); - if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath); + drv.env["out"] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign("out", DerivationOutput { - .path = std::move(outPath), - .hash = FixedOutputHash { - .method = ingestionMethod, - .hash = std::move(h), - }, + .output = DerivationOutputCAFixed { + .hash = FixedOutputHash { + .method = ingestionMethod, + .hash = std::move(h), + }, + }, }); } + else if (contentAddressed) { + HashType ht = parseHashType(outputHashAlgo); + for (auto & i : outputs) { + drv.env[i] = hashPlaceholder(i); + drv.outputs.insert_or_assign(i, DerivationOutput { + .output = DerivationOutputCAFloating { + .method = ingestionMethod, + .hashType = std::move(ht), + }, + }); + } + } + else { /* Compute a hash over the "masked" store derivation, which is the final one except that in the list of outputs, the @@ -792,48 +1160,77 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * that changes in the set of output names do get reflected in the hash. */ for (auto & i : outputs) { - if (!jsonObject) drv.env[i] = ""; + drv.env[i] = ""; drv.outputs.insert_or_assign(i, DerivationOutput { - .path = StorePath::dummy, - .hash = std::optional<FixedOutputHash> {}, + .output = DerivationOutputInputAddressed { + .path = StorePath::dummy, + }, }); } - Hash h = hashDerivationModulo(*state.store, Derivation(drv), true); + // Regular, non-CA derivation should always return a single hash and not + // hash per output. + auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true); + std::visit(overloaded { + [&](Hash h) { + for (auto & i : outputs) { + auto outPath = state.store->makeOutputPath(i, h, drvName); + drv.env[i] = state.store->printStorePath(outPath); + drv.outputs.insert_or_assign(i, + DerivationOutput { + .output = DerivationOutputInputAddressed { + .path = std::move(outPath), + }, + }); + } + }, + [&](CaOutputHashes) { + // Shouldn't happen as the toplevel derivation is not CA. + assert(false); + }, + [&](DeferredHash _) { + for (auto & i : outputs) { + drv.outputs.insert_or_assign(i, + DerivationOutput { + .output = DerivationOutputDeferred{}, + }); + } + }, + }, + hashModulo); - for (auto & i : outputs) { - auto outPath = state.store->makeOutputPath(i, h, drvName); - if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign(i, - DerivationOutput { - .path = std::move(outPath), - .hash = std::optional<FixedOutputHash>(), - }); - } } /* Write the resulting term into the Nix store directory. */ - auto drvPath = writeDerivation(state.store, drv, drvName, state.repair); + auto drvPath = writeDerivation(*state.store, drv, state.repair); auto drvPathS = state.store->printStorePath(drvPath); printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS); /* Optimisation, but required in read-only mode! because in that case we don't actually write store derivations, so we can't - read them later. */ - drvHashes.insert_or_assign(drvPath, - hashDerivationModulo(*state.store, Derivation(drv), false)); + read them later. + + However, we don't bother doing this for floating CA derivations because + their "hash modulo" is indeterminate until built. */ + if (drv.type() != DerivationType::CAFloating) { + auto h = hashDerivationModulo(*state.store, Derivation(drv), false); + drvHashes.lock()->insert_or_assign(drvPath, h); + } state.mkAttrs(v, 1 + drv.outputs.size()); mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS}); - for (auto & i : drv.outputs) { - mkString(*state.allocAttr(v, state.symbols.create(i.first)), - state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS}); - } + for (auto & i : drv.outputs) + mkOutputString(state, v, drvPath, drv, i); v.attrs->sort(); } +static RegisterPrimOp primop_derivationStrict(RegisterPrimOp::Info { + .name = "derivationStrict", + .arity = 1, + .fun = prim_derivationStrict, +}); /* Return a placeholder string for the specified output that will be substituted by the corresponding output path at build time. For @@ -847,6 +1244,17 @@ static void prim_placeholder(EvalState & state, const Pos & pos, Value * * args, mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); } +static RegisterPrimOp primop_placeholder({ + .name = "placeholder", + .args = {"output"}, + .doc = R"( + Return a placeholder string for the specified *output* that will be + substituted by the corresponding output path at build time. Typical + outputs would be `"out"`, `"bin"` or `"dev"`. + )", + .fun = prim_placeholder, +}); + /************************************************************* * Paths @@ -861,6 +1269,15 @@ static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, canonPath(path), context); } +static RegisterPrimOp primop_toPath({ + .name = "__toPath", + .args = {"s"}, + .doc = R"( + **DEPRECATED.** Use `/. + "/path"` to convert a string into an absolute + path. For relative paths, use `./. + "/path"`. + )", + .fun = prim_toPath, +}); /* Allow a valid store path to be used in an expression. This is useful in some generated expressions such as in nix-push, which @@ -872,6 +1289,12 @@ static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Valu corner cases. */ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, Value & v) { + if (evalSettings.pureEval) + throw EvalError({ + .msg = hintfmt("'%s' is not allowed in pure evaluation mode", "builtins.storePath"), + .errPos = pos + }); + PathSet context; Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink @@ -880,16 +1303,33 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V if (!state.store->isStorePath(path)) path = canonPath(path, true); if (!state.store->isInStore(path)) throw EvalError({ - .hint = hintfmt("path '%1%' is not in the Nix store", path), + .msg = hintfmt("path '%1%' is not in the Nix store", path), .errPos = pos }); - Path path2 = state.store->toStorePath(path); + auto path2 = state.store->toStorePath(path).first; if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(path2)); - context.insert(path2); + state.store->ensurePath(path2); + context.insert(state.store->printStorePath(path2)); mkString(v, path, context); } +static RegisterPrimOp primop_storePath({ + .name = "__storePath", + .args = {"path"}, + .doc = R"( + This function allows you to define a dependency on an already + existing store path. For example, the derivation attribute `src + = builtins.storePath /nix/store/f1d18v1y…-source` causes the + derivation to depend on the specified path, which must exist or + be substitutable. Note that this differs from a plain path + (e.g. `src = /nix/store/f1d18v1y…-source`) in that the latter + causes the path to be *copied* again to the Nix store, resulting + in a new path (e.g. `/nix/store/ld01dnzc…-source-source`). + + This function is not available in pure evaluation mode. + )", + .fun = prim_storePath, +}); static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -899,7 +1339,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt( + .msg = hintfmt( "cannot check the existence of '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos @@ -917,6 +1357,15 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, } } +static RegisterPrimOp primop_pathExists({ + .name = "__pathExists", + .args = {"path"}, + .doc = R"( + Return `true` if the path *path* exists at evaluation time, and + `false` otherwise. + )", + .fun = prim_pathExists, +}); /* Return the base name of the given string, i.e., everything following the last slash. */ @@ -926,6 +1375,16 @@ static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, mkString(v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), context); } +static RegisterPrimOp primop_baseNameOf({ + .name = "baseNameOf", + .args = {"s"}, + .doc = R"( + Return the *base name* of the string *s*, that is, everything + following the final slash in the string. This is similar to the GNU + `basename` command. + )", + .fun = prim_baseNameOf, +}); /* Return the directory of the given path, i.e., everything before the last slash. Return either a path or a string depending on the type @@ -934,20 +1393,35 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value { PathSet context; Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); - if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); + if (args[0]->type() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } +static RegisterPrimOp primop_dirOf({ + .name = "dirOf", + .args = {"s"}, + .doc = R"( + Return the directory part of the string *s*, that is, everything + before the final slash in the string. This is similar to the GNU + `dirname` command. + )", + .fun = prim_dirOf, +}); /* Return the contents of a file as a string. */ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; Path path = state.coerceToPath(pos, *args[0], context); + if (baseNameOf(path) == "flake.lock") + throw Error({ + .msg = hintfmt("cannot read '%s' because flake lock files can be out of sync", path), + .errPos = pos + }); try { state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -957,6 +1431,14 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va mkString(v, s.c_str()); } +static RegisterPrimOp primop_readFile({ + .name = "__readFile", + .args = {"path"}, + .doc = R"( + Return the contents of the file *path* as a string. + )", + .fun = prim_readFile, +}); /* Find a file in the Nix search path. Used to implement <x> paths, which are desugared to 'findFile __nixPath "x"'. */ @@ -975,12 +1457,13 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va if (i != v2.attrs->end()) prefix = state.forceStringNoCtx(*i->value, pos); - i = v2.attrs->find(state.symbols.create("path")); - if (i == v2.attrs->end()) - throw EvalError({ - .hint = hintfmt("attribute 'path' missing"), - .errPos = pos - }); + i = getAttr( + state, + "findFile", + "path", + v2.attrs, + pos + ); PathSet context; string path = state.coerceToString(pos, *i->value, context, false, false); @@ -989,7 +1472,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va state.realiseContext(context); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -1002,23 +1485,45 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); } +static RegisterPrimOp primop_findFile(RegisterPrimOp::Info { + .name = "__findFile", + .arity = 2, + .fun = prim_findFile, +}); + /* Return the cryptographic hash of a file in base-16. */ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) { string type = state.forceStringNoCtx(*args[0], pos); std::optional<HashType> ht = parseHashType(type); if (!ht) - throw Error({ - .hint = hintfmt("unknown hash type '%1%'", type), - .errPos = pos - }); + throw Error({ + .msg = hintfmt("unknown hash type '%1%'", type), + .errPos = pos + }); - PathSet context; // discarded - Path p = state.coerceToPath(pos, *args[1], context); + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError("cannot read '%s' since path '%s' is not valid, at %s", path, e.path, pos); + } - mkString(v, hashFile(*ht, state.checkSourcePath(p)).to_string(Base16, false), context); + mkString(v, hashFile(*ht, state.checkSourcePath(state.toRealPath(path, context))).to_string(Base16, false)); } +static RegisterPrimOp primop_hashFile({ + .name = "__hashFile", + .args = {"type", "p"}, + .doc = R"( + Return a base-16 representation of the cryptographic hash of the + file at path *p*. The hash algorithm specified by *type* must be one + of `"md5"`, `"sha1"`, `"sha256"` or `"sha512"`. + )", + .fun = prim_hashFile, +}); + /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1028,7 +1533,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val state.realiseContext(ctx); } catch (InvalidPathError & e) { throw EvalError({ - .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), + .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } @@ -1040,7 +1545,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name)); if (ent.type == DT_UNKNOWN) ent.type = getFileType(path + "/" + ent.name); - mkStringNoCopy(*ent_val, + ent_val->mkString( ent.type == DT_REG ? "regular" : ent.type == DT_DIR ? "directory" : ent.type == DT_LNK ? "symlink" : @@ -1050,6 +1555,25 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val v.attrs->sort(); } +static RegisterPrimOp primop_readDir({ + .name = "__readDir", + .args = {"path"}, + .doc = R"( + Return the contents of the directory *path* as a set mapping + directory entries to the corresponding file type. For instance, if + directory `A` contains a regular file `B` and another directory + `C`, then `builtins.readDir ./A` will return the set + + ```nix + { B = "regular"; C = "directory"; } + ``` + + The possible values for the file type are `"regular"`, + `"directory"`, `"symlink"` and `"unknown"`. + )", + .fun = prim_readDir, +}); + /************************************************************* * Creating files @@ -1067,6 +1591,102 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value mkString(v, out.str(), context); } +static RegisterPrimOp primop_toXML({ + .name = "__toXML", + .args = {"e"}, + .doc = R"( + Return a string containing an XML representation of *e*. The main + application for `toXML` is to communicate information with the + builder in a more structured format than plain environment + variables. + + Here is an example where this is the case: + + ```nix + { stdenv, fetchurl, libxslt, jira, uberwiki }: + + stdenv.mkDerivation (rec { + name = "web-server"; + + buildInputs = [ libxslt ]; + + builder = builtins.toFile "builder.sh" " + source $stdenv/setup + mkdir $out + echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml ① + "; + + stylesheet = builtins.toFile "stylesheet.xsl" ② + "<?xml version='1.0' encoding='UTF-8'?> + <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'> + <xsl:template match='/'> + <Configure> + <xsl:for-each select='/expr/list/attrs'> + <Call name='addWebApplication'> + <Arg><xsl:value-of select=\"attr[@name = 'path']/string/@value\" /></Arg> + <Arg><xsl:value-of select=\"attr[@name = 'war']/path/@value\" /></Arg> + </Call> + </xsl:for-each> + </Configure> + </xsl:template> + </xsl:stylesheet> + "; + + servlets = builtins.toXML [ ③ + { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; } + { path = "/wiki"; war = uberwiki + "/uberwiki.war"; } + ]; + }) + ``` + + The builder is supposed to generate the configuration file for a + [Jetty servlet container](http://jetty.mortbay.org/). A servlet + container contains a number of servlets (`*.war` files) each + exported under a specific URI prefix. So the servlet configuration + is a list of sets containing the `path` and `war` of the servlet + (①). This kind of information is difficult to communicate with the + normal method of passing information through an environment + variable, which just concatenates everything together into a + string (which might just work in this case, but wouldn’t work if + fields are optional or contain lists themselves). Instead the Nix + expression is converted to an XML representation with `toXML`, + which is unambiguous and can easily be processed with the + appropriate tools. For instance, in the example an XSLT stylesheet + (at point ②) is applied to it (at point ①) to generate the XML + configuration file for the Jetty server. The XML representation + produced at point ③ by `toXML` is as follows: + + ```xml + <?xml version='1.0' encoding='utf-8'?> + <expr> + <list> + <attrs> + <attr name="path"> + <string value="/bugtracker" /> + </attr> + <attr name="war"> + <path value="/nix/store/d1jh9pasa7k2...-jira/lib/atlassian-jira.war" /> + </attr> + </attrs> + <attrs> + <attr name="path"> + <string value="/wiki" /> + </attr> + <attr name="war"> + <path value="/nix/store/y6423b1yi4sx...-uberwiki/uberwiki.war" /> + </attr> + </attrs> + </list> + </expr> + ``` + + Note that we used the `toFile` built-in to write the builder and + the stylesheet “inline” in the Nix expression. The path of the + stylesheet is spliced into the builder using the syntax `xsltproc + ${stylesheet}`. + )", + .fun = prim_toXML, +}); /* Convert the argument (which can be any Nix expression) to a JSON string. Not all Nix expressions can be sensibly or completely @@ -1079,14 +1699,46 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, out.str(), context); } +static RegisterPrimOp primop_toJSON({ + .name = "__toJSON", + .args = {"e"}, + .doc = R"( + Return a string containing a JSON representation of *e*. Strings, + integers, floats, booleans, nulls and lists are mapped to their JSON + equivalents. Sets (except derivations) are represented as objects. + Derivations are translated to a JSON string containing the + derivation’s output path. Paths are copied to the store and + represented as a JSON string of the resulting store path. + )", + .fun = prim_toJSON, +}); /* Parse a JSON string to a value. */ static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) { string s = state.forceStringNoCtx(*args[0], pos); - parseJSON(state, s, v); + try { + parseJSON(state, s, v); + } catch (JSONParseError &e) { + e.addTrace(pos, "while decoding a JSON string"); + throw e; + } } +static RegisterPrimOp primop_fromJSON({ + .name = "__fromJSON", + .args = {"e"}, + .doc = R"( + Convert a JSON string to a Nix value. For example, + + ```nix + builtins.fromJSON ''{"x": [1, 2, 3], "y": null}'' + ``` + + returns the value `{ x = [ 1 2 3 ]; y = null; }`. + )", + .fun = prim_fromJSON, +}); /* Store a string in the Nix store as a source file that can be used as an input by derivations. */ @@ -1101,7 +1753,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu for (auto path : context) { if (path.at(0) != '/') throw EvalError( { - .hint = hintfmt( + .msg = hintfmt( "in 'toFile': the file named '%1%' must not contain a reference " "to a derivation but contains (%2%)", name, path), @@ -1121,9 +1773,86 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu mkString(v, storePath, {storePath}); } +static RegisterPrimOp primop_toFile({ + .name = "__toFile", + .args = {"name", "s"}, + .doc = R"( + Store the string *s* in a file in the Nix store and return its + path. The file has suffix *name*. This file can be used as an + input to derivations. One application is to write builders + “inline”. For instance, the following Nix expression combines the + [Nix expression for GNU Hello](expression-syntax.md) and its + [build script](build-script.md) into one file: + + ```nix + { stdenv, fetchurl, perl }: + + stdenv.mkDerivation { + name = "hello-2.1.1"; + + builder = builtins.toFile "builder.sh" " + source $stdenv/setup + + PATH=$perl/bin:$PATH + + tar xvfz $src + cd hello-* + ./configure --prefix=$out + make + make install + "; + + src = fetchurl { + url = "http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz"; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; + }; + inherit perl; + } + ``` + + It is even possible for one file to refer to another, e.g., + + ```nix + builder = let + configFile = builtins.toFile "foo.conf" " + # This is some dummy configuration file. + ... + "; + in builtins.toFile "builder.sh" " + source $stdenv/setup + ... + cp ${configFile} $out/etc/foo.conf + "; + ``` + + Note that `${configFile}` is an + [antiquotation](language-values.md), so the result of the + expression `configFile` + (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be + spliced into the resulting string. + + It is however *not* allowed to have files mutually referring to each + other, like so: + + ```nix + let + foo = builtins.toFile "foo" "...${bar}..."; + bar = builtins.toFile "bar" "...${foo}..."; + in foo + ``` + + This is not allowed because it would cause a cyclic dependency in + the computation of the cryptographic hashes for `foo` and `bar`. + + It is also not possible to reference the result of a derivation. If + you are using Nixpkgs, the `writeTextFile` function is able to do + that. + )", + .fun = prim_toFile, +}); static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v) + Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v) { const auto path = evalSettings.pureEval && expectedHash ? path_ : @@ -1154,7 +1883,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con std::optional<StorePath> expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name); + expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name); Path dstPath; if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode @@ -1175,21 +1904,63 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args Path path = state.coerceToPath(pos, *args[1], context); if (!context.empty()) throw EvalError({ - .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .msg = hintfmt("string '%1%' cannot refer to other paths", path), .errPos = pos }); state.forceValue(*args[0], pos); - if (args[0]->type != tLambda) + if (args[0]->type() != nFunction) throw TypeError({ - .hint = hintfmt( + .msg = hintfmt( "first argument in call to 'filterSource' is not a function but %1%", showType(*args[0])), .errPos = pos }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v); -} + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); +} + +static RegisterPrimOp primop_filterSource({ + .name = "__filterSource", + .args = {"e1", "e2"}, + .doc = R"( + This function allows you to copy sources into the Nix store while + filtering certain files. For instance, suppose that you want to use + the directory `source-dir` as an input to a Nix expression, e.g. + + ```nix + stdenv.mkDerivation { + ... + src = ./source-dir; + } + ``` + + However, if `source-dir` is a Subversion working copy, then all + those annoying `.svn` subdirectories will also be copied to the + store. Worse, the contents of those directories may change a lot, + causing lots of spurious rebuilds. With `filterSource` you can + filter out the `.svn` directories: + + ```nix + src = builtins.filterSource + (path: type: type != "directory" || baseNameOf path != ".svn") + ./source-dir; + ``` + + Thus, the first argument *e1* must be a predicate function that is + called for each regular file, directory or symlink in the source + tree *e2*. If the function returns `true`, the file is copied to the + Nix store, otherwise it is omitted. The function is called with two + arguments. The first is the full path of the file. The second is a + string that identifies the type of the file, which is either + `"regular"`, `"directory"`, `"symlink"` or `"unknown"` (for other + kinds of files such as device nodes or fifos — but note that those + cannot be copied to the Nix store, so if the predicate returns + `true` for them, the copy will fail). If you exclude a directory, + the entire corresponding subtree of *e2* will be excluded. + )", + .fun = prim_filterSource, +}); static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1198,7 +1969,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; - Hash expectedHash; + std::optional<Hash> expectedHash; for (auto & attr : *args[0]->attrs) { const string & n(attr.name); @@ -1207,7 +1978,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value path = state.coerceToPath(*attr.pos, *attr.value, context); if (!context.empty()) throw EvalError({ - .hint = hintfmt("string '%1%' cannot refer to other paths", path), + .msg = hintfmt("string '%1%' cannot refer to other paths", path), .errPos = *attr.pos }); } else if (attr.name == state.sName) @@ -1221,13 +1992,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); else throw EvalError({ - .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), + .msg = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name), .errPos = *attr.pos }); } if (path.empty()) throw EvalError({ - .hint = hintfmt("'path' required"), + .msg = hintfmt("'path' required"), .errPos = pos }); if (name.empty()) @@ -1236,6 +2007,41 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value addPath(state, pos, name, path, filterFun, method, expectedHash, v); } +static RegisterPrimOp primop_path({ + .name = "__path", + .args = {"args"}, + .doc = R"( + An enrichment of the built-in path type, based on the attributes + present in *args*. All are optional except `path`: + + - path\ + The underlying path. + + - name\ + The name of the path when added to the store. This can used to + reference paths that have nix-illegal characters in their names, + like `@`. + + - filter\ + A function of the type expected by `builtins.filterSource`, + with the same semantics. + + - recursive\ + When `false`, when `path` is added to the store it is with a + flat hash, rather than a hash of the NAR serialization of the + file. Thus, `path` must refer to a regular file, not a + directory. This allows similar behavior to `fetchurl`. Defaults + to `true`. + + - sha256\ + When provided, this is the expected hash of the file at the + path. Evaluation will fail if the hash is incorrect, and + providing a hash allows `builtins.path` to be used even when the + `pure-eval` nix config option is on. + )", + .fun = prim_path, +}); + /************************************************************* * Sets @@ -1258,6 +2064,16 @@ static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, V [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); } +static RegisterPrimOp primop_attrNames({ + .name = "__attrNames", + .args = {"set"}, + .doc = R"( + Return the names of the attributes in the set *set* in an + alphabetically sorted list. For instance, `builtins.attrNames { y + = 1; x = "foo"; }` evaluates to `[ "x" "y" ]`. + )", + .fun = prim_attrNames, +}); /* Return the values of the attributes in a set as a list, in the same order as attrNames. */ @@ -1278,6 +2094,15 @@ static void prim_attrValues(EvalState & state, const Pos & pos, Value * * args, v.listElems()[i] = ((Attr *) v.listElems()[i])->value; } +static RegisterPrimOp primop_attrValues({ + .name = "__attrValues", + .args = {"set"}, + .doc = R"( + Return the values of the attributes in the set *set* in the order + corresponding to the sorted attribute names. + )", + .fun = prim_attrValues, +}); /* Dynamic version of the `.' operator. */ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1285,21 +2110,33 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) string attr = state.forceStringNoCtx(*args[0], pos); state.forceAttrs(*args[1], pos); // !!! Should we create a symbol here or just do a lookup? - Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); - if (i == args[1]->attrs->end()) - throw EvalError({ - .hint = hintfmt("attribute '%1%' missing", attr), - .errPos = pos - }); + Bindings::iterator i = getAttr( + state, + "getAttr", + attr, + args[1]->attrs, + pos + ); // !!! add to stack trace? - if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; + if (state.countCalls && *i->pos != noPos) state.attrSelects[*i->pos]++; state.forceValue(*i->value, pos); v = *i->value; } +static RegisterPrimOp primop_getAttr({ + .name = "__getAttr", + .args = {"s", "set"}, + .doc = R"( + `getAttr` returns the attribute named *s* from *set*. Evaluation + aborts if the attribute doesn’t exist. This is a dynamic version of + the `.` operator, since *s* is an expression rather than an + identifier. + )", + .fun = prim_getAttr, +}); /* Return position information of the specified attribute. */ -void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, Value & v) { string attr = state.forceStringNoCtx(*args[0], pos); state.forceAttrs(*args[1], pos); @@ -1310,6 +2147,11 @@ void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, V state.mkPos(v, i->pos); } +static RegisterPrimOp primop_unsafeGetAttrPos(RegisterPrimOp::Info { + .name = "__unsafeGetAttrPos", + .arity = 2, + .fun = prim_unsafeGetAttrPos, +}); /* Dynamic version of the `?' operator. */ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1319,14 +2161,32 @@ static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Val mkBool(v, args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); } +static RegisterPrimOp primop_hasAttr({ + .name = "__hasAttr", + .args = {"s", "set"}, + .doc = R"( + `hasAttr` returns `true` if *set* has an attribute named *s*, and + `false` otherwise. This is a dynamic version of the `?` operator, + since *s* is an expression rather than an identifier. + )", + .fun = prim_hasAttr, +}); /* Determine whether the argument is a set. */ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tAttrs); + mkBool(v, args[0]->type() == nAttrs); } +static RegisterPrimOp primop_isAttrs({ + .name = "__isAttrs", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a set, and `false` otherwise. + )", + .fun = prim_isAttrs, +}); static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1350,6 +2210,21 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, } } +static RegisterPrimOp primop_removeAttrs({ + .name = "removeAttrs", + .args = {"set", "list"}, + .doc = R"( + Remove the attributes listed in *list* from *set*. The attributes + don’t have to exist in *set*. For instance, + + ```nix + removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ] + ``` + + evaluates to `{ y = 2; }`. + )", + .fun = prim_removeAttrs, +}); /* Builds a set from a list specifying (name, value) pairs. To be precise, a list [{name = "name1"; value = value1;} ... {name = @@ -1368,22 +2243,25 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v2(*args[0]->listElems()[i]); state.forceAttrs(v2, pos); - Bindings::iterator j = v2.attrs->find(state.sName); - if (j == v2.attrs->end()) - throw TypeError({ - .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"), - .errPos = pos - }); - string name = state.forceStringNoCtx(*j->value, pos); + Bindings::iterator j = getAttr( + state, + "listToAttrs", + state.sName, + v2.attrs, + pos + ); + + string name = state.forceStringNoCtx(*j->value, *j->pos); Symbol sym = state.symbols.create(name); if (seen.insert(sym).second) { - Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); - if (j2 == v2.attrs->end()) - throw TypeError({ - .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"), - .errPos = pos - }); + Bindings::iterator j2 = getAttr( + state, + "listToAttrs", + state.sValue, + v2.attrs, + pos + ); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); } } @@ -1391,10 +2269,31 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, v.attrs->sort(); } +static RegisterPrimOp primop_listToAttrs({ + .name = "__listToAttrs", + .args = {"e"}, + .doc = R"( + Construct a set from a list specifying the names and values of each + attribute. Each element of the list should be a set consisting of a + string-valued attribute `name` specifying the name of the attribute, + and an attribute `value` specifying its value. Example: + + ```nix + builtins.listToAttrs + [ { name = "foo"; value = 123; } + { name = "bar"; value = 456; } + ] + ``` + + evaluates to + + ```nix + { foo = 123; bar = 456; } + ``` + )", + .fun = prim_listToAttrs, +}); -/* Return the right-biased intersection of two sets as1 and as2, - i.e. a set that contains every attribute from as2 that is also a - member of as1. */ static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceAttrs(*args[0], pos); @@ -1409,14 +2308,16 @@ static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * ar } } +static RegisterPrimOp primop_intersectAttrs({ + .name = "__intersectAttrs", + .args = {"e1", "e2"}, + .doc = R"( + Return a set consisting of the attributes in the set *e2* that also + exist in the set *e1*. + )", + .fun = prim_intersectAttrs, +}); -/* Collect each attribute named `attr' from a list of attribute sets. - Sets that don't contain the named attribute are ignored. - - Example: - catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] - => [1 2] -*/ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { Symbol attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos)); @@ -1438,26 +2339,33 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va v.listElems()[n] = res[n]; } +static RegisterPrimOp primop_catAttrs({ + .name = "__catAttrs", + .args = {"attr", "list"}, + .doc = R"( + Collect each attribute named *attr* from a list of attribute + sets. Attrsets that don't contain the named attribute are + ignored. For example, -/* Return a set containing the names of the formal arguments expected - by the function `f'. The value of each attribute is a Boolean - denoting whether the corresponding argument has a default value. For instance, - - functionArgs ({ x, y ? 123}: ...) - => { x = false; y = true; } + ```nix + builtins.catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] + ``` - "Formal argument" here refers to the attributes pattern-matched by - the function. Plain lambdas are not included, e.g. + evaluates to `[1 2]`. + )", + .fun = prim_catAttrs, +}); - functionArgs (x: ...) - => { } -*/ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - if (args[0]->type != tLambda) + if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) { + state.mkAttrs(v, 0); + return; + } + if (!args[0]->isLambda()) throw TypeError({ - .hint = hintfmt("'functionArgs' requires a function"), + .msg = hintfmt("'functionArgs' requires a function"), .errPos = pos }); @@ -1470,14 +2378,30 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args for (auto & i : args[0]->lambda.fun->formals->formals) { // !!! should optimise booleans (allocate only once) Value * value = state.allocValue(); - v.attrs->push_back(Attr(i.name, value, &i.pos)); + v.attrs->push_back(Attr(i.name, value, ptr(&i.pos))); mkBool(*value, i.def); } v.attrs->sort(); } - -/* Apply a function to every element of an attribute set. */ +static RegisterPrimOp primop_functionArgs({ + .name = "__functionArgs", + .args = {"f"}, + .doc = R"( + Return a set containing the names of the formal arguments expected + by the function *f*. The value of each attribute is a Boolean + denoting whether the corresponding argument has a default value. For + instance, `functionArgs ({ x, y ? 123}: ...) = { x = false; y = + true; }`. + + "Formal argument" here refers to the attributes pattern-matched by + the function. Plain lambdas are not included, e.g. `functionArgs (x: + ...) = { }`. + )", + .fun = prim_functionArgs, +}); + +/* */ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceAttrs(*args[1], pos); @@ -1493,6 +2417,20 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va } } +static RegisterPrimOp primop_mapAttrs({ + .name = "__mapAttrs", + .args = {"f", "attrset"}, + .doc = R"( + Apply function *f* to every element of *attrset*. For example, + + ```nix + builtins.mapAttrs (name: value: value * 10) { a = 1; b = 2; } + ``` + + evaluates to `{ a = 10; b = 20; }`. + )", + .fun = prim_mapAttrs, +}); /************************************************************* @@ -1504,29 +2442,45 @@ static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Va static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->isList()); + mkBool(v, args[0]->type() == nList); } +static RegisterPrimOp primop_isList({ + .name = "__isList", + .args = {"e"}, + .doc = R"( + Return `true` if *e* evaluates to a list, and `false` otherwise. + )", + .fun = prim_isList, +}); static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v) { state.forceList(list, pos); if (n < 0 || (unsigned int) n >= list.listSize()) throw Error({ - .hint = hintfmt("list index %1% is out of bounds", n), + .msg = hintfmt("list index %1% is out of bounds", n), .errPos = pos }); state.forceValue(*list.listElems()[n], pos); v = *list.listElems()[n]; } - /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const Pos & pos, Value * * args, Value & v) { elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v); } +static RegisterPrimOp primop_elemAt({ + .name = "__elemAt", + .args = {"xs", "n"}, + .doc = R"( + Return element *n* from the list *xs*. Elements are counted starting + from 0. A fatal error occurs if the index is out of bounds. + )", + .fun = prim_elemAt, +}); /* Return the first element of a list. */ static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1534,6 +2488,16 @@ static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value elemAt(state, pos, *args[0], 0, v); } +static RegisterPrimOp primop_head({ + .name = "__head", + .args = {"list"}, + .doc = R"( + Return the first element of a list; abort evaluation if the argument + isn’t a list or is an empty list. You can test whether a list is + empty by comparing it with `[]`. + )", + .fun = prim_head, +}); /* Return a list consisting of everything but the first element of a list. Warning: this function takes O(n) time, so you probably @@ -1543,7 +2507,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value state.forceList(*args[0], pos); if (args[0]->listSize() == 0) throw Error({ - .hint = hintfmt("'tail' called on an empty list"), + .msg = hintfmt("'tail' called on an empty list"), .errPos = pos }); @@ -1552,6 +2516,21 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value v.listElems()[n] = args[0]->listElems()[n + 1]; } +static RegisterPrimOp primop_tail({ + .name = "__tail", + .args = {"list"}, + .doc = R"( + Return the second to last elements of a list; abort evaluation if + the argument isn’t a list or is an empty list. + + > **Warning** + > + > This function should generally be avoided since it's inefficient: + > unlike Haskell's `tail`, it takes O(n) time, so recursing over a + > list by repeatedly calling `tail` takes O(n^2) time. + )", + .fun = prim_tail, +}); /* Apply a function to every element of a list. */ static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1565,6 +2544,21 @@ static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & *args[0], *args[1]->listElems()[n]); } +static RegisterPrimOp primop_map({ + .name = "map", + .args = {"f", "list"}, + .doc = R"( + Apply the function *f* to each element in the list *list*. For + example, + + ```nix + map (x: "foo" + x) [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + )", + .fun = prim_map, +}); /* Filter a list using a predicate; that is, return a list containing every element from the list for which the predicate function @@ -1596,6 +2590,15 @@ static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Valu } } +static RegisterPrimOp primop_filter({ + .name = "__filter", + .args = {"f", "list"}, + .doc = R"( + Return a list consisting of the elements of *list* for which the + function *f* returns `true`. + )", + .fun = prim_filter, +}); /* Return true if a list contains a given element. */ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1610,6 +2613,15 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value mkBool(v, res); } +static RegisterPrimOp primop_elem({ + .name = "__elem", + .args = {"x", "xs"}, + .doc = R"( + Return `true` if a value equal to *x* occurs in the list *xs*, and + `false` otherwise. + )", + .fun = prim_elem, +}); /* Concatenate a list of lists. */ static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1618,6 +2630,14 @@ static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args, state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos); } +static RegisterPrimOp primop_concatLists({ + .name = "__concatLists", + .args = {"lists"}, + .doc = R"( + Concatenate a list of lists into a single list. + )", + .fun = prim_concatLists, +}); /* Return the length of a list. This is an O(1) time operation. */ static void prim_length(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1626,6 +2646,14 @@ static void prim_length(EvalState & state, const Pos & pos, Value * * args, Valu mkInt(v, args[0]->listSize()); } +static RegisterPrimOp primop_length({ + .name = "__length", + .args = {"e"}, + .doc = R"( + Return the length of the list *e*. + )", + .fun = prim_length, +}); /* Reduce a list by applying a binary operator, from left to right. The operator is applied strictly. */ @@ -1650,6 +2678,18 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, } } +static RegisterPrimOp primop_foldlStrict({ + .name = "__foldl'", + .args = {"op", "nul", "list"}, + .doc = R"( + Reduce a list by applying a binary operator, from left to right, + e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) + ...`. The operator is applied strictly, i.e., its arguments are + evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]` + evaluates to 6. + )", + .fun = prim_foldlStrict, +}); static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1675,12 +2715,30 @@ static void prim_any(EvalState & state, const Pos & pos, Value * * args, Value & anyOrAll(true, state, pos, args, v); } +static RegisterPrimOp primop_any({ + .name = "__any", + .args = {"pred", "list"}, + .doc = R"( + Return `true` if the function *pred* returns `true` for at least one + element of *list*, and `false` otherwise. + )", + .fun = prim_any, +}); static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value & v) { anyOrAll(false, state, pos, args, v); } +static RegisterPrimOp primop_all({ + .name = "__all", + .args = {"pred", "list"}, + .doc = R"( + Return `true` if the function *pred* returns `true` for all elements + of *list*, and `false` otherwise. + )", + .fun = prim_all, +}); static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1688,7 +2746,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val if (len < 0) throw EvalError({ - .hint = hintfmt("cannot create list of size %1%", len), + .msg = hintfmt("cannot create list of size %1%", len), .errPos = pos }); @@ -1701,6 +2759,21 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val } } +static RegisterPrimOp primop_genList({ + .name = "__genList", + .args = {"generator", "length"}, + .doc = R"( + Generate list of size *length*, with each element *i* equal to the + value returned by *generator* `i`. For example, + + ```nix + builtins.genList (x: x * x) 5 + ``` + + returns the list `[ 0 1 4 9 16 ]`. + )", + .fun = prim_genList, +}); static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v); @@ -1721,7 +2794,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ - if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan) + if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) return CompareValues()(a, b); Value vTmp1, vTmp2; @@ -1736,6 +2809,26 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value std::stable_sort(v.listElems(), v.listElems() + len, comparator); } +static RegisterPrimOp primop_sort({ + .name = "__sort", + .args = {"comparator", "list"}, + .doc = R"( + Return *list* in sorted order. It repeatedly calls the function + *comparator* with two elements. The comparator should return `true` + if the first element is less than the second, and `false` otherwise. + For example, + + ```nix + builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ] + ``` + + produces the list `[ 42 77 147 249 483 526 ]`. + + This is a stable sort: it preserves the relative order of elements + deemed equal by the comparator. + )", + .fun = prim_sort, +}); static void prim_partition(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1774,9 +2867,29 @@ static void prim_partition(EvalState & state, const Pos & pos, Value * * args, V v.attrs->sort(); } +static RegisterPrimOp primop_partition({ + .name = "__partition", + .args = {"pred", "list"}, + .doc = R"( + Given a predicate function *pred*, this function returns an + attrset containing a list named `right`, containing the elements + in *list* for which *pred* returned `true`, and a list named + `wrong`, containing the elements for which it returned + `false`. For example, + + ```nix + builtins.partition (x: x > 10) [1 23 9 3 42] + ``` + + evaluates to + + ```nix + { right = [ 23 42 ]; wrong = [ 1 9 3 ]; } + ``` + )", + .fun = prim_partition, +}); -/* concatMap = f: list: concatLists (map f list); */ -/* C++-version is to avoid allocating `mkApp', call `f' eagerly */ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceFunction(*args[0], pos); @@ -1789,7 +2902,12 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V for (unsigned int n = 0; n < nrLists; ++n) { Value * vElem = args[1]->listElems()[n]; state.callFunction(*args[0], *vElem, lists[n], pos); - state.forceList(lists[n], pos); + try { + state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos))); + } catch (TypeError &e) { + e.addTrace(pos, hintfmt("while invoking '%s'", "concatMap")); + throw e; + } len += lists[n].listSize(); } @@ -1803,6 +2921,16 @@ static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, V } } +static RegisterPrimOp primop_concatMap({ + .name = "__concatMap", + .args = {"f", "list"}, + .doc = R"( + This function is equivalent to `builtins.concatLists (map f list)` + but is more efficient. + )", + .fun = prim_concatMap, +}); + /************************************************************* * Integer arithmetic @@ -1813,34 +2941,58 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_add({ + .name = "__add", + .args = {"e1", "e2"}, + .doc = R"( + Return the sum of the numbers *e1* and *e2*. + )", + .fun = prim_add, +}); static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_sub({ + .name = "__sub", + .args = {"e1", "e2"}, + .doc = R"( + Return the difference between the numbers *e1* and *e2*. + )", + .fun = prim_sub, +}); static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_mul({ + .name = "__mul", + .args = {"e1", "e2"}, + .doc = R"( + Return the product of the numbers *e1* and *e2*. + )", + .fun = prim_mul, +}); static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1850,11 +3002,11 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & NixFloat f2 = state.forceFloat(*args[1], pos); if (f2 == 0) throw EvalError({ - .hint = hintfmt("division by zero"), + .msg = hintfmt("division by zero"), .errPos = pos }); - if (args[0]->type == tFloat || args[1]->type == tFloat) { + if (args[0]->type() == nFloat || args[1]->type() == nFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); } else { NixInt i1 = state.forceInt(*args[0], pos); @@ -1862,7 +3014,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1) throw EvalError({ - .hint = hintfmt("overflow in integer division"), + .msg = hintfmt("overflow in integer division"), .errPos = pos }); @@ -1870,21 +3022,57 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & } } +static RegisterPrimOp primop_div({ + .name = "__div", + .args = {"e1", "e2"}, + .doc = R"( + Return the quotient of the numbers *e1* and *e2*. + )", + .fun = prim_div, +}); + static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v) { mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_bitAnd({ + .name = "__bitAnd", + .args = {"e1", "e2"}, + .doc = R"( + Return the bitwise AND of the integers *e1* and *e2*. + )", + .fun = prim_bitAnd, +}); + static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v) { mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_bitOr({ + .name = "__bitOr", + .args = {"e1", "e2"}, + .doc = R"( + Return the bitwise OR of the integers *e1* and *e2*. + )", + .fun = prim_bitOr, +}); + static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v) { mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); } +static RegisterPrimOp primop_bitXor({ + .name = "__bitXor", + .args = {"e1", "e2"}, + .doc = R"( + Return the bitwise XOR of the integers *e1* and *e2*. + )", + .fun = prim_bitXor, +}); + static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); @@ -1893,6 +3081,17 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va mkBool(v, comp(args[0], args[1])); } +static RegisterPrimOp primop_lessThan({ + .name = "__lessThan", + .args = {"e1", "e2"}, + .doc = R"( + Return `true` if the number *e1* is less than the number *e2*, and + `false` otherwise. Evaluation aborts if either *e1* or *e2* does not + evaluate to a number. + )", + .fun = prim_lessThan, +}); + /************************************************************* * String manipulation @@ -1909,6 +3108,29 @@ static void prim_toString(EvalState & state, const Pos & pos, Value * * args, Va mkString(v, s, context); } +static RegisterPrimOp primop_toString({ + .name = "toString", + .args = {"e"}, + .doc = R"( + Convert the expression *e* to a string. *e* can be: + + - A string (in which case the string is returned unmodified). + + - A path (e.g., `toString /foo/bar` yields `"/foo/bar"`. + + - A set containing `{ __toString = self: ...; }` or `{ outPath = ...; }`. + + - An integer. + + - A list, in which case the string representations of its elements + are joined with spaces. + + - A Boolean (`false` yields `""`, `true` yields `"1"`). + + - `null`, which yields the empty string. + )", + .fun = prim_toString, +}); /* `substring start len str' returns the substring of `str' starting at character position `min(start, stringLength str)' inclusive and @@ -1923,13 +3145,32 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V if (start < 0) throw EvalError({ - .hint = hintfmt("negative start position in 'substring'"), + .msg = hintfmt("negative start position in 'substring'"), .errPos = pos }); mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); } +static RegisterPrimOp primop_substring({ + .name = "__substring", + .args = {"start", "len", "s"}, + .doc = R"( + Return the substring of *s* from character position *start* + (zero-based) up to but not including *start + len*. If *start* is + greater than the length of the string, an empty string is returned, + and if *start + len* lies beyond the end of the string, only the + substring up to the end of the string is returned. *start* must be + non-negative. For example, + + ```nix + builtins.substring 0 3 "nixos" + ``` + + evaluates to `"nix"`. + )", + .fun = prim_substring, +}); static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -1938,6 +3179,15 @@ static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args mkInt(v, s.size()); } +static RegisterPrimOp primop_stringLength({ + .name = "__stringLength", + .args = {"e"}, + .doc = R"( + Return the length of the string *e*. If *e* is not a string, + evaluation is aborted. + )", + .fun = prim_stringLength, +}); /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -1946,28 +3196,46 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, std::optional<HashType> ht = parseHashType(type); if (!ht) throw Error({ - .hint = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash type '%1%'", type), .errPos = pos }); PathSet context; // discarded string s = state.forceString(*args[1], context, pos); - mkString(v, hashString(*ht, s).to_string(Base16, false), context); + mkString(v, hashString(*ht, s).to_string(Base16, false)); } +static RegisterPrimOp primop_hashString({ + .name = "__hashString", + .args = {"type", "s"}, + .doc = R"( + Return a base-16 representation of the cryptographic hash of string + *s*. The hash algorithm specified by *type* must be one of `"md5"`, + `"sha1"`, `"sha256"` or `"sha512"`. + )", + .fun = prim_hashString, +}); + +struct RegexCache +{ + std::unordered_map<std::string, std::regex> cache; +}; + +std::shared_ptr<RegexCache> makeRegexCache() +{ + return std::make_shared<RegexCache>(); +} -/* Match a regular expression against a string and return either - ‘null’ or a list containing substring matches. */ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) { auto re = state.forceStringNoCtx(*args[0], pos); try { - auto regex = state.regexCache.find(re); - if (regex == state.regexCache.end()) - regex = state.regexCache.emplace(re, std::regex(re, std::regex::extended)).first; + auto regex = state.regexCache->cache.find(re); + if (regex == state.regexCache->cache.end()) + regex = state.regexCache->cache.emplace(re, std::regex(re, std::regex::extended)).first; PathSet context; const std::string str = state.forceString(*args[1], context, pos); @@ -1992,18 +3260,53 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ throw EvalError({ - .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), .errPos = pos }); } else { throw EvalError({ - .hint = hintfmt("invalid regular expression '%s'", re), + .msg = hintfmt("invalid regular expression '%s'", re), .errPos = pos }); } } } +static RegisterPrimOp primop_match({ + .name = "__match", + .args = {"regex", "str"}, + .doc = R"s( + Returns a list if the [extended POSIX regular + expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04) + *regex* matches *str* precisely, otherwise returns `null`. Each item + in the list is a regex group. + + ```nix + builtins.match "ab" "abc" + ``` + + Evaluates to `null`. + + ```nix + builtins.match "abc" "abc" + ``` + + Evaluates to `[ ]`. + + ```nix + builtins.match "a(b)(c)" "abc" + ``` + + Evaluates to `[ "b" "c" ]`. + + ```nix + builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " + ``` + + Evaluates to `[ "foo" ]`. + )s", + .fun = prim_match, +}); /* Split a string with a regular expression, and return a list of the non-matching parts interleaved by the lists of the matching groups. */ @@ -2065,20 +3368,56 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ throw EvalError({ - .hint = hintfmt("memory limit exceeded by regular expression '%s'", re), + .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), .errPos = pos }); } else { throw EvalError({ - .hint = hintfmt("invalid regular expression '%s'", re), + .msg = hintfmt("invalid regular expression '%s'", re), .errPos = pos }); } } } +static RegisterPrimOp primop_split({ + .name = "__split", + .args = {"regex", "str"}, + .doc = R"s( + Returns a list composed of non matched strings interleaved with the + lists of the [extended POSIX regular + expression](http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04) + *regex* matches of *str*. Each item in the lists of matched + sequences is a regex group. + + ```nix + builtins.split "(a)b" "abc" + ``` -static void prim_concatStringSep(EvalState & state, const Pos & pos, Value * * args, Value & v) + Evaluates to `[ "" [ "a" ] "c" ]`. + + ```nix + builtins.split "([ac])" "abc" + ``` + + Evaluates to `[ "" [ "a" ] "b" [ "c" ] "" ]`. + + ```nix + builtins.split "(a)|(c)" "abc" + ``` + + Evaluates to `[ "" [ "a" null ] "b" [ null "c" ] "" ]`. + + ```nix + builtins.split "([[:upper:]]+)" " FOO " + ``` + + Evaluates to `[ " " [ "FOO" ] " " ]`. + )s", + .fun = prim_split, +}); + +static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * * args, Value & v) { PathSet context; @@ -2097,6 +3436,16 @@ static void prim_concatStringSep(EvalState & state, const Pos & pos, Value * * a mkString(v, res, context); } +static RegisterPrimOp primop_concatStringsSep({ + .name = "__concatStringsSep", + .args = {"separator", "list"}, + .doc = R"( + Concatenate a list of strings with a separator between each + element, e.g. `concatStringsSep "/" ["usr" "local" "bin"] == + "usr/local/bin"`. + )", + .fun = prim_concatStringsSep, +}); static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2104,7 +3453,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar state.forceList(*args[1], pos); if (args[0]->listSize() != args[1]->listSize()) throw EvalError({ - .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), + .msg = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"), .errPos = pos }); @@ -2156,6 +3505,22 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar mkString(v, res, context); } +static RegisterPrimOp primop_replaceStrings({ + .name = "__replaceStrings", + .args = {"from", "to", "s"}, + .doc = R"( + Given string *s*, replace every occurrence of the strings in *from* + with the corresponding string in *to*. For example, + + ```nix + builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" + ``` + + evaluates to `"fabir"`. + )", + .fun = prim_replaceStrings, +}); + /************************************************************* * Versions @@ -2172,6 +3537,19 @@ static void prim_parseDrvName(EvalState & state, const Pos & pos, Value * * args v.attrs->sort(); } +static RegisterPrimOp primop_parseDrvName({ + .name = "__parseDrvName", + .args = {"s"}, + .doc = R"( + Split the string *s* into a package name and version. The package + name is everything up to but not including the first dash followed + by a digit, and the version is everything following that dash. The + result is returned in a set `{ name, version }`. Thus, + `builtins.parseDrvName "nix-0.12pre12876"` returns `{ name = + "nix"; version = "0.12pre12876"; }`. + )", + .fun = prim_parseDrvName, +}); static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2180,6 +3558,18 @@ static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * a mkInt(v, compareVersions(version1, version2)); } +static RegisterPrimOp primop_compareVersions({ + .name = "__compareVersions", + .args = {"s1", "s2"}, + .doc = R"( + Compare two strings representing versions and return `-1` if + version *s1* is older than version *s2*, `0` if they are the same, + and `1` if *s1* is newer than *s2*. The version comparison + algorithm is the same as the one used by [`nix-env + -u`](../command-ref/nix-env.md#operation---upgrade). + )", + .fun = prim_compareVersions, +}); static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v) { @@ -2200,6 +3590,17 @@ static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args } } +static RegisterPrimOp primop_splitVersion({ + .name = "__splitVersion", + .args = {"s"}, + .doc = R"( + Split a string representing a version into its components, by the + same version splitting logic underlying the version comparison in + [`nix-env -u`](../command-ref/nix-env.md#operation---upgrade). + )", + .fun = prim_splitVersion, +}); + /************************************************************* * Primop registration @@ -2213,7 +3614,20 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun, std::optional<std::string> requiredFeature) { if (!primOps) primOps = new PrimOps; - primOps->push_back({name, arity, fun, requiredFeature}); + primOps->push_back({ + .name = name, + .args = {}, + .arity = arity, + .requiredFeature = std::move(requiredFeature), + .fun = fun + }); +} + + +RegisterPrimOp::RegisterPrimOp(Info && info) +{ + if (!primOps) primOps = new PrimOps; + primOps->push_back(std::move(info)); } @@ -2237,21 +3651,10 @@ void EvalState::createBaseEnv() mkNull(v); addConstant("null", v); - auto vThrow = addPrimOp("throw", 1, prim_throw); - - auto addPurityError = [&](const std::string & name) { - Value * v2 = allocValue(); - mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name)); - mkApp(v, *vThrow, *v2); - addConstant(name, v); - }; - if (!evalSettings.pureEval) { mkInt(v, time(0)); addConstant("__currentTime", v); - } - if (!evalSettings.pureEval) { mkString(v, settings.thisSystem.get()); addConstant("__currentSystem", v); } @@ -2270,132 +3673,16 @@ void EvalState::createBaseEnv() addConstant("__langVersion", v); // Miscellaneous - auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport); - Value * v2 = allocValue(); - mkAttrs(*v2, 0); - mkApp(v, *vScopedImport, *v2); - forceValue(v); - addConstant("import", v); if (evalSettings.enableNativeCode) { addPrimOp("__importNative", 2, prim_importNative); addPrimOp("__exec", 1, prim_exec); } - addPrimOp("__typeOf", 1, prim_typeOf); - addPrimOp("isNull", 1, prim_isNull); - addPrimOp("__isFunction", 1, prim_isFunction); - addPrimOp("__isString", 1, prim_isString); - addPrimOp("__isInt", 1, prim_isInt); - addPrimOp("__isFloat", 1, prim_isFloat); - addPrimOp("__isBool", 1, prim_isBool); - addPrimOp("__isPath", 1, prim_isPath); - addPrimOp("__genericClosure", 1, prim_genericClosure); - addPrimOp("abort", 1, prim_abort); - addPrimOp("__addErrorContext", 2, prim_addErrorContext); - addPrimOp("__tryEval", 1, prim_tryEval); - addPrimOp("__getEnv", 1, prim_getEnv); - - // Strictness - addPrimOp("__seq", 2, prim_seq); - addPrimOp("__deepSeq", 2, prim_deepSeq); - - // Debugging - addPrimOp("__trace", 2, prim_trace); - - // Paths - addPrimOp("__toPath", 1, prim_toPath); - if (evalSettings.pureEval) - addPurityError("__storePath"); - else - addPrimOp("__storePath", 1, prim_storePath); - addPrimOp("__pathExists", 1, prim_pathExists); - addPrimOp("baseNameOf", 1, prim_baseNameOf); - addPrimOp("dirOf", 1, prim_dirOf); - addPrimOp("__readFile", 1, prim_readFile); - addPrimOp("__readDir", 1, prim_readDir); - addPrimOp("__findFile", 2, prim_findFile); - addPrimOp("__hashFile", 2, prim_hashFile); - - // Creating files - addPrimOp("__toXML", 1, prim_toXML); - addPrimOp("__toJSON", 1, prim_toJSON); - addPrimOp("__fromJSON", 1, prim_fromJSON); - addPrimOp("__toFile", 2, prim_toFile); - addPrimOp("__filterSource", 2, prim_filterSource); - addPrimOp("__path", 1, prim_path); - - // Sets - addPrimOp("__attrNames", 1, prim_attrNames); - addPrimOp("__attrValues", 1, prim_attrValues); - addPrimOp("__getAttr", 2, prim_getAttr); - addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos); - addPrimOp("__hasAttr", 2, prim_hasAttr); - addPrimOp("__isAttrs", 1, prim_isAttrs); - addPrimOp("removeAttrs", 2, prim_removeAttrs); - addPrimOp("__listToAttrs", 1, prim_listToAttrs); - addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); - addPrimOp("__catAttrs", 2, prim_catAttrs); - addPrimOp("__functionArgs", 1, prim_functionArgs); - addPrimOp("__mapAttrs", 2, prim_mapAttrs); - - // Lists - addPrimOp("__isList", 1, prim_isList); - addPrimOp("__elemAt", 2, prim_elemAt); - addPrimOp("__head", 1, prim_head); - addPrimOp("__tail", 1, prim_tail); - addPrimOp("map", 2, prim_map); - addPrimOp("__filter", 2, prim_filter); - addPrimOp("__elem", 2, prim_elem); - addPrimOp("__concatLists", 1, prim_concatLists); - addPrimOp("__length", 1, prim_length); - addPrimOp("__foldl'", 3, prim_foldlStrict); - addPrimOp("__any", 2, prim_any); - addPrimOp("__all", 2, prim_all); - addPrimOp("__genList", 2, prim_genList); - addPrimOp("__sort", 2, prim_sort); - addPrimOp("__partition", 2, prim_partition); - addPrimOp("__concatMap", 2, prim_concatMap); - - // Integer arithmetic - addPrimOp("__add", 2, prim_add); - addPrimOp("__sub", 2, prim_sub); - addPrimOp("__mul", 2, prim_mul); - addPrimOp("__div", 2, prim_div); - addPrimOp("__bitAnd", 2, prim_bitAnd); - addPrimOp("__bitOr", 2, prim_bitOr); - addPrimOp("__bitXor", 2, prim_bitXor); - addPrimOp("__lessThan", 2, prim_lessThan); - - // String manipulation - addPrimOp("toString", 1, prim_toString); - addPrimOp("__substring", 3, prim_substring); - addPrimOp("__stringLength", 1, prim_stringLength); - addPrimOp("__hashString", 2, prim_hashString); - addPrimOp("__match", 2, prim_match); - addPrimOp("__split", 2, prim_split); - addPrimOp("__concatStringsSep", 2, prim_concatStringSep); - addPrimOp("__replaceStrings", 3, prim_replaceStrings); - - // Versions - addPrimOp("__parseDrvName", 1, prim_parseDrvName); - addPrimOp("__compareVersions", 2, prim_compareVersions); - addPrimOp("__splitVersion", 1, prim_splitVersion); - - // Derivations - addPrimOp("derivationStrict", 1, prim_derivationStrict); - addPrimOp("placeholder", 1, prim_placeholder); - - /* Add a wrapper around the derivation primop that computes the - `drvPath' and `outPath' attributes lazily. */ - string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); - sDerivationNix = symbols.create(path); - evalFile(path, v); - addConstant("derivation", v); /* Add a value containing the current Nix expression search path. */ mkList(v, searchPath.size()); int n = 0; for (auto & i : searchPath) { - v2 = v.listElems()[n++] = allocValue(); + auto v2 = v.listElems()[n++] = allocValue(); mkAttrs(*v2, 2); mkString(*allocAttr(*v2, symbols.create("path")), i.second); mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); @@ -2406,7 +3693,21 @@ void EvalState::createBaseEnv() if (RegisterPrimOp::primOps) for (auto & primOp : *RegisterPrimOp::primOps) if (!primOp.requiredFeature || settings.isExperimentalFeatureEnabled(*primOp.requiredFeature)) - addPrimOp(primOp.name, primOp.arity, primOp.primOp); + addPrimOp({ + .fun = primOp.fun, + .arity = std::max(primOp.args.size(), primOp.arity), + .name = symbols.create(primOp.name), + .args = std::move(primOp.args), + .doc = primOp.doc, + }); + + /* Add a wrapper around the derivation primop that computes the + `drvPath' and `outPath' attributes lazily. */ + sDerivationNix = symbols.create("//builtin/derivation.nix"); + eval(parse( + #include "primops/derivation.nix.gen.hh" + , foFile, sDerivationNix, "/", staticBaseEnv), v); + addConstant("derivation", v); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 75c460ecf..9d42d6539 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -1,3 +1,5 @@ +#pragma once + #include "eval.hh" #include <tuple> @@ -10,9 +12,11 @@ struct RegisterPrimOp struct Info { std::string name; - size_t arity; - PrimOpFun primOp; + std::vector<std::string> args; + size_t arity = 0; + const char * doc; std::optional<std::string> requiredFeature; + PrimOpFun fun; }; typedef std::vector<Info> PrimOps; @@ -26,6 +30,8 @@ struct RegisterPrimOp size_t arity, PrimOpFun fun, std::optional<std::string> requiredFeature = {}); + + RegisterPrimOp(Info && info); }; /* These primops are disabled without enableNativeCode, but plugins diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index dbb93bae6..31cf812b4 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -11,7 +11,7 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, mkString(v, s, PathSet()); } -static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); +static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) @@ -21,7 +21,7 @@ static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, mkBool(v, !context.empty()); } -static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); +static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a @@ -42,7 +42,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & po mkString(v, s, context2); } -static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); +static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); /* Extract the context of a string as a structured Nix value. @@ -127,7 +127,7 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, v.attrs->sort(); } -static RegisterPrimOp r4("__getContext", 1, prim_getContext); +static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); /* Append the given context to a given string. @@ -147,7 +147,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg for (auto & i : *args[1]->attrs) { if (!state.store->isStorePath(i.name)) throw EvalError({ - .hint = hintfmt("Context key '%s' is not a store path", i.name), + .msg = hintfmt("Context key '%s' is not a store path", i.name), .errPos = *i.pos }); if (!settings.readOnlyMode) @@ -164,7 +164,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg if (state.forceBool(*iter->value, *iter->pos)) { if (!isDerivation(i.name)) { throw EvalError({ - .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), + .msg = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name), .errPos = *i.pos }); } @@ -177,7 +177,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg state.forceList(*iter->value, *iter->pos); if (iter->value->listSize() && !isDerivation(i.name)) { throw EvalError({ - .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), + .msg = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name), .errPos = *i.pos }); } @@ -191,6 +191,6 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg mkString(v, orig, context); } -static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); +static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); } diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix new file mode 100644 index 000000000..c0fbe8082 --- /dev/null +++ b/src/libexpr/primops/derivation.nix @@ -0,0 +1,27 @@ +/* This is the implementation of the ‘derivation’ builtin function. + It's actually a wrapper around the ‘derivationStrict’ primop. */ + +drvAttrs @ { outputs ? [ "out" ], ... }: + +let + + strict = derivationStrict drvAttrs; + + commonAttrs = drvAttrs // (builtins.listToAttrs outputsList) // + { all = map (x: x.value) outputsList; + inherit drvAttrs; + }; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + outPath = builtins.getAttr outputName strict; + drvPath = strict.drvPath; + type = "derivation"; + inherit outputName; + }; + }; + + outputsList = map outputToAttrListElement outputs; + +in (builtins.head outputsList).value diff --git a/src/libexpr/primops/fetchGit.cc b/src/libexpr/primops/fetchGit.cc deleted file mode 100644 index 36b0db2bd..000000000 --- a/src/libexpr/primops/fetchGit.cc +++ /dev/null @@ -1,91 +0,0 @@ -#include "primops.hh" -#include "eval-inline.hh" -#include "store-api.hh" -#include "hash.hh" -#include "fetchers.hh" -#include "url.hh" - -namespace nix { - -static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - std::string url; - std::optional<std::string> ref; - std::optional<Hash> rev; - std::string name = "source"; - bool fetchSubmodules = false; - PathSet context; - - state.forceValue(*args[0]); - - if (args[0]->type == tAttrs) { - - state.forceAttrs(*args[0], pos); - - for (auto & attr : *args[0]->attrs) { - string n(attr.name); - if (n == "url") - url = state.coerceToString(*attr.pos, *attr.value, context, false, false); - else if (n == "ref") - ref = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "rev") - rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1); - else if (n == "name") - name = state.forceStringNoCtx(*attr.value, *attr.pos); - else if (n == "submodules") - fetchSubmodules = state.forceBool(*attr.value, *attr.pos); - else - throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name), - .errPos = *attr.pos - }); - } - - if (url.empty()) - throw EvalError({ - .hint = hintfmt("'url' argument required"), - .errPos = pos - }); - - } else - url = state.coerceToString(pos, *args[0], context, false, false); - - // FIXME: git externals probably can be used to bypass the URI - // whitelist. Ah well. - state.checkURI(url); - - if (evalSettings.pureEval && !rev) - throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); - - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "git"); - attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); - if (ref) attrs.insert_or_assign("ref", *ref); - if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - if (fetchSubmodules) attrs.insert_or_assign("submodules", true); - auto input = fetchers::inputFromAttrs(attrs); - - // FIXME: use name? - auto [tree, input2] = input->fetchTree(state.store); - - state.mkAttrs(v, 8); - auto storePath = state.store->printStorePath(tree.storePath); - mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - // Backward compatibility: set 'rev' to - // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); - mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev()); - // Backward compatibility: set 'revCount' to 0 for a dirty tree. - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), - tree.info.revCount.value_or(0)); - mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules); - v.attrs->sort(); - - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); -} - -static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); - -} diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 59166b777..3f88ccb91 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -3,8 +3,7 @@ #include "store-api.hh" #include "fetchers.hh" #include "url.hh" - -#include <regex> +#include "url-parts.hh" namespace nix { @@ -18,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); @@ -31,7 +30,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar // be both a revision or a branch/tag name. auto value = state.forceStringNoCtx(*attr.value, *attr.pos); if (std::regex_match(value, revRegex)) - rev = Hash(value, htSHA1); + rev = Hash::parseAny(value, htSHA1); else ref = value; } @@ -39,14 +38,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), + .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name), .errPos = *attr.pos }); } if (url.empty()) throw EvalError({ - .hint = hintfmt("'url' argument required"), + .msg = hintfmt("'url' argument required"), .errPos = pos }); @@ -63,31 +62,32 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar fetchers::Attrs attrs; attrs.insert_or_assign("type", "hg"); attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); + attrs.insert_or_assign("name", name); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::inputFromAttrs(attrs); + auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input->fetchTree(state.store); + auto [tree, input2] = input.fetch(state.store); state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - if (input2->getRef()) - mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef()); + if (input2.getRef()) + mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2->getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(htSHA1)); mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev()); mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12)); - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (auto revCount = input2.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); v.attrs->sort(); if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); } -static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); +static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 01d6ad8b0..9990a0207 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -3,91 +3,181 @@ #include "store-api.hh" #include "fetchers.hh" #include "filetransfer.hh" +#include "registry.hh" #include <ctime> #include <iomanip> +#include <regex> namespace nix { void emitTreeAttrs( EvalState & state, const fetchers::Tree & tree, - std::shared_ptr<const fetchers::Input> input, - Value & v) + const fetchers::Input & input, + Value & v, + bool emptyRevFallback) { + assert(input.isImmutable()); + state.mkAttrs(v, 8); auto storePath = state.store->printStorePath(tree.storePath); mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath})); - assert(tree.info.narHash); - mkString(*state.allocAttr(v, state.symbols.create("narHash")), - tree.info.narHash.to_string(SRI, true)); + // FIXME: support arbitrary input attributes. - if (input->getRev()) { - mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev()); - mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev()); + auto narHash = input.getNarHash(); + assert(narHash); + mkString(*state.allocAttr(v, state.symbols.create("narHash")), + narHash->to_string(SRI, true)); + + if (auto rev = input.getRev()) { + mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev()); + } else if (emptyRevFallback) { + // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev + auto emptyHash = Hash(htSHA1); + mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev()); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev()); } - if (tree.info.revCount) - mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); + if (input.getType() == "git") + mkBool(*state.allocAttr(v, state.symbols.create("submodules")), + fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(true)); + + if (auto revCount = input.getRevCount()) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); + else if (emptyRevFallback) + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0); - if (tree.info.lastModified) - mkString(*state.allocAttr(v, state.symbols.create("lastModified")), - fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + if (auto lastModified = input.getLastModified()) { + mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified); + mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")), + fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S"))); + } v.attrs->sort(); } -static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file") { - settings.requireExperimentalFeature("flakes"); + state.checkURI(uri); + return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; +} - std::shared_ptr<const fetchers::Input> input; +std::string fixURIForGit(std::string uri, EvalState & state) +{ + static std::regex scp_uri("([^/].*)@(.*):(.*)"); + if (uri[0] != '/' && std::regex_match(uri, scp_uri)) + return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); + else + return fixURI(uri, state); +} + +void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) +{ + string n(name); + attrs.emplace(name, n == "url" ? fixURI(v, state) : v); +} + +struct FetchTreeParams { + bool emptyRevFallback = false; + bool allowNameArgument = false; +}; + +static void fetchTree( + EvalState &state, + const Pos &pos, + Value **args, + Value &v, + const std::optional<std::string> type, + const FetchTreeParams & params = FetchTreeParams{} +) { + fetchers::Input input; PathSet context; state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); fetchers::Attrs attrs; for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); - if (attr.value->type == tString) - attrs.emplace(attr.name, attr.value->string.s); - else if (attr.value->type == tBool) - attrs.emplace(attr.name, attr.value->boolean); + if (attr.value->type() == nPath || attr.value->type() == nString) + addURI( + state, + attrs, + attr.name, + state.coerceToString(*attr.pos, *attr.value, context, false, false) + ); + else if (attr.value->type() == nString) + addURI(state, attrs, attr.name, attr.value->string.s); + else if (attr.value->type() == nBool) + attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean}); + else if (attr.value->type() == nInt) + attrs.emplace(attr.name, uint64_t(attr.value->integer)); else - throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", + throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", attr.name, showType(*attr.value)); } + if (type) + attrs.emplace("type", type.value()); + if (!attrs.count("type")) throw Error({ - .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), .errPos = pos }); - input = fetchers::inputFromAttrs(attrs); - } else - input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + if (!params.allowNameArgument) + if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) + throw Error({ + .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), + .errPos = pos + }); + + + input = fetchers::Input::fromAttrs(std::move(attrs)); + } else { + auto url = state.coerceToString(pos, *args[0], context, false, false); + + if (type == "git") { + fetchers::Attrs attrs; + attrs.emplace("type", "git"); + attrs.emplace("url", fixURIForGit(url, state)); + input = fetchers::Input::fromAttrs(std::move(attrs)); + } else { + input = fetchers::Input::fromURL(fixURI(url, state)); + } + } - if (evalSettings.pureEval && !input->isImmutable()) - throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); + if (!evalSettings.pureEval && !input.isDirect()) + input = lookupInRegistries(state.store, input).first; - // FIXME: use fetchOrSubstituteTree - auto [tree, input2] = input->fetchTree(state.store); + if (evalSettings.pureEval && !input.isImmutable()) + throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos); + + auto [tree, input2] = input.fetch(state.store); if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); - emitTreeAttrs(state, tree, input2, v); + emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback); +} + +static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + settings.requireExperimentalFeature("flakes"); + fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } -static RegisterPrimOp r("fetchTree", 1, prim_fetchTree); +// FIXME: document +static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, const string & who, bool unpack, std::string name) @@ -97,7 +187,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); @@ -111,14 +201,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, name = state.forceStringNoCtx(*attr.value, *attr.pos); else throw EvalError({ - .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), + .msg = hintfmt("unsupported argument '%s' to '%s'", attr.name, who), .errPos = *attr.pos }); } if (!url) throw EvalError({ - .hint = hintfmt("'url' argument required"), + .msg = hintfmt("'url' argument required"), .errPos = pos }); } else @@ -136,7 +226,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; auto path = state.store->toRealPath(storePath); @@ -146,7 +236,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, ? state.store->queryPathInfo(storePath)->narHash : hashFile(htSHA256, path); if (hash != *expectedHash) - throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } @@ -161,12 +251,192 @@ static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Va fetch(state, pos, args, v, "fetchurl", false, ""); } +static RegisterPrimOp primop_fetchurl({ + .name = "__fetchurl", + .args = {"url"}, + .doc = R"( + Download the specified URL and return the path of the downloaded + file. This function is not available if [restricted evaluation + mode](../command-ref/conf-file.md) is enabled. + )", + .fun = prim_fetchurl, +}); + static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) { fetch(state, pos, args, v, "fetchTarball", true, "source"); } -static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl); -static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball); +static RegisterPrimOp primop_fetchTarball({ + .name = "fetchTarball", + .args = {"args"}, + .doc = R"( + Download the specified URL, unpack it and return the path of the + unpacked tree. The file must be a tape archive (`.tar`) compressed + with `gzip`, `bzip2` or `xz`. The top-level path component of the + files in the tarball is removed, so it is best if the tarball + contains a single directory at top level. The typical use of the + function is to obtain external Nix expression dependencies, such as + a particular version of Nixpkgs, e.g. + + ```nix + with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; + + stdenv.mkDerivation { … } + ``` + + The fetched tarball is cached for a certain amount of time (1 hour + by default) in `~/.cache/nix/tarballs/`. You can change the cache + timeout either on the command line with `--option tarball-ttl number + of seconds` or in the Nix configuration file with this option: ` + number of seconds to cache `. + + Note that when obtaining the hash with ` nix-prefetch-url ` the + option `--unpack` is required. + + This function can also verify the contents against a hash. In that + case, the function takes a set instead of a URL. The set requires + the attribute `url` and the attribute `sha256`, e.g. + + ```nix + with import (fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz"; + sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2"; + }) {}; + + stdenv.mkDerivation { … } + ``` + + This function is not available if [restricted evaluation + mode](../command-ref/conf-file.md) is enabled. + )", + .fun = prim_fetchTarball, +}); + +static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) +{ + fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); +} + +static RegisterPrimOp primop_fetchGit({ + .name = "fetchGit", + .args = {"args"}, + .doc = R"( + Fetch a path from git. *args* can be a URL, in which case the HEAD + of the repo at that URL is fetched. Otherwise, it can be an + attribute with the following attributes (all except `url` optional): + + - url\ + The URL of the repo. + + - name\ + The name of the directory the repo should be exported to in the + store. Defaults to the basename of the URL. + + - rev\ + The git revision to fetch. Defaults to the tip of `ref`. + + - ref\ + The git ref to look for the requested revision under. This is + often a branch or tag name. Defaults to `HEAD`. + + By default, the `ref` value is prefixed with `refs/heads/`. As + of Nix 2.3.0 Nix will not prefix `refs/heads/` if `ref` starts + with `refs/`. + + - submodules\ + A Boolean parameter that specifies whether submodules should be + checked out. Defaults to `false`. + + - allRefs\ + Whether to fetch all refs of the repository. With this argument being + true, it's possible to load a `rev` from *any* `ref` (by default only + `rev`s from the specified `ref` are supported). + + Here are some examples of how to use `fetchGit`. + + - To fetch a private repository over SSH: + + ```nix + builtins.fetchGit { + url = "git@github.com:my-secret/repository.git"; + ref = "master"; + rev = "adab8b916a45068c044658c4158d81878f9ed1c3"; + } + ``` + + - To fetch an arbitrary reference: + + ```nix + builtins.fetchGit { + url = "https://github.com/NixOS/nix.git"; + ref = "refs/heads/0.5-release"; + } + ``` + + - If the revision you're looking for is in the default branch of + the git repository you don't strictly need to specify the branch + name in the `ref` attribute. + + However, if the revision you're looking for is in a future + branch for the non-default branch you will need to specify the + the `ref` attribute as well. + + ```nix + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; + ref = "1.11-maintenance"; + } + ``` + + > **Note** + > + > It is nice to always specify the branch which a revision + > belongs to. Without the branch being specified, the fetcher + > might fail if the default branch changes. Additionally, it can + > be confusing to try a commit from a non-default branch and see + > the fetch fail. If the branch is specified the fault is much + > more obvious. + + - If the revision you're looking for is in the default branch of + the git repository you may omit the `ref` attribute. + + ```nix + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; + } + ``` + + - To fetch a specific tag: + + ```nix + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + ref = "refs/tags/1.9"; + } + ``` + + - To fetch the latest version of a remote branch: + + ```nix + builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + ref = "master"; + } + ``` + + > **Note** + > + > Nix will refetch the branch in accordance with + > the option `tarball-ttl`. + + > **Note** + > + > This behavior is disabled in *Pure evaluation mode*. + )", + .fun = prim_fetchGit, +}); } diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index b00827a4b..4c6682dfd 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -82,12 +82,12 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va visit(v, parser(tomlStream).parse()); } catch (std::runtime_error & e) { throw EvalError({ - .hint = hintfmt("while parsing a TOML string: %s", e.what()), + .msg = hintfmt("while parsing a TOML string: %s", e.what()), .errPos = pos }); } } -static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); +static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML); } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 7ba5e1c14..4eb6dac81 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -28,6 +28,12 @@ public: return s == s2.s; } + // FIXME: remove + bool operator == (std::string_view s2) const + { + return s->compare(s2) == 0; + } + bool operator != (const Symbol & s2) const { return s != s2.s; @@ -68,9 +74,10 @@ private: Symbols symbols; public: - Symbol create(const string & s) + Symbol create(std::string_view s) { - std::pair<Symbols::iterator, bool> res = symbols.insert(s); + // FIXME: avoid allocation if 's' already exists in the symbol table. + std::pair<Symbols::iterator, bool> res = symbols.emplace(std::string(s)); return Symbol(&*res.first); } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 6ec8315ba..bfea24d40 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -16,30 +16,30 @@ void printValueAsJSON(EvalState & state, bool strict, if (strict) state.forceValue(v); - switch (v.type) { + switch (v.type()) { - case tInt: + case nInt: out.write(v.integer); break; - case tBool: + case nBool: out.write(v.boolean); break; - case tString: + case nString: copyContext(v, context); out.write(v.string.s); break; - case tPath: + case nPath: out.write(state.copyPathToStore(context, v.path)); break; - case tNull: + case nNull: out.write(nullptr); break; - case tAttrs: { + case nAttrs: { auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); if (maybeString) { out.write(*maybeString); @@ -61,7 +61,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; } - case tList1: case tList2: case tListN: { + case nList: { auto list(out.list()); for (unsigned int n = 0; n < v.listSize(); ++n) { auto placeholder(list.placeholder()); @@ -70,15 +70,18 @@ void printValueAsJSON(EvalState & state, bool strict, break; } - case tExternal: + case nExternal: v.external->printValueAsJSON(state, strict, out, context); break; - case tFloat: + case nFloat: out.write(v.fpoint); break; - default: + case nThunk: + throw TypeError("cannot convert %1% to JSON", showType(v)); + + case nFunction: throw TypeError("cannot convert %1% to JSON", showType(v)); } } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 1f0b1541d..2ddc5f751 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -42,7 +42,7 @@ static void showAttrs(EvalState & state, bool strict, bool location, XMLAttrs xmlAttrs; xmlAttrs["name"] = i; - if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos); + if (location && a.pos != ptr(&noPos)) posToXML(xmlAttrs, *a.pos); XMLOpenElement _(doc, "attr", xmlAttrs); printValueAsXML(state, strict, location, @@ -58,31 +58,31 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (strict) state.forceValue(v); - switch (v.type) { + switch (v.type()) { - case tInt: + case nInt: doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); break; - case tBool: + case nBool: doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); break; - case tString: + case nString: /* !!! show the context? */ copyContext(v, context); doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); break; - case tPath: + case nPath: doc.writeEmptyElement("path", singletonAttrs("value", v.path)); break; - case tNull: + case nNull: doc.writeEmptyElement("null"); break; - case tAttrs: + case nAttrs: if (state.isDerivation(v)) { XMLAttrs xmlAttrs; @@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->type == tString) + if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->type == tString) + if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->string.s; } @@ -118,14 +118,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; - case tList1: case tList2: case tListN: { + case nList: { XMLOpenElement _(doc, "list"); for (unsigned int n = 0; n < v.listSize(); ++n) printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); break; } - case tLambda: { + case nFunction: { + if (!v.isLambda()) { + // FIXME: Serialize primops and primopapps + doc.writeEmptyElement("unevaluated"); + break; + } XMLAttrs xmlAttrs; if (location) posToXML(xmlAttrs, v.lambda.fun->pos); XMLOpenElement _(doc, "function", xmlAttrs); @@ -143,15 +148,15 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; } - case tExternal: + case nExternal: v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); break; - case tFloat: + case nFloat: doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); break; - default: + case nThunk: doc.writeEmptyElement("unevaluated"); } } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 71025824e..a1f131f9e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -27,8 +27,24 @@ typedef enum { tPrimOpApp, tExternal, tFloat -} ValueType; +} InternalType; +// This type abstracts over all actual value types in the language, +// grouping together implementation details like tList*, different function +// types, and types in non-normal form (so thunks and co.) +typedef enum { + nThunk, + nInt, + nFloat, + nBool, + nString, + nPath, + nNull, + nAttrs, + nList, + nFunction, + nExternal +} ValueType; class Bindings; struct Env; @@ -90,7 +106,28 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); struct Value { - ValueType type; +private: + InternalType internalType; + +friend std::string showType(const Value & v); +friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v); + +public: + + // Functions needed to distinguish the type + // These should be removed eventually, by putting the functionality that's + // needed by callers into methods of this type + + // type() == nThunk + inline bool isThunk() const { return internalType == tThunk; }; + inline bool isApp() const { return internalType == tApp; }; + inline bool isBlackhole() const { return internalType == tBlackhole; }; + + // type() == nFunction + inline bool isLambda() const { return internalType == tLambda; }; + inline bool isPrimOp() const { return internalType == tPrimOp; }; + inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; + union { NixInt integer; @@ -147,108 +184,210 @@ struct Value NixFloat fpoint; }; + // Returns the normal type of a Value. This only returns nThunk if the + // Value hasn't been forceValue'd + inline ValueType type() const + { + switch (internalType) { + case tInt: return nInt; + case tBool: return nBool; + case tString: return nString; + case tPath: return nPath; + case tNull: return nNull; + case tAttrs: return nAttrs; + case tList1: case tList2: case tListN: return nList; + case tLambda: case tPrimOp: case tPrimOpApp: return nFunction; + case tExternal: return nExternal; + case tFloat: return nFloat; + case tThunk: case tApp: case tBlackhole: return nThunk; + } + abort(); + } + + /* After overwriting an app node, be sure to clear pointers in the + Value to ensure that the target isn't kept alive unnecessarily. */ + inline void clearValue() + { + app.left = app.right = 0; + } + + inline void mkInt(NixInt n) + { + clearValue(); + internalType = tInt; + integer = n; + } + + inline void mkBool(bool b) + { + clearValue(); + internalType = tBool; + boolean = b; + } + + inline void mkString(const char * s, const char * * context = 0) + { + internalType = tString; + string.s = s; + string.context = context; + } + + inline void mkPath(const char * s) + { + clearValue(); + internalType = tPath; + path = s; + } + + inline void mkNull() + { + clearValue(); + internalType = tNull; + } + + inline void mkAttrs(Bindings * a) + { + clearValue(); + internalType = tAttrs; + attrs = a; + } + + inline void mkList(size_t size) + { + clearValue(); + if (size == 1) + internalType = tList1; + else if (size == 2) + internalType = tList2; + else { + internalType = tListN; + bigList.size = size; + } + } + + inline void mkThunk(Env * e, Expr * ex) + { + internalType = tThunk; + thunk.env = e; + thunk.expr = ex; + } + + inline void mkApp(Value * l, Value * r) + { + internalType = tApp; + app.left = l; + app.right = r; + } + + inline void mkLambda(Env * e, ExprLambda * f) + { + internalType = tLambda; + lambda.env = e; + lambda.fun = f; + } + + inline void mkBlackhole() + { + internalType = tBlackhole; + // Value will be overridden anyways + } + + inline void mkPrimOp(PrimOp * p) + { + clearValue(); + internalType = tPrimOp; + primOp = p; + } + + + inline void mkPrimOpApp(Value * l, Value * r) + { + internalType = tPrimOpApp; + app.left = l; + app.right = r; + } + + inline void mkExternal(ExternalValueBase * e) + { + clearValue(); + internalType = tExternal; + external = e; + } + + inline void mkFloat(NixFloat n) + { + clearValue(); + internalType = tFloat; + fpoint = n; + } + bool isList() const { - return type == tList1 || type == tList2 || type == tListN; + return internalType == tList1 || internalType == tList2 || internalType == tListN; } Value * * listElems() { - return type == tList1 || type == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } const Value * const * listElems() const { - return type == tList1 || type == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } size_t listSize() const { - return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; + return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size; } -}; + Pos determinePos(const Pos &pos) const; -/* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ -static inline void clearValue(Value & v) -{ - v.app.left = v.app.right = 0; -} + /* Check whether forcing this value requires a trivial amount of + computation. In particular, function applications are + non-trivial. */ + bool isTrivial() const; + + std::vector<std::pair<Path, std::string>> getContext(); +}; + +// TODO: Remove these static functions, replace call sites with v.mk* instead static inline void mkInt(Value & v, NixInt n) { - clearValue(v); - v.type = tInt; - v.integer = n; + v.mkInt(n); } - static inline void mkFloat(Value & v, NixFloat n) { - clearValue(v); - v.type = tFloat; - v.fpoint = n; + v.mkFloat(n); } - static inline void mkBool(Value & v, bool b) { - clearValue(v); - v.type = tBool; - v.boolean = b; + v.mkBool(b); } - static inline void mkNull(Value & v) { - clearValue(v); - v.type = tNull; + v.mkNull(); } - static inline void mkApp(Value & v, Value & left, Value & right) { - v.type = tApp; - v.app.left = &left; - v.app.right = &right; + v.mkApp(&left, &right); } - -static inline void mkPrimOpApp(Value & v, Value & left, Value & right) -{ - v.type = tPrimOpApp; - v.app.left = &left; - v.app.right = &right; -} - - -static inline void mkStringNoCopy(Value & v, const char * s) -{ - v.type = tString; - v.string.s = s; - v.string.context = 0; -} - - static inline void mkString(Value & v, const Symbol & s) { - mkStringNoCopy(v, ((const string &) s).c_str()); + v.mkString(((const string &) s).c_str()); } void mkString(Value & v, const char * s); -static inline void mkPathNoCopy(Value & v, const char * s) -{ - clearValue(v); - v.type = tPath; - v.path = s; -} - - void mkPath(Value & v, const char * s); |