diff options
author | John Ericson <John.Ericson@Obsidian.Systems> | 2021-10-08 23:59:15 +0000 |
---|---|---|
committer | John Ericson <John.Ericson@Obsidian.Systems> | 2021-10-08 23:59:15 +0000 |
commit | 195daa82995b43b3cbd552735a678afb85f4ae28 (patch) | |
tree | fb15fcc7e12b0ac5e0abb37cea947fd014290be9 /src/libexpr | |
parent | edf67e1508523593cf549a579e8dbcc2e89c8004 (diff) | |
parent | 01e9f046a8f8fafb4d084153d4b30dd3a8d7aef5 (diff) |
Merge remote-tracking branch 'upstream/master' into ca-drv-exotic
Diffstat (limited to 'src/libexpr')
-rw-r--r-- | src/libexpr/eval.cc | 25 | ||||
-rw-r--r-- | src/libexpr/eval.hh | 9 | ||||
-rw-r--r-- | src/libexpr/flake/flake.cc | 5 | ||||
-rw-r--r-- | src/libexpr/nixexpr.cc | 4 | ||||
-rw-r--r-- | src/libexpr/nixexpr.hh | 6 | ||||
-rw-r--r-- | src/libexpr/parser.y | 8 | ||||
-rw-r--r-- | src/libexpr/primops.cc | 160 | ||||
-rw-r--r-- | src/libexpr/primops/fetchMercurial.cc | 3 | ||||
-rw-r--r-- | src/libexpr/primops/fetchTree.cc | 72 | ||||
-rw-r--r-- | src/libexpr/value-to-xml.cc | 2 |
10 files changed, 170 insertions, 124 deletions
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 800839a8d..3ae05a8d8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -445,12 +445,12 @@ EvalState::EvalState( StorePathSet closure; store->computeFSClosure(store->toStorePath(r.second).first, closure); for (auto & path : closure) - allowedPaths->insert(store->printStorePath(path)); + allowPath(path); } catch (InvalidPath &) { - allowedPaths->insert(r.second); + allowPath(r.second); } } else - allowedPaths->insert(r.second); + allowPath(r.second); } } @@ -482,6 +482,18 @@ void EvalState::requireExperimentalFeatureOnEvaluation( } } +void EvalState::allowPath(const Path & path) +{ + if (allowedPaths) + allowedPaths->insert(path); +} + +void EvalState::allowPath(const StorePath & storePath) +{ + if (allowedPaths) + allowedPaths->insert(store->toRealPath(storePath)); +} + Path EvalState::checkSourcePath(const Path & path_) { if (!allowedPaths) return path_; @@ -1316,13 +1328,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po auto size = (lambda.arg.empty() ? 0 : 1) + - (lambda.matchAttrs ? lambda.formals->formals.size() : 0); + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); Env & env2(allocEnv(size)); env2.up = fun.lambda.env; size_t displ = 0; - if (!lambda.matchAttrs) + if (!lambda.hasFormals()) env2.values[displ++] = &arg; else { @@ -1402,7 +1414,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } } - if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) { + if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) { res = fun; return; } @@ -1889,6 +1901,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); dstPath = store->printStorePath(p); + allowPath(p); srcToStore.insert_or_assign(path, std::move(p)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9df6150c6..7cc16ef0a 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -150,6 +150,15 @@ public: SearchPath getSearchPath() { return searchPath; } + /* Allow access to a path. */ + void allowPath(const Path & path); + + /* Allow access to a store path. Note that this gets remapped to + the real store path if `store` is a chroot store. */ + void allowPath(const StorePath & storePath); + + /* Check whether access to a path is allowed and throw an error if + not. Otherwise return the canonicalised path. */ Path checkSourcePath(const Path & path); void checkURI(const std::string & uri); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 1a1fa6938..43bfc3644 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -64,8 +64,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( debug("got tree '%s' from '%s'", state.store->printStorePath(tree.storePath), lockedRef); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); @@ -231,7 +230,7 @@ static Flake getFlake( if (auto outputs = vInfo.attrs->get(sOutputs)) { expectType(state, nFunction, *outputs->value, *outputs->pos); - if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) { + if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) { for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 492b819e7..0d0f3e469 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -124,7 +124,7 @@ void ExprList::show(std::ostream & str) const void ExprLambda::show(std::ostream & str) const { str << "("; - if (matchAttrs) { + if (hasFormals()) { str << "{ "; bool first = true; for (auto & i : formals->formals) { @@ -348,7 +348,7 @@ void ExprLambda::bindVars(const StaticEnv & env) if (!arg.empty()) newEnv.vars[arg] = displ++; - if (matchAttrs) { + if (hasFormals()) { for (auto & i : formals->formals) newEnv.vars[i.name] = displ++; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 51a14cd59..851e875bd 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -233,11 +233,10 @@ struct ExprLambda : Expr Pos pos; Symbol name; Symbol arg; - bool matchAttrs; Formals * formals; Expr * body; - ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body) - : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) + ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body) + : pos(pos), arg(arg), formals(formals), body(body) { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) throw ParseError({ @@ -247,6 +246,7 @@ struct ExprLambda : Expr }; void setName(Symbol & name); string showNamePos() const; + inline bool hasFormals() const { return formals != nullptr; } COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e3749783a..8a0a79c96 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -324,13 +324,13 @@ expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), $2, $5); } | '{' formals '}' '@' ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), $2, $7); } | ID '@' '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), $4, $7); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } | WITH expr ';' expr_function diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 25b9c32b2..8f6e937ec 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -56,13 +56,9 @@ void EvalState::realiseContext(const PathSet & context) "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", store->printStorePath(drvs.begin()->drvPath)); - /* For performance, prefetch all substitute info. */ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; + /* Build/substitute the context. */ std::vector<DerivedPath> buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); - store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(buildReqs); /* Add the output of this derivations to the allowed @@ -414,7 +410,7 @@ static RegisterPrimOp primop_isNull({ Return `true` if *e* evaluates to `null`, and `false` otherwise. > **Warning** - > + > > This function is *deprecated*; just write `e == null` instead. )", .fun = prim_isNull, @@ -1851,56 +1847,87 @@ static RegisterPrimOp primop_toFile({ .fun = prim_toFile, }); -static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v) +static void addPath( + EvalState & state, + const Pos & pos, + const string & name, + Path path, + Value * filterFun, + FileIngestionMethod method, + const std::optional<Hash> expectedHash, + Value & v, + const PathSet & context) { - const auto path = evalSettings.pureEval && expectedHash ? - path_ : - state.checkSourcePath(path_); - PathFilter filter = filterFun ? ([&](const Path & path) { - auto st = lstat(path); + try { + // FIXME: handle CA derivation outputs (where path needs to + // be rewritten to the actual output). + state.realiseContext(context); - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - mkString(arg1, path); + if (state.store->isInStore(path)) { + auto [storePath, subPath] = state.store->toStorePath(path); + auto info = state.store->queryPathInfo(storePath); + if (!info->references.empty()) + throw EvalError("store path '%s' is not allowed to have references", + state.store->printStorePath(storePath)); + path = state.store->toRealPath(storePath) + subPath; + } - Value fun2; - state.callFunction(*filterFun, arg1, fun2, noPos); + path = evalSettings.pureEval && expectedHash + ? path + : state.checkSourcePath(path); - Value arg2; - mkString(arg2, - S_ISREG(st.st_mode) ? "regular" : - S_ISDIR(st.st_mode) ? "directory" : - S_ISLNK(st.st_mode) ? "symlink" : - "unknown" /* not supported, will fail! */); + PathFilter filter = filterFun ? ([&](const Path & path) { + auto st = lstat(path); - Value res; - state.callFunction(fun2, arg2, res, noPos); + /* Call the filter function. The first argument is the path, + the second is a string indicating the type of the file. */ + Value arg1; + mkString(arg1, path); - return state.forceBool(res, pos); - }) : defaultPathFilter; + Value fun2; + state.callFunction(*filterFun, arg1, fun2, noPos); - std::optional<StorePath> expectedStorePath; - if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - { - .method = method, - .hash = *expectedHash, - }, - {}, - }); - Path dstPath; - if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - dstPath = state.store->printStorePath(settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); - if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) - throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); - } else - dstPath = state.store->printStorePath(*expectedStorePath); + Value arg2; + mkString(arg2, + S_ISREG(st.st_mode) ? "regular" : + S_ISDIR(st.st_mode) ? "directory" : + S_ISLNK(st.st_mode) ? "symlink" : + "unknown" /* not supported, will fail! */); + + Value res; + state.callFunction(fun2, arg2, res, noPos); + + return state.forceBool(res, pos); + }) : defaultPathFilter; + + std::optional<StorePath> expectedStorePath; + if (expectedHash) + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); + + Path dstPath; + if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { + dstPath = state.store->printStorePath(settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); + if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) + throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); + } else + dstPath = state.store->printStorePath(*expectedStorePath); - mkString(v, dstPath, {dstPath}); + mkString(v, dstPath, {dstPath}); + + state.allowPath(dstPath); + + } catch (Error & e) { + e.addTrace(pos, "while adding path '%s'", path); + throw; + } } @@ -1908,11 +1935,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args { PathSet context; Path path = state.coerceToPath(pos, *args[1], context); - if (!context.empty()) - throw EvalError({ - .msg = hintfmt("string '%1%' cannot refer to other paths", path), - .errPos = pos - }); state.forceValue(*args[0], pos); if (args[0]->type() != nFunction) @@ -1923,13 +1945,26 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ .name = "__filterSource", .args = {"e1", "e2"}, .doc = R"( + > **Warning** + > + > `filterSource` should not be used to filter store paths. Since + > `filterSource` uses the name of the input directory while naming + > the output directory, doing so will produce a directory name in + > the form of `<hash2>-<hash>-<name>`, where `<hash>-<name>` is + > the name of the input directory. Since `<hash>` depends on the + > unfiltered directory, the name of the output directory will + > indirectly depend on files that are filtered out by the + > function. This will trigger a rebuild even when a filtered out + > file is changed. Use `builtins.path` instead, which allows + > specifying the name of the output directory. + This function allows you to copy sources into the Nix store while filtering certain files. For instance, suppose that you want to use the directory `source-dir` as an input to a Nix expression, e.g. @@ -1976,18 +2011,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; std::optional<Hash> expectedHash; + PathSet context; for (auto & attr : *args[0]->attrs) { const string & n(attr.name); - if (n == "path") { - PathSet context; + if (n == "path") path = state.coerceToPath(*attr.pos, *attr.value, context); - if (!context.empty()) - throw EvalError({ - .msg = hintfmt("string '%1%' cannot refer to other paths", path), - .errPos = *attr.pos - }); - } else if (attr.name == state.sName) + else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { state.forceValue(*attr.value, pos); @@ -2010,7 +2040,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value if (name.empty()) name = baseNameOf(path); - addPath(state, pos, name, path, filterFun, method, expectedHash, v); + addPath(state, pos, name, path, filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ @@ -2375,7 +2405,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - if (!args[0]->lambda.fun->matchAttrs) { + if (!args[0]->lambda.fun->hasFormals()) { state.mkAttrs(v, 0); return; } @@ -2530,7 +2560,7 @@ static RegisterPrimOp primop_tail({ the argument isn’t a list or is an empty list. > **Warning** - > + > > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 3f88ccb91..1cd481243 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -84,8 +84,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); v.attrs->sort(); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); } static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 06bdec003..f570f19ae 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -66,7 +66,7 @@ void emitTreeAttrs( v.attrs->sort(); } -std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file") +std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") { state.checkURI(uri); return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; @@ -81,23 +81,17 @@ std::string fixURIForGit(std::string uri, EvalState & state) return fixURI(uri, state); } -void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) -{ - string n(name); - attrs.emplace(name, n == "url" ? fixURI(v, state) : v); -} - struct FetchTreeParams { bool emptyRevFallback = false; bool allowNameArgument = false; }; static void fetchTree( - EvalState &state, - const Pos &pos, - Value **args, - Value &v, - const std::optional<std::string> type, + EvalState & state, + const Pos & pos, + Value * * args, + Value & v, + std::optional<std::string> type, const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; @@ -110,17 +104,33 @@ static void fetchTree( fetchers::Attrs attrs; + if (auto aType = args[0]->attrs->get(state.sType)) { + if (type) + throw Error({ + .msg = hintfmt("unexpected attribute 'type'"), + .errPos = pos + }); + type = state.forceStringNoCtx(*aType->value, *aType->pos); + } else if (!type) + throw Error({ + .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .errPos = pos + }); + + attrs.emplace("type", type.value()); + for (auto & attr : *args[0]->attrs) { + if (attr.name == state.sType) continue; state.forceValue(*attr.value); - if (attr.value->type() == nPath || attr.value->type() == nString) - addURI( - state, - attrs, - attr.name, - state.coerceToString(*attr.pos, *attr.value, context, false, false) - ); - else if (attr.value->type() == nString) - addURI(state, attrs, attr.name, attr.value->string.s); + if (attr.value->type() == nPath || attr.value->type() == nString) { + auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false); + attrs.emplace(attr.name, + attr.name == "url" + ? type == "git" + ? fixURIForGit(s, state) + : fixURI(s, state) + : s); + } else if (attr.value->type() == nBool) attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean}); else if (attr.value->type() == nInt) @@ -130,15 +140,6 @@ static void fetchTree( attr.name, showType(*attr.value)); } - if (type) - attrs.emplace("type", type.value()); - - if (!attrs.count("type")) - throw Error({ - .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), - .errPos = pos - }); - if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) throw Error({ @@ -146,7 +147,6 @@ static void fetchTree( .errPos = pos }); - input = fetchers::Input::fromAttrs(std::move(attrs)); } else { auto url = state.coerceToString(pos, *args[0], context, false, false); @@ -169,8 +169,7 @@ static void fetchTree( auto [tree, input2] = input.fetch(state.store); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); } @@ -234,19 +233,16 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; - auto realPath = state.store->toRealPath(storePath); - if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, realPath); + : hashFile(htSHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } - if (state.allowedPaths) - state.allowedPaths->insert(realPath); + state.allowPath(storePath); auto path = state.store->printStorePath(storePath); mkString(v, path, PathSet({path})); diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 2ddc5f751..b44455f5f 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -135,7 +135,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (location) posToXML(xmlAttrs, v.lambda.fun->pos); XMLOpenElement _(doc, "function", xmlAttrs); - if (v.lambda.fun->matchAttrs) { + if (v.lambda.fun->hasFormals()) { XMLAttrs attrs; if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; |