diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/libexpr/eval.cc | 25 | ||||
-rw-r--r-- | src/libexpr/eval.hh | 9 | ||||
-rw-r--r-- | src/libexpr/flake/flake.cc | 5 | ||||
-rw-r--r-- | src/libexpr/nixexpr.cc | 4 | ||||
-rw-r--r-- | src/libexpr/nixexpr.hh | 6 | ||||
-rw-r--r-- | src/libexpr/parser.y | 8 | ||||
-rw-r--r-- | src/libexpr/primops.cc | 160 | ||||
-rw-r--r-- | src/libexpr/primops/fetchMercurial.cc | 3 | ||||
-rw-r--r-- | src/libexpr/primops/fetchTree.cc | 72 | ||||
-rw-r--r-- | src/libexpr/value-to-xml.cc | 2 | ||||
-rw-r--r-- | src/libstore/build/local-derivation-goal.cc | 14 | ||||
-rw-r--r-- | src/libstore/build/worker.cc | 2 | ||||
-rw-r--r-- | src/libstore/references.cc | 93 | ||||
-rw-r--r-- | src/libstore/references.hh | 24 | ||||
-rw-r--r-- | src/libstore/tests/local.mk | 15 | ||||
-rw-r--r-- | src/libstore/tests/references.cc | 45 | ||||
-rw-r--r-- | src/libstore/uds-remote-store.cc | 11 | ||||
-rw-r--r-- | src/libutil/archive.cc | 4 | ||||
-rw-r--r-- | src/libutil/util.cc | 81 | ||||
-rw-r--r-- | src/libutil/util.hh | 10 | ||||
-rw-r--r-- | src/nix/daemon.cc | 6 | ||||
-rw-r--r-- | src/nix/flake.cc | 6 | ||||
-rw-r--r-- | src/nix/flake.md | 2 | ||||
-rw-r--r-- | src/nix/main.cc | 7 | ||||
-rw-r--r-- | src/nix/profile.cc | 6 |
25 files changed, 389 insertions, 231 deletions
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 800839a8d..3ae05a8d8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -445,12 +445,12 @@ EvalState::EvalState( StorePathSet closure; store->computeFSClosure(store->toStorePath(r.second).first, closure); for (auto & path : closure) - allowedPaths->insert(store->printStorePath(path)); + allowPath(path); } catch (InvalidPath &) { - allowedPaths->insert(r.second); + allowPath(r.second); } } else - allowedPaths->insert(r.second); + allowPath(r.second); } } @@ -482,6 +482,18 @@ void EvalState::requireExperimentalFeatureOnEvaluation( } } +void EvalState::allowPath(const Path & path) +{ + if (allowedPaths) + allowedPaths->insert(path); +} + +void EvalState::allowPath(const StorePath & storePath) +{ + if (allowedPaths) + allowedPaths->insert(store->toRealPath(storePath)); +} + Path EvalState::checkSourcePath(const Path & path_) { if (!allowedPaths) return path_; @@ -1316,13 +1328,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po auto size = (lambda.arg.empty() ? 0 : 1) + - (lambda.matchAttrs ? lambda.formals->formals.size() : 0); + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); Env & env2(allocEnv(size)); env2.up = fun.lambda.env; size_t displ = 0; - if (!lambda.matchAttrs) + if (!lambda.hasFormals()) env2.values[displ++] = &arg; else { @@ -1402,7 +1414,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } } - if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) { + if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) { res = fun; return; } @@ -1889,6 +1901,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path) ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); dstPath = store->printStorePath(p); + allowPath(p); srcToStore.insert_or_assign(path, std::move(p)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9df6150c6..7cc16ef0a 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -150,6 +150,15 @@ public: SearchPath getSearchPath() { return searchPath; } + /* Allow access to a path. */ + void allowPath(const Path & path); + + /* Allow access to a store path. Note that this gets remapped to + the real store path if `store` is a chroot store. */ + void allowPath(const StorePath & storePath); + + /* Check whether access to a path is allowed and throw an error if + not. Otherwise return the canonicalised path. */ Path checkSourcePath(const Path & path); void checkURI(const std::string & uri); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 1a1fa6938..43bfc3644 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -64,8 +64,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( debug("got tree '%s' from '%s'", state.store->printStorePath(tree.storePath), lockedRef); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); @@ -231,7 +230,7 @@ static Flake getFlake( if (auto outputs = vInfo.attrs->get(sOutputs)) { expectType(state, nFunction, *outputs->value, *outputs->pos); - if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) { + if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) { for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 492b819e7..0d0f3e469 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -124,7 +124,7 @@ void ExprList::show(std::ostream & str) const void ExprLambda::show(std::ostream & str) const { str << "("; - if (matchAttrs) { + if (hasFormals()) { str << "{ "; bool first = true; for (auto & i : formals->formals) { @@ -348,7 +348,7 @@ void ExprLambda::bindVars(const StaticEnv & env) if (!arg.empty()) newEnv.vars[arg] = displ++; - if (matchAttrs) { + if (hasFormals()) { for (auto & i : formals->formals) newEnv.vars[i.name] = displ++; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 51a14cd59..851e875bd 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -233,11 +233,10 @@ struct ExprLambda : Expr Pos pos; Symbol name; Symbol arg; - bool matchAttrs; Formals * formals; Expr * body; - ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body) - : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) + ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body) + : pos(pos), arg(arg), formals(formals), body(body) { if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) throw ParseError({ @@ -247,6 +246,7 @@ struct ExprLambda : Expr }; void setName(Symbol & name); string showNamePos() const; + inline bool hasFormals() const { return formals != nullptr; } COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e3749783a..8a0a79c96 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -324,13 +324,13 @@ expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); } | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), $2, $5); } | '{' formals '}' '@' ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), $2, $7); } | ID '@' '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); } + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), $4, $7); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } | WITH expr ';' expr_function diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 25b9c32b2..8f6e937ec 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -56,13 +56,9 @@ void EvalState::realiseContext(const PathSet & context) "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", store->printStorePath(drvs.begin()->drvPath)); - /* For performance, prefetch all substitute info. */ - StorePathSet willBuild, willSubstitute, unknown; - uint64_t downloadSize, narSize; + /* Build/substitute the context. */ std::vector<DerivedPath> buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); - store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize); - store->buildPaths(buildReqs); /* Add the output of this derivations to the allowed @@ -414,7 +410,7 @@ static RegisterPrimOp primop_isNull({ Return `true` if *e* evaluates to `null`, and `false` otherwise. > **Warning** - > + > > This function is *deprecated*; just write `e == null` instead. )", .fun = prim_isNull, @@ -1851,56 +1847,87 @@ static RegisterPrimOp primop_toFile({ .fun = prim_toFile, }); -static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, - Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v) +static void addPath( + EvalState & state, + const Pos & pos, + const string & name, + Path path, + Value * filterFun, + FileIngestionMethod method, + const std::optional<Hash> expectedHash, + Value & v, + const PathSet & context) { - const auto path = evalSettings.pureEval && expectedHash ? - path_ : - state.checkSourcePath(path_); - PathFilter filter = filterFun ? ([&](const Path & path) { - auto st = lstat(path); + try { + // FIXME: handle CA derivation outputs (where path needs to + // be rewritten to the actual output). + state.realiseContext(context); - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - mkString(arg1, path); + if (state.store->isInStore(path)) { + auto [storePath, subPath] = state.store->toStorePath(path); + auto info = state.store->queryPathInfo(storePath); + if (!info->references.empty()) + throw EvalError("store path '%s' is not allowed to have references", + state.store->printStorePath(storePath)); + path = state.store->toRealPath(storePath) + subPath; + } - Value fun2; - state.callFunction(*filterFun, arg1, fun2, noPos); + path = evalSettings.pureEval && expectedHash + ? path + : state.checkSourcePath(path); - Value arg2; - mkString(arg2, - S_ISREG(st.st_mode) ? "regular" : - S_ISDIR(st.st_mode) ? "directory" : - S_ISLNK(st.st_mode) ? "symlink" : - "unknown" /* not supported, will fail! */); + PathFilter filter = filterFun ? ([&](const Path & path) { + auto st = lstat(path); - Value res; - state.callFunction(fun2, arg2, res, noPos); + /* Call the filter function. The first argument is the path, + the second is a string indicating the type of the file. */ + Value arg1; + mkString(arg1, path); - return state.forceBool(res, pos); - }) : defaultPathFilter; + Value fun2; + state.callFunction(*filterFun, arg1, fun2, noPos); - std::optional<StorePath> expectedStorePath; - if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - { - .method = method, - .hash = *expectedHash, - }, - {}, - }); - Path dstPath; - if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - dstPath = state.store->printStorePath(settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); - if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) - throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); - } else - dstPath = state.store->printStorePath(*expectedStorePath); + Value arg2; + mkString(arg2, + S_ISREG(st.st_mode) ? "regular" : + S_ISDIR(st.st_mode) ? "directory" : + S_ISLNK(st.st_mode) ? "symlink" : + "unknown" /* not supported, will fail! */); + + Value res; + state.callFunction(fun2, arg2, res, noPos); + + return state.forceBool(res, pos); + }) : defaultPathFilter; + + std::optional<StorePath> expectedStorePath; + if (expectedHash) + expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { + { + .method = method, + .hash = *expectedHash, + }, + {}, + }); + + Path dstPath; + if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { + dstPath = state.store->printStorePath(settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); + if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) + throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); + } else + dstPath = state.store->printStorePath(*expectedStorePath); - mkString(v, dstPath, {dstPath}); + mkString(v, dstPath, {dstPath}); + + state.allowPath(dstPath); + + } catch (Error & e) { + e.addTrace(pos, "while adding path '%s'", path); + throw; + } } @@ -1908,11 +1935,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args { PathSet context; Path path = state.coerceToPath(pos, *args[1], context); - if (!context.empty()) - throw EvalError({ - .msg = hintfmt("string '%1%' cannot refer to other paths", path), - .errPos = pos - }); state.forceValue(*args[0], pos); if (args[0]->type() != nFunction) @@ -1923,13 +1945,26 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v); + addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ .name = "__filterSource", .args = {"e1", "e2"}, .doc = R"( + > **Warning** + > + > `filterSource` should not be used to filter store paths. Since + > `filterSource` uses the name of the input directory while naming + > the output directory, doing so will produce a directory name in + > the form of `<hash2>-<hash>-<name>`, where `<hash>-<name>` is + > the name of the input directory. Since `<hash>` depends on the + > unfiltered directory, the name of the output directory will + > indirectly depend on files that are filtered out by the + > function. This will trigger a rebuild even when a filtered out + > file is changed. Use `builtins.path` instead, which allows + > specifying the name of the output directory. + This function allows you to copy sources into the Nix store while filtering certain files. For instance, suppose that you want to use the directory `source-dir` as an input to a Nix expression, e.g. @@ -1976,18 +2011,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; std::optional<Hash> expectedHash; + PathSet context; for (auto & attr : *args[0]->attrs) { const string & n(attr.name); - if (n == "path") { - PathSet context; + if (n == "path") path = state.coerceToPath(*attr.pos, *attr.value, context); - if (!context.empty()) - throw EvalError({ - .msg = hintfmt("string '%1%' cannot refer to other paths", path), - .errPos = *attr.pos - }); - } else if (attr.name == state.sName) + else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, *attr.pos); else if (n == "filter") { state.forceValue(*attr.value, pos); @@ -2010,7 +2040,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value if (name.empty()) name = baseNameOf(path); - addPath(state, pos, name, path, filterFun, method, expectedHash, v); + addPath(state, pos, name, path, filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ @@ -2375,7 +2405,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args .errPos = pos }); - if (!args[0]->lambda.fun->matchAttrs) { + if (!args[0]->lambda.fun->hasFormals()) { state.mkAttrs(v, 0); return; } @@ -2530,7 +2560,7 @@ static RegisterPrimOp primop_tail({ the argument isn’t a list or is an empty list. > **Warning** - > + > > This function should generally be avoided since it's inefficient: > unlike Haskell's `tail`, it takes O(n) time, so recursing over a > list by repeatedly calling `tail` takes O(n^2) time. diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 3f88ccb91..1cd481243 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -84,8 +84,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount); v.attrs->sort(); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); } static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 06bdec003..f570f19ae 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -66,7 +66,7 @@ void emitTreeAttrs( v.attrs->sort(); } -std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file") +std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") { state.checkURI(uri); return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; @@ -81,23 +81,17 @@ std::string fixURIForGit(std::string uri, EvalState & state) return fixURI(uri, state); } -void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) -{ - string n(name); - attrs.emplace(name, n == "url" ? fixURI(v, state) : v); -} - struct FetchTreeParams { bool emptyRevFallback = false; bool allowNameArgument = false; }; static void fetchTree( - EvalState &state, - const Pos &pos, - Value **args, - Value &v, - const std::optional<std::string> type, + EvalState & state, + const Pos & pos, + Value * * args, + Value & v, + std::optional<std::string> type, const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; @@ -110,17 +104,33 @@ static void fetchTree( fetchers::Attrs attrs; + if (auto aType = args[0]->attrs->get(state.sType)) { + if (type) + throw Error({ + .msg = hintfmt("unexpected attribute 'type'"), + .errPos = pos + }); + type = state.forceStringNoCtx(*aType->value, *aType->pos); + } else if (!type) + throw Error({ + .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), + .errPos = pos + }); + + attrs.emplace("type", type.value()); + for (auto & attr : *args[0]->attrs) { + if (attr.name == state.sType) continue; state.forceValue(*attr.value); - if (attr.value->type() == nPath || attr.value->type() == nString) - addURI( - state, - attrs, - attr.name, - state.coerceToString(*attr.pos, *attr.value, context, false, false) - ); - else if (attr.value->type() == nString) - addURI(state, attrs, attr.name, attr.value->string.s); + if (attr.value->type() == nPath || attr.value->type() == nString) { + auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false); + attrs.emplace(attr.name, + attr.name == "url" + ? type == "git" + ? fixURIForGit(s, state) + : fixURI(s, state) + : s); + } else if (attr.value->type() == nBool) attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean}); else if (attr.value->type() == nInt) @@ -130,15 +140,6 @@ static void fetchTree( attr.name, showType(*attr.value)); } - if (type) - attrs.emplace("type", type.value()); - - if (!attrs.count("type")) - throw Error({ - .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), - .errPos = pos - }); - if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) throw Error({ @@ -146,7 +147,6 @@ static void fetchTree( .errPos = pos }); - input = fetchers::Input::fromAttrs(std::move(attrs)); } else { auto url = state.coerceToString(pos, *args[0], context, false, false); @@ -169,8 +169,7 @@ static void fetchTree( auto [tree, input2] = input.fetch(state.store); - if (state.allowedPaths) - state.allowedPaths->insert(tree.actualPath); + state.allowPath(tree.storePath); emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); } @@ -234,19 +233,16 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; - auto realPath = state.store->toRealPath(storePath); - if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, realPath); + : hashFile(htSHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)); } - if (state.allowedPaths) - state.allowedPaths->insert(realPath); + state.allowPath(storePath); auto path = state.store->printStorePath(storePath); mkString(v, path, PathSet({path})); diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 2ddc5f751..b44455f5f 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -135,7 +135,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (location) posToXML(xmlAttrs, v.lambda.fun->pos); XMLOpenElement _(doc, "function", xmlAttrs); - if (v.lambda.fun->matchAttrs) { + if (v.lambda.fun->hasFormals()) { XMLAttrs attrs; if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index f574e64cf..e739f4d9d 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -711,6 +711,7 @@ void LocalDerivationGoal::startBuilder() if (!builderOut.readSide) throw SysError("opening pseudoterminal master"); + // FIXME: not thread-safe, use ptsname_r std::string slaveName(ptsname(builderOut.readSide.get())); if (buildUser) { @@ -754,7 +755,6 @@ void LocalDerivationGoal::startBuilder() result.startTime = time(0); /* Fork a child to build the package. */ - ProcessOptions options; #if __linux__ if (useChroot) { @@ -797,8 +797,6 @@ void LocalDerivationGoal::startBuilder() userNamespaceSync.create(); - options.allowVfork = false; - Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces"; static bool userNamespacesEnabled = pathExists(maxUserNamespaces) @@ -856,7 +854,7 @@ void LocalDerivationGoal::startBuilder() writeFull(builderOut.writeSide.get(), fmt("%d %d\n", usingUserNamespace, child)); _exit(0); - }, options); + }); int res = helper.wait(); if (res != 0 && settings.sandboxFallback) { @@ -920,10 +918,9 @@ void LocalDerivationGoal::startBuilder() #endif { fallback: - options.allowVfork = !buildUser && !drv->isBuiltin(); pid = startProcess([&]() { runChild(); - }, options); + }); } /* parent */ @@ -2140,8 +2137,7 @@ void LocalDerivationGoal::registerOutputs() /* Pass blank Sink as we are not ready to hash data at this stage. */ NullSink blank; - auto references = worker.store.parseStorePathSet( - scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths))); + auto references = scanForReferences(blank, actualPath, referenceablePaths); outputReferencesIfUnregistered.insert_or_assign( outputName, @@ -2208,7 +2204,7 @@ void LocalDerivationGoal::registerOutputs() auto rewriteOutput = [&]() { /* Apply hash rewriting if necessary. */ if (!outputRewrites.empty()) { - warn("rewriting hashes in '%1%'; cross fingers", actualPath); + debug("rewriting hashes in '%1%'; cross fingers", actualPath); /* FIXME: this is in-memory. */ StringSink sink; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index a7a6b92a6..55afb5cca 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -239,7 +239,7 @@ void Worker::run(const Goals & _topGoals) } } - /* Call queryMissing() efficiently query substitutes. */ + /* Call queryMissing() to efficiently query substitutes. */ StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index 3a07c1411..c369b14ac 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -11,11 +11,13 @@ namespace nix { -static unsigned int refLength = 32; /* characters */ +static size_t refLength = 32; /* characters */ -static void search(const unsigned char * s, size_t len, - StringSet & hashes, StringSet & seen) +static void search( + std::string_view s, + StringSet & hashes, + StringSet & seen) { static std::once_flag initialised; static bool isBase32[256]; @@ -25,7 +27,7 @@ static void search(const unsigned char * s, size_t len, isBase32[(unsigned char) base32Chars[i]] = true; }); - for (size_t i = 0; i + refLength <= len; ) { + for (size_t i = 0; i + refLength <= s.size(); ) { int j; bool match = true; for (j = refLength - 1; j >= 0; --j) @@ -35,7 +37,7 @@ static void search(const unsigned char * s, size_t len, break; } if (!match) continue; - string ref((const char *) s + i, refLength); + std::string ref(s.substr(i, refLength)); if (hashes.erase(ref)) { debug(format("found reference to '%1%' at offset '%2%'") % ref % i); @@ -46,69 +48,60 @@ static void search(const unsigned char * s, size_t len, } -struct RefScanSink : Sink +void RefScanSink::operator () (std::string_view data) { - StringSet hashes; - StringSet seen; - - string tail; - - RefScanSink() { } - - void operator () (std::string_view data) override - { - /* It's possible that a reference spans the previous and current - fragment, so search in the concatenation of the tail of the - previous fragment and the start of the current fragment. */ - string s = tail + std::string(data, 0, refLength); - search((const unsigned char *) s.data(), s.size(), hashes, seen); - - search((const unsigned char *) data.data(), data.size(), hashes, seen); - - size_t tailLen = data.size() <= refLength ? data.size() : refLength; - tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)); - tail.append({data.data() + data.size() - tailLen, tailLen}); - } -}; + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + auto s = tail; + s.append(data.data(), refLength); + search(s, hashes, seen); + + search(data, hashes, seen); + + auto tailLen = std::min(data.size(), refLength); + auto rest = refLength - tailLen; + if (rest < tail.size()) + tail = tail.substr(tail.size() - rest); + tail.append(data.data() + data.size() - tailLen, tailLen); +} -std::pair<PathSet, HashResult> scanForReferences(const string & path, - const PathSet & refs) +std::pair<StorePathSet, HashResult> scanForReferences( + const string & path, + const StorePathSet & refs) { HashSink hashSink { htSHA256 }; auto found = scanForReferences(hashSink, path, refs); auto hash = hashSink.finish(); - return std::pair<PathSet, HashResult>(found, hash); + return std::pair<StorePathSet, HashResult>(found, hash); } -PathSet scanForReferences(Sink & toTee, - const string & path, const PathSet & refs) +StorePathSet scanForReferences( + Sink & toTee, + const Path & path, + const StorePathSet & refs) { - RefScanSink refsSink; - TeeSink sink { refsSink, toTee }; - std::map<string, Path> backMap; + StringSet hashes; + std::map<std::string, StorePath> backMap; for (auto & i : refs) { - auto baseName = std::string(baseNameOf(i)); - string::size_type pos = baseName.find('-'); - if (pos == string::npos) - throw Error("bad reference '%1%'", i); - string s = string(baseName, 0, pos); - assert(s.size() == refLength); - assert(backMap.find(s) == backMap.end()); - // parseHash(htSHA256, s); - refsSink.hashes.insert(s); - backMap[s] = i; + std::string hashPart(i.hashPart()); + auto inserted = backMap.emplace(hashPart, i).second; + assert(inserted); + hashes.insert(hashPart); } /* Look for the hashes in the NAR dump of the path. */ + RefScanSink refsSink(std::move(hashes)); + TeeSink sink { refsSink, toTee }; dumpPath(path, sink); /* Map the hashes found back to their store paths. */ - PathSet found; - for (auto & i : refsSink.seen) { - std::map<string, Path>::iterator j; - if ((j = backMap.find(i)) == backMap.end()) abort(); + StorePathSet found; + for (auto & i : refsSink.getResult()) { + auto j = backMap.find(i); + assert(j != backMap.end()); found.insert(j->second); } diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 4f12e6b21..a6119c861 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -1,13 +1,31 @@ #pragma once -#include "types.hh" #include "hash.hh" +#include "path.hh" namespace nix { -std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs); +std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs); -PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs); +StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); + +class RefScanSink : public Sink +{ + StringSet hashes; + StringSet seen; + + std::string tail; + +public: + + RefScanSink(StringSet && hashes) : hashes(hashes) + { } + + StringSet & getResult() + { return seen; } + + void operator () (std::string_view data) override; +}; struct RewritingSink : Sink { diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk new file mode 100644 index 000000000..f74295d97 --- /dev/null +++ b/src/libstore/tests/local.mk @@ -0,0 +1,15 @@ +check: libstore-tests_RUN + +programs += libstore-tests + +libstore-tests_DIR := $(d) + +libstore-tests_INSTALL_DIR := + +libstore-tests_SOURCES := $(wildcard $(d)/*.cc) + +libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil + +libstore-tests_LIBS = libstore libutil + +libstore-tests_LDFLAGS := $(GTEST_LIBS) diff --git a/src/libstore/tests/references.cc b/src/libstore/tests/references.cc new file mode 100644 index 000000000..d91d1cedd --- /dev/null +++ b/src/libstore/tests/references.cc @@ -0,0 +1,45 @@ +#include "references.hh" + +#include <gtest/gtest.h> + +namespace nix { + +TEST(references, scan) +{ + std::string hash1 = "dc04vv14dak1c1r48qa0m23vr9jy8sm0"; + std::string hash2 = "zc842j0rz61mjsp3h3wp5ly71ak6qgdn"; + + { + RefScanSink scanner(StringSet{hash1}); + auto s = "foobar"; + scanner(s); + ASSERT_EQ(scanner.getResult(), StringSet{}); + } + + { + RefScanSink scanner(StringSet{hash1}); + auto s = "foobar" + hash1 + "xyzzy"; + scanner(s); + ASSERT_EQ(scanner.getResult(), StringSet{hash1}); + } + + { + RefScanSink scanner(StringSet{hash1, hash2}); + auto s = "foobar" + hash1 + "xyzzy" + hash2; + scanner(((std::string_view) s).substr(0, 10)); + scanner(((std::string_view) s).substr(10, 5)); + scanner(((std::string_view) s).substr(15, 5)); + scanner(((std::string_view) s).substr(20)); + ASSERT_EQ(scanner.getResult(), StringSet({hash1, hash2})); + } + + { + RefScanSink scanner(StringSet{hash1, hash2}); + auto s = "foobar" + hash1 + "xyzzy" + hash2; + for (auto & i : s) + scanner(std::string(1, i)); + ASSERT_EQ(scanner.getResult(), StringSet({hash1, hash2})); + } +} + +} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index cfadccf68..02e81b022 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -65,16 +65,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() throw SysError("cannot create Unix domain socket"); closeOnExec(conn->fd.get()); - string socketPath = path ? *path : settings.nixDaemonSocketFile; - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (socketPath.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%1%' is too long", socketPath); - strcpy(addr.sun_path, socketPath.c_str()); - - if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to daemon at '%1%'", socketPath); + nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index ed0eb2fb5..d78ec2b93 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -42,7 +42,7 @@ static string caseHackSuffix = "~nix~case~hack~"; PathFilter defaultPathFilter = [](const Path &) { return true; }; -static void dumpContents(const Path & path, size_t size, +static void dumpContents(const Path & path, off_t size, Sink & sink) { sink << "contents" << size; @@ -76,7 +76,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter) sink << "type" << "regular"; if (st.st_mode & S_IXUSR) sink << "executable" << ""; - dumpContents(path, (size_t) st.st_size, sink); + dumpContents(path, st.st_size, sink); } else if (S_ISDIR(st.st_mode)) { diff --git a/src/libutil/util.cc b/src/libutil/util.cc index bc841f425..563a72c12 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -903,7 +903,7 @@ int Pid::wait() return status; } if (errno != EINTR) - throw SysError("cannot get child exit status"); + throw SysError("cannot get exit status of PID %d", pid); checkInterrupt(); } } @@ -939,9 +939,6 @@ void killUser(uid_t uid) users to which the current process can send signals. So we fork a process, switch to uid, and send a mass kill. */ - ProcessOptions options; - options.allowVfork = false; - Pid pid = startProcess([&]() { if (setuid(uid) == -1) @@ -964,7 +961,7 @@ void killUser(uid_t uid) } _exit(0); - }, options); + }); int status = pid.wait(); if (status != 0) @@ -1085,8 +1082,7 @@ void runProgram2(const RunOptions & options) // vfork implies that the environment of the main process and the fork will // be shared (technically this is undefined, but in practice that's the // case), so we can't use it if we alter the environment - if (options.environment) - processOptions.allowVfork = false; + processOptions.allowVfork = !options.environment; /* Fork. */ Pid pid = startProcess([&]() { @@ -1686,16 +1682,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) closeOnExec(fdSocket.get()); - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - if (path.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%1%' is too long", path); - strcpy(addr.sun_path, path.c_str()); - - unlink(path.c_str()); - - if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%1%'", path); + bind(fdSocket.get(), path); if (chmod(path.c_str(), mode) == -1) throw SysError("changing permissions on '%1%'", path); @@ -1707,6 +1694,66 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) } +void bind(int fd, const std::string & path) +{ + unlink(path.c_str()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + auto dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + strcpy(addr.sun_path, base.c_str()); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot bind to socket '%s'", path); + } else { + strcpy(addr.sun_path, path.c_str()); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + } +} + + +void connect(int fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + auto dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + strcpy(addr.sun_path, base.c_str()); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot connect to socket at '%s'", path); + } else { + strcpy(addr.sun_path, path.c_str()); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + } +} + + string showBytes(uint64_t bytes) { return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index bee77b53f..29232453f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -259,10 +259,10 @@ void killUser(uid_t uid); pid to the caller. */ struct ProcessOptions { - string errorPrefix = "error: "; + string errorPrefix = ""; bool dieWithParent = true; bool runExitHandlers = false; - bool allowVfork = true; + bool allowVfork = false; }; pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); @@ -574,6 +574,12 @@ void commonChildInit(Pipe & logPipe); /* Create a Unix domain socket in listen mode. */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); +/* Bind a Unix domain socket to a path. */ +void bind(int fd, const std::string & path); + +/* Connect to a Unix domain socket. */ +void connect(int fd, const std::string & path); + // A Rust/Python-like enumerate() iterator adapter. // Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17. diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 2cf2a04c9..6a40a0bd3 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -156,9 +156,6 @@ static void daemonLoop() if (chdir("/") == -1) throw SysError("cannot change current directory"); - // Get rid of children automatically; don't let them become zombies. - setSigChldAction(true); - AutoCloseFD fdSocket; // Handle socket-based activation by systemd. @@ -176,6 +173,9 @@ static void daemonLoop() fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666); } + // Get rid of children automatically; don't let them become zombies. + setSigChldAction(true); + // Loop accepting connections. while (1) { diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 7d7ada707..7e4d23f6e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -346,10 +346,10 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") + if (!v.isLambda() || v.lambda.fun->hasFormals() || std::string(v.lambda.fun->arg) != "final") throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body); - if (!body || body->matchAttrs || std::string(body->arg) != "prev") + if (!body || body->hasFormals() || std::string(body->arg) != "prev") throw Error("overlay does not take an argument named 'prev'"); // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. @@ -363,7 +363,7 @@ struct CmdFlakeCheck : FlakeCommand try { state->forceValue(v, pos); if (v.isLambda()) { - if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) + if (!v.lambda.fun->hasFormals() || !v.lambda.fun->formals->ellipsis) throw Error("module must match an open attribute set ('{ config, ... }')"); } else if (v.type() == nAttrs) { for (auto & attr : *v.attrs) diff --git a/src/nix/flake.md b/src/nix/flake.md index 3d273100b..3b5812a0a 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -225,7 +225,7 @@ Currently the `type` attribute can be one of the following: [flake:]<flake-id>(/<rev-or-ref>(/rev)?)? ``` - These perform a lookup of `<flake-id>` in the flake registry. or + These perform a lookup of `<flake-id>` in the flake registry. For example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake references. The specified `rev` and/or `ref` are merged with the entry in the registry; see [nix registry](./nix3-registry.md) for diff --git a/src/nix/main.cc b/src/nix/main.cc index 8aaf08813..2c3976689 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -187,11 +187,14 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve , "/"), *vUtils); - auto vJson = state.allocValue(); + auto vArgs = state.allocValue(); + state.mkAttrs(*vArgs, 16); + auto vJson = state.allocAttr(*vArgs, state.symbols.create("command")); mkString(*vJson, toplevel.toJSON().dump()); + vArgs->attrs->sort(); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, *vJson, *vRes, noPos); + state.callFunction(*vGenerateManpage, *vArgs, *vRes, noPos); auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); if (!attr) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 916966997..b49be256e 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -98,10 +98,8 @@ struct ProfileManifest else if (pathExists(profile + "/manifest.nix")) { // FIXME: needed because of pure mode; ugly. - if (state.allowedPaths) { - state.allowedPaths->insert(state.store->followLinksToStore(profile)); - state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix")); - } + state.allowPath(state.store->followLinksToStore(profile)); + state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix")); auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); |