diff options
Diffstat (limited to 'src')
77 files changed, 1976 insertions, 1132 deletions
diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index fd3edfc46..429cd32cc 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -73,8 +73,13 @@ ref<Store> EvalCommand::getEvalStore() ref<EvalState> EvalCommand::getEvalState() { - if (!evalState) - evalState = std::make_shared<EvalState>(searchPath, getEvalStore(), getStore()); + if (!evalState) evalState = +#if HAVE_BOEHMGC + std::allocate_shared<EvalState>(traceable_allocator<EvalState>(), +#else + std::make_shared<EvalState>( +#endif + searchPath, getEvalStore(), getStore()); return ref<EvalState>(evalState); } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5758b52ad..ef200b1d2 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -291,6 +291,9 @@ void completeFlakeRefWithFragment( void completeFlakeRef(ref<Store> store, std::string_view prefix) { + if (!settings.isExperimentalFeatureEnabled(Xp::Flakes)) + return; + if (prefix == "") completions->add("."); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index db1e7e56d..b987e1888 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -119,8 +119,8 @@ void printValue(std::ostream & str, std::set<const Value *> & active, const Valu case tList2: case tListN: str << "[ "; - for (unsigned int n = 0; n < v.listSize(); ++n) { - printValue(str, active, *v.listElems()[n]); + for (auto v2 : v.listItems()) { + printValue(str, active, *v2); str << " "; } str << "]"; @@ -519,8 +519,12 @@ Path EvalState::checkSourcePath(const Path & path_) } } - if (!found) - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath); + if (!found) { + auto modeInformation = evalSettings.pureEval + ? "in pure eval mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation); + } /* Resolve symlinks. */ debug(format("checking access to '%s'") % abspath); @@ -533,7 +537,7 @@ Path EvalState::checkSourcePath(const Path & path_) } } - throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); + throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path); } @@ -583,14 +587,20 @@ Value * EvalState::addConstant(const string & name, Value & v) { Value * v2 = allocValue(); *v2 = v; - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; - baseEnv.values[baseEnvDispl++] = v2; - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + addConstant(name, v2); return v2; } +void EvalState::addConstant(const string & name, Value * v) +{ + staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); + baseEnv.values[baseEnvDispl++] = v; + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v)); +} + + Value * EvalState::addPrimOp(const string & name, size_t arity, PrimOpFun primOp) { @@ -609,7 +619,7 @@ Value * EvalState::addPrimOp(const string & name, Value * v = allocValue(); v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym }); - staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + staticBaseEnv.vars.emplace_back(symbols.create(name), baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); return v; @@ -635,7 +645,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) Value * v = allocValue(); v->mkPrimOp(new PrimOp(std::move(primOp))); - staticBaseEnv.vars[envName] = baseEnvDispl; + staticBaseEnv.vars.emplace_back(envName, baseEnvDispl); baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); return v; @@ -785,7 +795,7 @@ void mkPath(Value & v, const char * s) inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) { - for (size_t l = var.level; l; --l, env = env->up) ; + for (auto l = var.level; l; --l, env = env->up) ; if (!var.fromWith) return env->values[var.displ]; @@ -1058,7 +1068,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - size_t displ = 0; + Displacement displ = 0; for (auto & i : attrs) { Value * vAttr; if (hasOverrides && !i.second.inherited) { @@ -1134,7 +1144,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ - size_t displ = 0; + Displacement displ = 0; for (auto & i : attrs->attrs) env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); @@ -1145,8 +1155,8 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) void ExprList::eval(EvalState & state, Env & env, Value & v) { state.mkList(v, elems.size()); - for (size_t n = 0; n < elems.size(); ++n) - v.listElems()[n] = elems[n]->maybeThunk(state, env); + for (auto [n, v2] : enumerate(v.listItems())) + const_cast<Value * &>(v2) = elems[n]->maybeThunk(state, env); } @@ -1251,144 +1261,184 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) } -void ExprApp::eval(EvalState & state, Env & env, Value & v) +void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos) { - /* FIXME: vFun prevents GCC from doing tail call optimisation. */ - Value vFun; - e1->eval(state, env, vFun); - state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos); -} + auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr; + forceValue(fun, pos); -void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) -{ - /* Figure out the number of arguments still needed. */ - size_t argsDone = 0; - Value * primOp = &fun; - while (primOp->isPrimOpApp()) { - argsDone++; - primOp = primOp->primOpApp.left; - } - assert(primOp->isPrimOp()); - auto arity = primOp->primOp->arity; - auto argsLeft = arity - argsDone; - - if (argsLeft == 1) { - /* We have all the arguments, so call the primop. */ - - /* Put all the arguments in an array. */ - Value * vArgs[arity]; - auto n = arity - 1; - vArgs[n--] = &arg; - for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left) - vArgs[n--] = arg->primOpApp.right; - - /* And call the primop. */ - nrPrimOpCalls++; - if (countCalls) primOpCalls[primOp->primOp->name]++; - primOp->primOp->fun(*this, pos, vArgs, v); - } else { - Value * fun2 = allocValue(); - *fun2 = fun; - v.mkPrimOpApp(fun2, &arg); - } -} + Value vCur(fun); -void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) -{ - auto trace = evalSettings.traceFunctionCalls ? std::make_unique<FunctionCallTrace>(pos) : nullptr; + auto makeAppChain = [&]() + { + vRes = vCur; + for (size_t i = 0; i < nrArgs; ++i) { + auto fun2 = allocValue(); + *fun2 = vRes; + vRes.mkPrimOpApp(fun2, args[i]); + } + }; - forceValue(fun, pos); + Attr * functor; - if (fun.isPrimOp() || fun.isPrimOpApp()) { - callPrimOp(fun, arg, v, pos); - return; - } + while (nrArgs > 0) { - if (fun.type() == nAttrs) { - auto found = fun.attrs->find(sFunctor); - if (found != fun.attrs->end()) { - /* fun may be allocated on the stack of the calling function, - * but for functors we may keep a reference, so heap-allocate - * a copy and use that instead. - */ - auto & fun2 = *allocValue(); - fun2 = fun; - /* !!! Should we use the attr pos here? */ - Value v2; - callFunction(*found->value, fun2, v2, pos); - return callFunction(v2, arg, v, pos); - } - } + if (vCur.isLambda()) { - if (!fun.isLambda()) - throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); + ExprLambda & lambda(*vCur.lambda.fun); - ExprLambda & lambda(*fun.lambda.fun); + auto size = + (lambda.arg.empty() ? 0 : 1) + + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); + Env & env2(allocEnv(size)); + env2.up = vCur.lambda.env; - auto size = - (lambda.arg.empty() ? 0 : 1) + - (lambda.hasFormals() ? lambda.formals->formals.size() : 0); - Env & env2(allocEnv(size)); - env2.up = fun.lambda.env; + Displacement displ = 0; - size_t displ = 0; + if (!lambda.hasFormals()) + env2.values[displ++] = args[0]; - if (!lambda.hasFormals()) - env2.values[displ++] = &arg; + else { + forceAttrs(*args[0], pos); - else { - forceAttrs(arg, pos); - - if (!lambda.arg.empty()) - env2.values[displ++] = &arg; - - /* For each formal argument, get the actual argument. If - there is no matching actual argument but the formal - argument has a default, use the default. */ - size_t attrsUsed = 0; - for (auto & i : lambda.formals->formals) { - Bindings::iterator j = arg.attrs->find(i.name); - if (j == arg.attrs->end()) { - if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", - lambda, i.name); - env2.values[displ++] = i.def->maybeThunk(*this, env2); + if (!lambda.arg.empty()) + env2.values[displ++] = args[0]; + + /* For each formal argument, get the actual argument. If + there is no matching actual argument but the formal + argument has a default, use the default. */ + size_t attrsUsed = 0; + for (auto & i : lambda.formals->formals) { + auto j = args[0]->attrs->get(i.name); + if (!j) { + if (!i.def) throwTypeError(pos, "%1% called without required argument '%2%'", + lambda, i.name); + env2.values[displ++] = i.def->maybeThunk(*this, env2); + } else { + attrsUsed++; + env2.values[displ++] = j->value; + } + } + + /* Check that each actual argument is listed as a formal + argument (unless the attribute match specifies a `...'). */ + if (!lambda.formals->ellipsis && attrsUsed != args[0]->attrs->size()) { + /* Nope, so show the first unexpected argument to the + user. */ + for (auto & i : *args[0]->attrs) + if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) + throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); + abort(); // can't happen + } + } + + nrFunctionCalls++; + if (countCalls) incrFunctionCall(&lambda); + + /* Evaluate the body. */ + try { + lambda.body->eval(*this, env2, vCur); + } catch (Error & e) { + if (loggerSettings.showTrace.get()) { + addErrorTrace(e, lambda.pos, "while evaluating %s", + (lambda.name.set() + ? "'" + (string) lambda.name + "'" + : "anonymous lambda")); + addErrorTrace(e, pos, "from call site%s", ""); + } + throw; + } + + nrArgs--; + args += 1; + } + + else if (vCur.isPrimOp()) { + + size_t argsLeft = vCur.primOp->arity; + + if (nrArgs < argsLeft) { + /* We don't have enough arguments, so create a tPrimOpApp chain. */ + makeAppChain(); + return; } else { - attrsUsed++; - env2.values[displ++] = j->value; + /* We have all the arguments, so call the primop. */ + nrPrimOpCalls++; + if (countCalls) primOpCalls[vCur.primOp->name]++; + vCur.primOp->fun(*this, pos, args, vCur); + + nrArgs -= argsLeft; + args += argsLeft; } } - /* Check that each actual argument is listed as a formal - argument (unless the attribute match specifies a `...'). */ - if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { - /* Nope, so show the first unexpected argument to the - user. */ - for (auto & i : *arg.attrs) - if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) - throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name); - abort(); // can't happen + else if (vCur.isPrimOpApp()) { + /* Figure out the number of arguments still needed. */ + size_t argsDone = 0; + Value * primOp = &vCur; + while (primOp->isPrimOpApp()) { + argsDone++; + primOp = primOp->primOpApp.left; + } + assert(primOp->isPrimOp()); + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; + + if (nrArgs < argsLeft) { + /* We still don't have enough arguments, so extend the tPrimOpApp chain. */ + makeAppChain(); + return; + } else { + /* We have all the arguments, so call the primop with + the previous and new arguments. */ + + Value * vArgs[arity]; + auto n = argsDone; + for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) + vArgs[--n] = arg->primOpApp.right; + + for (size_t i = 0; i < argsLeft; ++i) + vArgs[argsDone + i] = args[i]; + + nrPrimOpCalls++; + if (countCalls) primOpCalls[primOp->primOp->name]++; + primOp->primOp->fun(*this, pos, vArgs, vCur); + + nrArgs -= argsLeft; + args += argsLeft; + } + } + + else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) { + /* 'vCur' may be allocated on the stack of the calling + function, but for functors we may keep a reference, so + heap-allocate a copy and use that instead. */ + Value * args2[] = {allocValue(), args[0]}; + *args2[0] = vCur; + /* !!! Should we use the attr pos here? */ + callFunction(*functor->value, 2, args2, vCur, pos); + nrArgs--; + args++; } + + else + throwTypeError(pos, "attempt to call something which is not a function but %1%", vCur); } - nrFunctionCalls++; - if (countCalls) incrFunctionCall(&lambda); + vRes = vCur; +} - /* Evaluate the body. This is conditional on showTrace, because - catching exceptions makes this function not tail-recursive. */ - if (loggerSettings.showTrace.get()) - try { - lambda.body->eval(*this, env2, v); - } catch (Error & e) { - addErrorTrace(e, lambda.pos, "while evaluating %s", - (lambda.name.set() - ? "'" + (string) lambda.name + "'" - : "anonymous lambda")); - addErrorTrace(e, pos, "from call site%s", ""); - throw; - } - else - fun.lambda.fun->body->eval(*this, env2, v); + +void ExprCall::eval(EvalState & state, Env & env, Value & v) +{ + Value vFun; + fun->eval(state, env, vFun); + + Value * vArgs[args.size()]; + for (size_t i = 0; i < args.size(); ++i) + vArgs[i] = args[i]->maybeThunk(state, env); + + state.callFunction(vFun, args.size(), vArgs, v, pos); } @@ -1686,8 +1736,8 @@ void EvalState::forceValueDeep(Value & v) } else if (v.isList()) { - for (size_t n = 0; n < v.listSize(); ++n) - recurse(*v.listElems()[n]); + for (auto v2 : v.listItems()) + recurse(*v2); } }; @@ -1871,12 +1921,12 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, if (v.isList()) { string result; - for (size_t n = 0; n < v.listSize(); ++n) { - result += coerceToString(pos, *v.listElems()[n], + for (auto [n, v2] : enumerate(v.listItems())) { + result += coerceToString(pos, *v2, context, coerceMore, copyToStore); if (n < v.listSize() - 1 /* !!! not quite correct */ - && (!v.listElems()[n]->isList() || v.listElems()[n]->listSize() != 0)) + && (!v2->isList() || v2->listSize() != 0)) result += " "; } return result; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 69119599a..1aab8e166 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -277,6 +277,8 @@ private: Value * addConstant(const string & name, Value & v); + void addConstant(const string & name, Value * v); + Value * addPrimOp(const string & name, size_t arity, PrimOpFun primOp); @@ -316,8 +318,14 @@ public: bool isFunctor(Value & fun); - void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos); - void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos); + // FIXME: use std::span + void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const Pos & pos); + + void callFunction(Value & fun, Value & arg, Value & vRes, const Pos & pos) + { + Value * args[] = {&arg}; + callFunction(fun, 1, args, vRes, pos); + } /* Automatically call a function for which each argument has a default value or has a binding in the `args' map. */ diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index 41b6f78ed..c03f4106c 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,4 +1,5 @@ #include "flake.hh" +#include "globals.hh" #include <nlohmann/json.hpp> @@ -52,21 +53,19 @@ void ConfigFile::apply() auto trustedList = readTrustedList(); bool trusted = false; - - if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) { + if (nix::settings.acceptFlakeConfig){ + trusted = true; + } else if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) { trusted = *saved; + warn("Using saved setting for '%s = %s' from ~/.local/share/nix/trusted-settings.json.", name,valueS); } else { // FIXME: filter ANSI escapes, newlines, \r, etc. - if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') { - if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = false; - writeTrustedList(trustedList); - } - } else { - if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') { - trustedList[name][valueS] = trusted = true; - writeTrustedList(trustedList); - } + if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) == 'y') { + trusted = true; + } + if (std::tolower(logger->ask(fmt("do you want to permanently mark this value as %s (y/N)?", trusted ? "trusted": "untrusted" )).value_or('n')) == 'y') { + trustedList[name][valueS] = trusted; + writeTrustedList(trustedList); } } diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index c9d848495..33d253eee 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -257,8 +257,7 @@ static Flake getFlake( flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); else if (setting.value->type() == nList) { std::vector<std::string> ss; - for (unsigned int n = 0; n < setting.value->listSize(); ++n) { - auto elem = setting.value->listElems()[n]; + for (auto elem : setting.value->listItems()) { if (elem->type() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", setting.name, showType(*setting.value)); @@ -307,7 +306,7 @@ LockedFlake lockFlake( if (lockFlags.applyNixConfig) { flake.config.apply(); - // FIXME: send new config to the daemon. + state.store->setOptions(); } try { @@ -446,22 +445,18 @@ LockedFlake lockFlake( update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); - auto hasChildUpdate = + auto mustRefetch = lb != lockFlags.inputUpdates.end() && lb->size() > inputPath.size() && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); - if (hasChildUpdate) { - auto inputFlake = getFlake( - state, oldLock->lockedRef, false, flakeCache); - computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, parent, parentPath); - } else { + FlakeInputs fakeInputs; + + if (!mustRefetch) { /* No need to fetch this flake, we can be lazy. However there may be new overrides on the inputs of this flake, so we need to check those. */ - FlakeInputs fakeInputs; - for (auto & i : oldLock->inputs) { if (auto lockedNode = std::get_if<0>(&i.second)) { fakeInputs.emplace(i.first, FlakeInput { @@ -469,15 +464,33 @@ LockedFlake lockFlake( .isFlake = (*lockedNode)->isFlake, }); } else if (auto follows = std::get_if<1>(&i.second)) { + auto o = input.overrides.find(i.first); + // If the override disappeared, we have to refetch the flake, + // since some of the inputs may not be present in the lockfile. + if (o == input.overrides.end()) { + mustRefetch = true; + // There's no point populating the rest of the fake inputs, + // since we'll refetch the flake anyways. + break; + } fakeInputs.emplace(i.first, FlakeInput { .follows = *follows, }); } } - - computeLocks(fakeInputs, childNode, inputPath, oldLock, parent, parentPath); } + LockParent newParent { + .path = inputPath, + .absolute = false + }; + + computeLocks( + mustRefetch + ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs + : fakeInputs, + childNode, inputPath, oldLock, newParent, parentPath); + } else { /* We need to create a new lock file entry. So fetch this input. */ diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index f774e6493..ed4c47fbb 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -102,9 +102,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) state->forceList(*i->value, *i->pos); /* For each output... */ - for (unsigned int j = 0; j < i->value->listSize(); ++j) { + for (auto elem : i->value->listItems()) { /* Evaluate the corresponding set. */ - string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos); + string name = state->forceStringNoCtx(*elem, *i->pos); Bindings::iterator out = attrs->find(state->symbols.create(name)); if (out == attrs->end()) continue; // FIXME: throw error? state->forceAttrs(*out->value); @@ -128,9 +128,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) throw errMsg; Outputs result; - for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->type() != nString) throw errMsg; - auto out = outputs.find((*i)->string.s); + for (auto elem : outTI->listItems()) { + if (elem->type() != nString) throw errMsg; + auto out = outputs.find(elem->string.s); if (out == outputs.end()) throw errMsg; result.insert(*out); } @@ -174,8 +174,8 @@ bool DrvInfo::checkMeta(Value & v) { state->forceValue(v); if (v.type() == nList) { - for (unsigned int n = 0; n < v.listSize(); ++n) - if (!checkMeta(*v.listElems()[n])) return false; + for (auto elem : v.listItems()) + if (!checkMeta(*elem)) return false; return true; } else if (v.type() == nAttrs) { @@ -364,10 +364,10 @@ static void getDerivations(EvalState & state, Value & vIn, } else if (v.type() == nList) { - for (unsigned int n = 0; n < v.listSize(); ++n) { - string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); - if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) - getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + for (auto [n, elem] : enumerate(v.listItems())) { + string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n)); + if (getDerivation(state, *elem, pathPrefix2, drvs, done, ignoreAssertionFailures)) + getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 51593eccd..c18877e29 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -64,6 +64,7 @@ static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) } +// FIXME: optimize static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) { string t; diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 0d0f3e469..57c2f6e44 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -143,6 +143,16 @@ void ExprLambda::show(std::ostream & str) const str << ": " << *body << ")"; } +void ExprCall::show(std::ostream & str) const +{ + str << '(' << *fun; + for (auto e : args) { + str << ' '; + str << *e; + } + str << ')'; +} + void ExprLet::show(std::ostream & str) const { str << "(let "; @@ -263,13 +273,13 @@ void ExprVar::bindVars(const StaticEnv & env) /* Check whether the variable appears in the environment. If so, set its level and displacement. */ const StaticEnv * curEnv; - unsigned int level; + Level level; int withLevel = -1; for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { if (curEnv->isWith) { if (withLevel == -1) withLevel = level; } else { - StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); + auto i = curEnv->find(name); if (i != curEnv->vars.end()) { fromWith = false; this->level = level; @@ -311,14 +321,16 @@ void ExprOpHasAttr::bindVars(const StaticEnv & env) void ExprAttrs::bindVars(const StaticEnv & env) { const StaticEnv * dynamicEnv = &env; - StaticEnv newEnv(false, &env); + StaticEnv newEnv(false, &env, recursive ? attrs.size() : 0); if (recursive) { dynamicEnv = &newEnv; - unsigned int displ = 0; + Displacement displ = 0; for (auto & i : attrs) - newEnv.vars[i.first] = i.second.displ = displ++; + newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + + // No need to sort newEnv since attrs is in sorted order. for (auto & i : attrs) i.second.e->bindVars(i.second.inherited ? env : newEnv); @@ -342,15 +354,20 @@ void ExprList::bindVars(const StaticEnv & env) void ExprLambda::bindVars(const StaticEnv & env) { - StaticEnv newEnv(false, &env); + StaticEnv newEnv( + false, &env, + (hasFormals() ? formals->formals.size() : 0) + + (arg.empty() ? 0 : 1)); - unsigned int displ = 0; + Displacement displ = 0; - if (!arg.empty()) newEnv.vars[arg] = displ++; + if (!arg.empty()) newEnv.vars.emplace_back(arg, displ++); if (hasFormals()) { for (auto & i : formals->formals) - newEnv.vars[i.name] = displ++; + newEnv.vars.emplace_back(i.name, displ++); + + newEnv.sort(); for (auto & i : formals->formals) if (i.def) i.def->bindVars(newEnv); @@ -359,13 +376,22 @@ void ExprLambda::bindVars(const StaticEnv & env) body->bindVars(newEnv); } +void ExprCall::bindVars(const StaticEnv & env) +{ + fun->bindVars(env); + for (auto e : args) + e->bindVars(env); +} + void ExprLet::bindVars(const StaticEnv & env) { - StaticEnv newEnv(false, &env); + StaticEnv newEnv(false, &env, attrs->attrs.size()); - unsigned int displ = 0; + Displacement displ = 0; for (auto & i : attrs->attrs) - newEnv.vars[i.first] = i.second.displ = displ++; + newEnv.vars.emplace_back(i.first, i.second.displ = displ++); + + // No need to sort newEnv since attrs->attrs is in sorted order. for (auto & i : attrs->attrs) i.second.e->bindVars(i.second.inherited ? env : newEnv); @@ -379,7 +405,7 @@ void ExprWith::bindVars(const StaticEnv & env) level so that `lookupVar' can look up variables in the previous `with' if this one doesn't contain the desired attribute. */ const StaticEnv * curEnv; - unsigned int level; + Level level; prevWith = 0; for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++) if (curEnv->isWith) { @@ -452,5 +478,4 @@ size_t SymbolTable::totalSize() const return n; } - } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 851e875bd..13256272c 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -4,8 +4,6 @@ #include "symbol-table.hh" #include "error.hh" -#include <map> - namespace nix { @@ -135,6 +133,9 @@ struct ExprPath : Expr Value * maybeThunk(EvalState & state, Env & env); }; +typedef uint32_t Level; +typedef uint32_t Displacement; + struct ExprVar : Expr { Pos pos; @@ -150,8 +151,8 @@ struct ExprVar : Expr value is obtained by getting the attribute named `name' from the set stored in the environment that is `level' levels up from the current one.*/ - unsigned int level; - unsigned int displ; + Level level; + Displacement displ; ExprVar(const Symbol & name) : name(name) { }; ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; @@ -185,7 +186,7 @@ struct ExprAttrs : Expr bool inherited; Expr * e; Pos pos; - unsigned int displ; // displacement + Displacement displ; // displacement AttrDef(Expr * e, const Pos & pos, bool inherited=false) : inherited(inherited), e(e), pos(pos) { }; AttrDef() { }; @@ -250,6 +251,17 @@ struct ExprLambda : Expr COMMON_METHODS }; +struct ExprCall : Expr +{ + Expr * fun; + std::vector<Expr *> args; + Pos pos; + ExprCall(const Pos & pos, Expr * fun, std::vector<Expr *> && args) + : fun(fun), args(args), pos(pos) + { } + COMMON_METHODS +}; + struct ExprLet : Expr { ExprAttrs * attrs; @@ -308,7 +320,6 @@ struct ExprOpNot : Expr void eval(EvalState & state, Env & env, Value & v); \ }; -MakeBinOp(ExprApp, "") MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=") MakeBinOp(ExprOpAnd, "&&") @@ -342,9 +353,28 @@ struct StaticEnv { bool isWith; const StaticEnv * up; - typedef std::map<Symbol, unsigned int> Vars; + + // Note: these must be in sorted order. + typedef std::vector<std::pair<Symbol, Displacement>> Vars; Vars vars; - StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { }; + + StaticEnv(bool isWith, const StaticEnv * up, size_t expectedSize = 0) : isWith(isWith), up(up) { + vars.reserve(expectedSize); + }; + + void sort() + { + std::sort(vars.begin(), vars.end(), + [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; }); + } + + Vars::const_iterator find(const Symbol & name) const + { + Vars::value_type key(name, 0); + auto i = std::lower_bound(vars.begin(), vars.end(), key); + if (i != vars.end() && i->first == name) return i; + return vars.end(); + } }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 813ff2fc3..c1f4e72e0 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -33,11 +33,9 @@ namespace nix { Symbol file; FileOrigin origin; std::optional<ErrorInfo> error; - Symbol sLetBody; ParseData(EvalState & state) : state(state) , symbols(state.symbols) - , sLetBody(symbols.create("<let-body>")) { }; }; @@ -126,14 +124,14 @@ static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, auto j2 = jAttrs->attrs.find(ad.first); if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. dupAttr(ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs[ad.first] = ad.second; + jAttrs->attrs.emplace(ad.first, ad.second); } } else { dupAttr(attrPath, pos, j->second.pos); } } else { // This attr path is not defined. Let's create it. - attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); + attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); e->setName(i->symbol); } } else { @@ -283,7 +281,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err } %type <e> start expr expr_function expr_if expr_op -%type <e> expr_app expr_select expr_simple +%type <e> expr_select expr_simple expr_app %type <list> expr_list %type <attrs> binds %type <formals> formals @@ -353,13 +351,13 @@ expr_if expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } - | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); } + | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {new ExprInt(0), $2}); } | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } - | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); } - | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); } - | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); } - | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); } + | expr_op '<' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3}); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1})); } + | expr_op '>' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$3, $1}); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__lessThan")), {$1, $3})); } | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } @@ -367,17 +365,22 @@ expr_op | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } | expr_op '+' expr_op { $$ = new ExprConcatStrings(CUR_POS, false, new vector<Expr *>({$1, $3})); } - | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); } - | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); } - | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); } + | expr_op '-' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__sub")), {$1, $3}); } + | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); } + | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); } | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } | expr_app ; expr_app - : expr_app expr_select - { $$ = new ExprApp(CUR_POS, $1, $2); } - | expr_select { $$ = $1; } + : expr_app expr_select { + if (auto e2 = dynamic_cast<ExprCall *>($1)) { + e2->args.push_back($2); + $$ = $1; + } else + $$ = new ExprCall(CUR_POS, $1, {$2}); + } + | expr_select ; expr_select @@ -388,7 +391,7 @@ expr_select | /* Backwards compatibility: because Nixpkgs has a rarely used function named ‘or’, allow stuff like ‘map or [...]’. */ expr_simple OR_KW - { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.create("or"))); } + { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, data->symbols.create("or"))}); } | expr_simple { $$ = $1; } ; @@ -412,10 +415,10 @@ expr_simple } | SPATH { string path($1 + 1, strlen($1) - 2); - $$ = new ExprApp(CUR_POS, - new ExprApp(new ExprVar(data->symbols.create("__findFile")), - new ExprVar(data->symbols.create("__nixPath"))), - new ExprString(data->symbols.create(path))); + $$ = new ExprCall(CUR_POS, + new ExprVar(data->symbols.create("__findFile")), + {new ExprVar(data->symbols.create("__nixPath")), + new ExprString(data->symbols.create(path))}); } | URI { static bool noURLLiterals = settings.isExperimentalFeatureEnabled(Xp::NoUrlLiterals); @@ -483,7 +486,7 @@ binds if ($$->attrs.find(i.symbol) != $$->attrs.end()) dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); Pos pos = makeCurPos(@3, data); - $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); } } | binds INHERIT '(' expr ')' attrs ';' @@ -492,7 +495,7 @@ binds for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); - $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)); + $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data))); } } | { $$ = new ExprAttrs(makeCurPos(@0, data)); } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 6b3cafec8..66af373d7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -70,7 +70,7 @@ void EvalState::realiseContext(const PathSet & context) if (outputPaths.count(outputName) == 0) throw Error("derivation '%s' does not have an output named '%s'", store->printStorePath(drvPath), outputName); - allowedPaths->insert(store->printStorePath(outputPaths.at(outputName))); + allowPath(outputPaths.at(outputName)); } } } @@ -184,14 +184,17 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS Env * env = &state.allocEnv(vScope->attrs->size()); env->up = &state.baseEnv; - StaticEnv staticEnv(false, &state.staticBaseEnv); + StaticEnv staticEnv(false, &state.staticBaseEnv, vScope->attrs->size()); unsigned int displ = 0; for (auto & attr : *vScope->attrs) { - staticEnv.vars[attr.name] = displ; + staticEnv.vars.emplace_back(attr.name, displ); env->values[displ++] = attr.value; } + // No need to call staticEnv.sort(), because + // args[0]->attrs is already sorted. + printTalkative("evaluating file '%1%'", realPath); Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); @@ -332,9 +335,8 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) PathSet context; auto program = state.coerceToString(pos, *elems[0], context, false, false); Strings commandArgs; - for (unsigned int i = 1; i < args[0]->listSize(); ++i) { + for (unsigned int i = 1; i < args[0]->listSize(); ++i) commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false)); - } try { state.realiseContext(context); } catch (InvalidPathError & e) { @@ -514,7 +516,11 @@ static RegisterPrimOp primop_isPath({ struct CompareValues { - bool operator () (const Value * v1, const Value * v2) const + EvalState & state; + + CompareValues(EvalState & state) : state(state) { }; + + bool operator () (Value * v1, Value * v2) const { if (v1->type() == nFloat && v2->type() == nInt) return v1->fpoint < v2->integer; @@ -531,6 +537,17 @@ struct CompareValues return strcmp(v1->string.s, v2->string.s) < 0; case nPath: return strcmp(v1->path, v2->path) < 0; + case nList: + // Lexicographic comparison + for (size_t i = 0;; i++) { + if (i == v2->listSize()) { + return false; + } else if (i == v1->listSize()) { + return true; + } else if (!state.eqValues(*v1->listElems()[i], *v2->listElems()[i])) { + return (*this)(v1->listElems()[i], v2->listElems()[i]); + } + } default: throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); } @@ -598,8 +615,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar state.forceList(*startSet->value, pos); ValueList workSet; - for (unsigned int n = 0; n < startSet->value->listSize(); ++n) - workSet.push_back(startSet->value->listElems()[n]); + for (auto elem : startSet->value->listItems()) + workSet.push_back(elem); /* Get the operator. */ Bindings::iterator op = getAttr( @@ -618,7 +635,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar ValueList res; // `doneKeys' doesn't need to be a GC root, because its values are // reachable from res. - set<Value *, CompareValues> doneKeys; + auto cmp = CompareValues(state); + set<Value *, decltype(cmp)> doneKeys(cmp); while (!workSet.empty()) { Value * e = *(workSet.begin()); workSet.pop_front(); @@ -643,9 +661,9 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar state.forceList(call, pos); /* Add the values returned by the operator to the work set. */ - for (unsigned int n = 0; n < call.listSize(); ++n) { - state.forceValue(*call.listElems()[n], pos); - workSet.push_back(call.listElems()[n]); + for (auto elem : call.listItems()) { + state.forceValue(*elem, pos); + workSet.push_back(elem); } } @@ -985,16 +1003,17 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } if (i->name == state.sContentAddressed) { - settings.requireExperimentalFeature(Xp::CaDerivations); contentAddressed = state.forceBool(*i->value, pos); + if (contentAddressed) + settings.requireExperimentalFeature(Xp::CaDerivations); } /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { state.forceList(*i->value, pos); - for (unsigned int n = 0; n < i->value->listSize(); ++n) { - string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); + for (auto elem : i->value->listItems()) { + string s = state.coerceToString(posDrvName, *elem, context, true); drv.args.push_back(s); } } @@ -1008,7 +1027,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (i->name == state.sStructuredAttrs) continue; auto placeholder(jsonObject->placeholder(key)); - printValueAsJSON(state, true, *i->value, placeholder, context); + printValueAsJSON(state, true, *i->value, pos, placeholder, context); if (i->name == state.sBuilder) drv.builder = state.forceString(*i->value, context, posDrvName); @@ -1024,8 +1043,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Require ‘outputs’ to be a list of strings. */ state.forceList(*i->value, posDrvName); Strings ss; - for (unsigned int n = 0; n < i->value->listSize(); ++n) - ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName)); + for (auto elem : i->value->listItems()) + ss.emplace_back(state.forceStringNoCtx(*elem, posDrvName)); handleOutputs(ss); } @@ -1440,20 +1459,19 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va SearchPath searchPath; - for (unsigned int n = 0; n < args[0]->listSize(); ++n) { - Value & v2(*args[0]->listElems()[n]); - state.forceAttrs(v2, pos); + for (auto v2 : args[0]->listItems()) { + state.forceAttrs(*v2, pos); string prefix; - Bindings::iterator i = v2.attrs->find(state.symbols.create("prefix")); - if (i != v2.attrs->end()) + Bindings::iterator i = v2->attrs->find(state.symbols.create("prefix")); + if (i != v2->attrs->end()) prefix = state.forceStringNoCtx(*i->value, pos); i = getAttr( state, "findFile", "path", - v2.attrs, + v2->attrs, pos ); @@ -1579,7 +1597,7 @@ static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value { std::ostringstream out; PathSet context; - printValueAsXML(state, true, false, *args[0], out, context); + printValueAsXML(state, true, false, *args[0], out, context, pos); mkString(v, out.str(), context); } @@ -1687,7 +1705,7 @@ static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Valu { std::ostringstream out; PathSet context; - printValueAsJSON(state, true, *args[0], out, context); + printValueAsJSON(state, true, *args[0], pos, out, context); mkString(v, out.str(), context); } @@ -1859,12 +1877,12 @@ static void addPath( // be rewritten to the actual output). state.realiseContext(context); + StorePathSet refs; + if (state.store->isInStore(path)) { auto [storePath, subPath] = state.store->toStorePath(path); - auto info = state.store->queryPathInfo(storePath); - if (!info->references.empty()) - throw EvalError("store path '%s' is not allowed to have references", - state.store->printStorePath(storePath)); + // FIXME: we should scanForReferences on the path before adding it + refs = state.store->queryPathInfo(storePath)->references; path = state.store->toRealPath(storePath) + subPath; } @@ -1880,9 +1898,6 @@ static void addPath( Value arg1; mkString(arg1, path); - Value fun2; - state.callFunction(*filterFun, arg1, fun2, noPos); - Value arg2; mkString(arg2, S_ISREG(st.st_mode) ? "regular" : @@ -1890,8 +1905,9 @@ static void addPath( S_ISLNK(st.st_mode) ? "symlink" : "unknown" /* not supported, will fail! */); + Value * args []{&arg1, &arg2}; Value res; - state.callFunction(fun2, arg2, res, noPos); + state.callFunction(*filterFun, 2, args, res, pos); return state.forceBool(res, pos); }) : defaultPathFilter; @@ -1904,7 +1920,7 @@ static void addPath( if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { dstPath = state.store->printStorePath(settings.readOnlyMode ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair)); + : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs)); if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath)) throw Error("store path mismatch in (possibly filtered) path added from '%s'", path); } else @@ -2221,9 +2237,9 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, /* Get the attribute names to be removed. */ std::set<Symbol> names; - for (unsigned int i = 0; i < args[1]->listSize(); ++i) { - state.forceStringNoCtx(*args[1]->listElems()[i], pos); - names.insert(state.symbols.create(args[1]->listElems()[i]->string.s)); + for (auto elem : args[1]->listItems()) { + state.forceStringNoCtx(*elem, pos); + names.insert(state.symbols.create(elem->string.s)); } /* Copy all attributes not in that set. Note that we don't need @@ -2231,7 +2247,7 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, vector. */ state.mkAttrs(v, args[0]->attrs->size()); for (auto & i : *args[0]->attrs) { - if (names.find(i.name) == names.end()) + if (!names.count(i.name)) v.attrs->push_back(i); } } @@ -2265,15 +2281,14 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, std::set<Symbol> seen; - for (unsigned int i = 0; i < args[0]->listSize(); ++i) { - Value & v2(*args[0]->listElems()[i]); - state.forceAttrs(v2, pos); + for (auto v2 : args[0]->listItems()) { + state.forceAttrs(*v2, pos); Bindings::iterator j = getAttr( state, "listToAttrs", state.sName, - v2.attrs, + v2->attrs, pos ); @@ -2285,7 +2300,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, state, "listToAttrs", state.sValue, - v2.attrs, + v2->attrs, pos ); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); @@ -2352,11 +2367,10 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va Value * res[args[1]->listSize()]; unsigned int found = 0; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - Value & v2(*args[1]->listElems()[n]); - state.forceAttrs(v2, pos); - Bindings::iterator i = v2.attrs->find(attrName); - if (i != v2.attrs->end()) + for (auto v2 : args[1]->listItems()) { + state.forceAttrs(*v2, pos); + Bindings::iterator i = v2->attrs->find(attrName); + if (i != v2->attrs->end()) res[found++] = i->value; } @@ -2631,8 +2645,8 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value { bool res = false; state.forceList(*args[1], pos); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) - if (state.eqValues(*args[0], *args[1]->listElems()[n])) { + for (auto elem : args[1]->listItems()) + if (state.eqValues(*args[0], *elem)) { res = true; break; } @@ -2691,11 +2705,10 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, if (args[2]->listSize()) { Value * vCur = args[1]; - for (unsigned int n = 0; n < args[2]->listSize(); ++n) { - Value vTmp; - state.callFunction(*args[0], *vCur, vTmp, pos); + for (auto [n, elem] : enumerate(args[2]->listItems())) { + Value * vs []{vCur, elem}; vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); - state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); + state.callFunction(*args[0], 2, vs, *vCur, pos); } state.forceValue(v, pos); } else { @@ -2709,9 +2722,9 @@ static RegisterPrimOp primop_foldlStrict({ .args = {"op", "nul", "list"}, .doc = R"( Reduce a list by applying a binary operator, from left to right, - e.g. `foldl’ op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) + e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) ...`. The operator is applied strictly, i.e., its arguments are - evaluated first. For example, `foldl’ (x: y: x + y) 0 [1 2 3]` + evaluated first. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. )", .fun = prim_foldlStrict, @@ -2723,8 +2736,8 @@ static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * arg state.forceList(*args[1], pos); Value vTmp; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos); + for (auto elem : args[1]->listItems()) { + state.callFunction(*args[0], *elem, vTmp, pos); bool res = state.forceBool(vTmp, pos); if (res == any) { mkBool(v, any); @@ -2816,17 +2829,16 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value v.listElems()[n] = args[1]->listElems()[n]; } - auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) - return CompareValues()(a, b); + return CompareValues(state)(a, b); - Value vTmp1, vTmp2; - state.callFunction(*args[0], *a, vTmp1, pos); - state.callFunction(vTmp1, *b, vTmp2, pos); - return state.forceBool(vTmp2, pos); + Value * vs[] = {a, b}; + Value vBool; + state.callFunction(*args[0], 2, vs, vBool, pos); + return state.forceBool(vBool, pos); }; /* FIXME: std::sort can segfault if the comparator is not a strict @@ -2916,6 +2928,56 @@ static RegisterPrimOp primop_partition({ .fun = prim_partition, }); +static void prim_groupBy(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + ValueVectorMap attrs; + + for (auto vElem : args[1]->listItems()) { + Value res; + state.callFunction(*args[0], *vElem, res, pos); + string name = state.forceStringNoCtx(res, pos); + Symbol sym = state.symbols.create(name); + auto vector = attrs.try_emplace(sym, ValueVector()).first; + vector->second.push_back(vElem); + } + + state.mkAttrs(v, attrs.size()); + + for (auto & i : attrs) { + Value * list = state.allocAttr(v, i.first); + auto size = i.second.size(); + state.mkList(*list, size); + memcpy(list->listElems(), i.second.data(), sizeof(Value *) * size); + } +} + +static RegisterPrimOp primop_groupBy({ + .name = "__groupBy", + .args = {"f", "list"}, + .doc = R"( + Groups elements of *list* together by the string returned from the + function *f* called on each element. It returns an attribute set + where each attribute value contains the elements of *list* that are + mapped to the same corresponding attribute name returned by *f*. + + For example, + + ```nix + builtins.groupBy (builtins.substring 0 1) ["foo" "bar" "baz"] + ``` + + evaluates to + + ```nix + { b = [ "bar" "baz" ]; f = [ "foo" ]; } + ``` + )", + .fun = prim_groupBy, +}); + static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceFunction(*args[0], pos); @@ -3103,7 +3165,7 @@ static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Va { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - CompareValues comp; + CompareValues comp{state}; mkBool(v, comp(args[0], args[1])); } @@ -3454,9 +3516,9 @@ static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * * res.reserve((args[1]->listSize() + 32) * sep.size()); bool first = true; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + for (auto elem : args[1]->listItems()) { if (first) first = false; else res += sep; - res += state.coerceToString(pos, *args[1]->listElems()[n], context); + res += state.coerceToString(pos, *elem, context); } mkString(v, res, context); @@ -3485,14 +3547,14 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar vector<string> from; from.reserve(args[0]->listSize()); - for (unsigned int n = 0; n < args[0]->listSize(); ++n) - from.push_back(state.forceString(*args[0]->listElems()[n], pos)); + for (auto elem : args[0]->listItems()) + from.push_back(state.forceString(*elem, pos)); vector<std::pair<string, PathSet>> to; to.reserve(args[1]->listSize()); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + for (auto elem : args[1]->listItems()) { PathSet ctx; - auto s = state.forceString(*args[1]->listElems()[n], ctx, pos); + auto s = state.forceString(*elem, ctx, pos); to.push_back(std::make_pair(std::move(s), std::move(ctx))); } @@ -3693,7 +3755,7 @@ void EvalState::createBaseEnv() language feature gets added. It's not necessary to increase it when primops get added, because you can just use `builtins ? primOp' to check. */ - mkInt(v, 5); + mkInt(v, 6); addConstant("__langVersion", v); // Miscellaneous @@ -3720,21 +3782,27 @@ void EvalState::createBaseEnv() .fun = primOp.fun, .arity = std::max(primOp.args.size(), primOp.arity), .name = symbols.create(primOp.name), - .args = std::move(primOp.args), + .args = primOp.args, .doc = primOp.doc, }); /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ sDerivationNix = symbols.create("//builtin/derivation.nix"); - eval(parse( - #include "primops/derivation.nix.gen.hh" - , foFile, sDerivationNix, "/", staticBaseEnv), v); - addConstant("derivation", v); + auto vDerivation = allocValue(); + addConstant("derivation", vDerivation); /* Now that we've added all primops, sort the `builtins' set, because attribute lookups expect it to be sorted. */ baseEnv.values[0]->attrs->sort(); + + staticBaseEnv.sort(); + + /* Note: we have to initialize the 'derivation' constant *after* + building baseEnv/staticBaseEnv because it uses 'builtins'. */ + eval(parse( + #include "primops/derivation.nix.gen.hh" + , foFile, sDerivationNix, "/", staticBaseEnv), *vDerivation); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 31cf812b4..20545afd0 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -118,9 +118,8 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); state.mkList(outputsVal, info.second.outputs.size()); size_t i = 0; - for (const auto & output : info.second.outputs) { + for (const auto & output : info.second.outputs) mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); - } } infoVal.attrs->sort(); } @@ -181,8 +180,8 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg .errPos = *i.pos }); } - for (unsigned int n = 0; n < iter->value->listSize(); ++n) { - auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); + for (auto elem : iter->value->listItems()) { + auto name = state.forceStringNoCtx(*elem, *iter->pos); context.insert("!" + name + "!" + string(i.name)); } } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e6becdafc..079513873 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -74,7 +74,10 @@ std::string fixURI(std::string uri, EvalState & state, const std::string & defau std::string fixURIForGit(std::string uri, EvalState & state) { - static std::regex scp_uri("([^/].*)@(.*):(.*)"); + /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + * them by removing the `:` and assuming a scheme of `ssh://` + * */ + static std::regex scp_uri("([^/]*)@(.*):(.*)"); if (uri[0] != '/' && std::regex_match(uri, scp_uri)) return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); else diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index bfea24d40..517da4c01 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -10,11 +10,11 @@ namespace nix { void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context) + Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context) { checkInterrupt(); - if (strict) state.forceValue(v); + if (strict) state.forceValue(v, pos); switch (v.type()) { @@ -40,7 +40,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nAttrs: { - auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); + auto maybeString = state.tryAttrsToString(pos, v, context, false, false); if (maybeString) { out.write(*maybeString); break; @@ -54,18 +54,18 @@ void printValueAsJSON(EvalState & state, bool strict, for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); auto placeholder(obj.placeholder(j)); - printValueAsJSON(state, strict, *a.value, placeholder, context); + printValueAsJSON(state, strict, *a.value, *a.pos, placeholder, context); } } else - printValueAsJSON(state, strict, *i->value, out, context); + printValueAsJSON(state, strict, *i->value, *i->pos, out, context); break; } case nList: { auto list(out.list()); - for (unsigned int n = 0; n < v.listSize(); ++n) { + for (auto elem : v.listItems()) { auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context); + printValueAsJSON(state, strict, *elem, pos, placeholder, context); } break; } @@ -79,18 +79,20 @@ void printValueAsJSON(EvalState & state, bool strict, break; case nThunk: - throw TypeError("cannot convert %1% to JSON", showType(v)); - case nFunction: - throw TypeError("cannot convert %1% to JSON", showType(v)); + auto e = TypeError({ + .msg = hintfmt("cannot convert %1% to JSON", showType(v)), + .errPos = v.determinePos(pos) + }); + throw e.addTrace(pos, hintfmt("message for the trace")); } } void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context) + Value & v, const Pos & pos, std::ostream & str, PathSet & context) { JSONPlaceholder out(str); - printValueAsJSON(state, strict, v, out, context); + printValueAsJSON(state, strict, v, pos, out, context); } void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 67fed6487..c2f797b29 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { class JSONPlaceholder; void printValueAsJSON(EvalState & state, bool strict, - Value & v, JSONPlaceholder & out, PathSet & context); + Value & v, const Pos & pos, JSONPlaceholder & out, PathSet & context); void printValueAsJSON(EvalState & state, bool strict, - Value & v, std::ostream & str, PathSet & context); + Value & v, const Pos & pos, std::ostream & str, PathSet & context); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index b44455f5f..a875f82d7 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -18,7 +18,8 @@ static XMLAttrs singletonAttrs(const string & name, const string & value) static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos); static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) @@ -46,17 +47,18 @@ static void showAttrs(EvalState & state, bool strict, bool location, XMLOpenElement _(doc, "attr", xmlAttrs); printValueAsXML(state, strict, location, - *a.value, doc, context, drvsSeen); + *a.value, doc, context, drvsSeen, *a.pos); } } static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) { checkInterrupt(); - if (strict) state.forceValue(v); + if (strict) state.forceValue(v, pos); switch (v.type()) { @@ -91,14 +93,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, Path drvPath; a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); + if (strict) state.forceValue(*a->value, *a->pos); if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { - if (strict) state.forceValue(*a->value); + if (strict) state.forceValue(*a->value, *a->pos); if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->string.s; } @@ -120,8 +122,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nList: { XMLOpenElement _(doc, "list"); - for (unsigned int n = 0; n < v.listSize(); ++n) - printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); + for (auto v2 : v.listItems()) + printValueAsXML(state, strict, location, *v2, doc, context, drvsSeen, pos); break; } @@ -149,7 +151,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, } case nExternal: - v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); + v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); break; case nFloat: @@ -163,19 +165,20 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const + bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) const { doc.writeEmptyElement("unevaluated"); } void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context) + Value & v, std::ostream & out, PathSet & context, const Pos & pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); PathSet drvsSeen; - printValueAsXML(state, strict, location, v, doc, context, drvsSeen); + printValueAsXML(state, strict, location, v, doc, context, drvsSeen, pos); } diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index 97657327e..cc778a2cb 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -9,6 +9,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context); - + Value & v, std::ostream & out, PathSet & context, const Pos & pos); + } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index a1f131f9e..6b4f3c0ae 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,5 +1,7 @@ #pragma once +#include <cassert> + #include "symbol-table.hh" #if HAVE_BOEHMGC @@ -94,7 +96,8 @@ class ExternalValueBase /* Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const; + XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + const Pos & pos) const; virtual ~ExternalValueBase() { @@ -349,6 +352,34 @@ public: bool isTrivial() const; std::vector<std::pair<Path, std::string>> getContext(); + + auto listItems() + { + struct ListIterable + { + typedef Value * const * iterator; + iterator _begin, _end; + iterator begin() const { return _begin; } + iterator end() const { return _end; } + }; + assert(isList()); + auto begin = listElems(); + return ListIterable { begin, begin + listSize() }; + } + + auto listItems() const + { + struct ConstListIterable + { + typedef const Value * const * iterator; + iterator _begin, _end; + iterator begin() const { return _begin; } + iterator end() const { return _end; } + }; + assert(isList()); + auto begin = listElems(); + return ConstListIterable { begin, begin + listSize() }; + } }; @@ -394,9 +425,11 @@ void mkPath(Value & v, const char * s); #if HAVE_BOEHMGC typedef std::vector<Value *, traceable_allocator<Value *> > ValueVector; typedef std::map<Symbol, Value *, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, Value *> > > ValueMap; +typedef std::map<Symbol, ValueVector, std::less<Symbol>, traceable_allocator<std::pair<const Symbol, ValueVector> > > ValueVectorMap; #else typedef std::vector<Value *> ValueVector; typedef std::map<Symbol, Value *> ValueMap; +typedef std::map<Symbol, ValueVector> ValueVectorMap; #endif diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8468d2afc..544d2ffbf 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -51,7 +51,7 @@ struct GitInputScheme : InputScheme for (auto &[name, value] : url.query) { if (name == "rev" || name == "ref") attrs.emplace(name, value); - else if (name == "shallow") + else if (name == "shallow" || name == "submodules") attrs.emplace(name, Explicit<bool> { value == "1" }); else url2.query.emplace(name, value); @@ -324,17 +324,13 @@ struct GitInputScheme : InputScheme Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); repoDir = cacheDir; - Path cacheDirLock = cacheDir + ".lock"; createDirs(dirOf(cacheDir)); - AutoCloseFD lock = openLockFile(cacheDirLock, true); - lockFile(lock.get(), ltWrite, true); + PathLocks cacheDirLock({cacheDir + ".lock"}); if (!pathExists(cacheDir)) { runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); } - deleteLockFile(cacheDirLock, lock.get()); - Path localRefFile = input.getRef()->compare(0, 5, "refs/") == 0 ? cacheDir + "/" + *input.getRef() @@ -399,6 +395,8 @@ struct GitInputScheme : InputScheme if (!input.getRev()) input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + + // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ffc44e9e2..1c539b80e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -300,7 +300,7 @@ struct GitLabInputScheme : GitArchiveInputScheme if ("PAT" == token.substr(0, fldsplit)) return std::make_pair("Private-token", token.substr(fldsplit+1)); warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit)); - return std::nullopt; + return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1)); } Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index fb5702c4c..07e543c53 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -97,7 +97,7 @@ struct PathInputScheme : InputScheme // for security, ensure that if the parent is a store path, it's inside it if (store->isInStore(parent)) { auto storePath = store->printStorePath(store->toStorePath(parent).first); - if (!isInDir(absPath, storePath)) + if (!isDirOrInDir(absPath, storePath)) throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); } } else diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index b2a6e2a82..f4306ab91 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -11,7 +11,7 @@ namespace nix { -static std::string getS(const std::vector<Logger::Field> & fields, size_t n) +static std::string_view getS(const std::vector<Logger::Field> & fields, size_t n) { assert(n < fields.size()); assert(fields[n].type == Logger::Field::tString); @@ -103,17 +103,19 @@ public: ~ProgressBar() { stop(); - updateThread.join(); } void stop() override { - auto state(state_.lock()); - if (!state->active) return; - state->active = false; - writeToStderr("\r\e[K"); - updateCV.notify_one(); - quitCV.notify_one(); + { + auto state(state_.lock()); + if (!state->active) return; + state->active = false; + writeToStderr("\r\e[K"); + updateCV.notify_one(); + quitCV.notify_one(); + } + updateThread.join(); } bool isVerbose() override { diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 85f9f0d58..4404e0195 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -15,9 +15,14 @@ #include <sys/stat.h> #include <unistd.h> #include <signal.h> -#include <sys/types.h> -#include <sys/socket.h> -#include <netdb.h> +#ifdef __linux__ +#include <features.h> +#endif +#ifdef __GLIBC__ +#include <gnu/lib-names.h> +#include <nss.h> +#include <dlfcn.h> +#endif #include <openssl/crypto.h> @@ -121,21 +126,30 @@ static void preloadNSS() { been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to load its lookup libraries in the parent before any child gets a chance to. */ std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN. - We need the resolution to be done locally, as nscd socket will not be accessible in the - sandbox. */ - char * previous_env = getenv("LOCALDOMAIN"); - setenv("LOCALDOMAIN", "invalid", 1); - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) { - if (res) freeaddrinfo(res); - } - if (previous_env) { - setenv("LOCALDOMAIN", previous_env, 1); - } else { - unsetenv("LOCALDOMAIN"); +#ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ + if (dlopen (LIBNSS_DNS_SO, RTLD_NOW) == NULL) { + printMsg(Verbosity::lvlWarn, fmt("Unable to load nss_dns backend")); } + __nss_configure_lookup ("hosts", "dns"); +#endif }); } @@ -413,7 +427,7 @@ RunPager::RunPager() }); pid.setKillSignal(SIGINT); - + stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); } @@ -424,7 +438,7 @@ RunPager::~RunPager() try { if (pid != -1) { std::cout.flush(); - close(STDOUT_FILENO); + dup2(stdout, STDOUT_FILENO); pid.wait(); } } catch (...) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 05277d90a..ed012959b 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -88,6 +88,7 @@ public: private: Pid pid; + int stdout; }; extern volatile ::sig_atomic_t blockInt; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 8fce94264..13c086a46 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -111,15 +111,15 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo) upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - std::string hashPart(narInfo->path.hashPart()); - { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) }); + state_->pathInfoCache.upsert( + std::string(narInfo->path.to_string()), + PathInfoCacheValue { .value = std::shared_ptr<NarInfo>(narInfo) }); } if (diskCache) - diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo)); + diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo)); } AutoCloseFD openFile(const Path & path) @@ -308,16 +308,17 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, nar.first, name), + makeFixedOutputPath(method, nar.first, name, references), nar.first, }; info.narSize = nar.second; + info.references = references; return info; })->path; } @@ -385,7 +386,7 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default @@ -404,10 +405,11 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath }); return addToStoreCommon(*source, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { - makeFixedOutputPath(method, h, name), + makeFixedOutputPath(method, h, name, references), nar.first, }; info.narSize = nar.second; + info.references = references; info.ca = FixedOutputHash { .method = method, .hash = h, @@ -437,40 +439,29 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s })->path; } -std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id) +void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { - if (diskCache) { - auto [cacheOutcome, maybeCachedRealisation] = - diskCache->lookupRealisation(getUri(), id); - switch (cacheOutcome) { - case NarInfoDiskCache::oValid: - debug("Returning a cached realisation for %s", id.to_string()); - return *maybeCachedRealisation; - case NarInfoDiskCache::oInvalid: - debug("Returning a cached missing realisation for %s", id.to_string()); - return {}; - case NarInfoDiskCache::oUnknown: - break; - } - } - auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; - auto rawOutputInfo = getFile(outputInfoFilePath); - if (rawOutputInfo) { - auto realisation = Realisation::fromJSON( - nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath); + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); - if (diskCache) - diskCache->upsertRealisation( - getUri(), realisation); + Callback<std::shared_ptr<std::string>> newCallback = { + [=](std::future<std::shared_ptr<std::string>> fut) { + try { + auto data = fut.get(); + if (!data) return (*callbackPtr)(nullptr); - return {realisation}; - } else { - if (diskCache) - diskCache->upsertAbsentRealisation(getUri(), id); - return std::nullopt; - } + auto realisation = Realisation::fromJSON( + nlohmann::json::parse(*data), outputInfoFilePath); + return (*callbackPtr)(std::make_shared<const Realisation>(realisation)); + } catch (...) { + callbackPtr->rethrow(); + } + } + }; + + getFile(outputInfoFilePath, std::move(newCallback)); } void BinaryCacheStore::registerDrvOutput(const Realisation& info) { diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 723f2e805..9815af591 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -97,18 +97,19 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references ) override; StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override; + PathFilter & filter, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; void registerDrvOutput(const Realisation & info) override; - std::optional<const Realisation> queryRealisation(const DrvOutput &) override; + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; void narFromPath(const StorePath & path, Sink & sink) override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b924d23b2..60945403e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -655,7 +655,7 @@ void DerivationGoal::tryLocalBuild() { throw Error( "unable to build with a primary store that isn't a local store; " "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); } diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index be270d079..b9602e696 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -1,6 +1,8 @@ #include "drv-output-substitution-goal.hh" +#include "finally.hh" #include "worker.hh" #include "substitution-goal.hh" +#include "callback.hh" namespace nix { @@ -50,14 +52,42 @@ void DrvOutputSubstitutionGoal::tryNext() return; } - auto sub = subs.front(); + sub = subs.front(); subs.pop_front(); // FIXME: Make async - outputInfo = sub->queryRealisation(id); + // outputInfo = sub->queryRealisation(id); + outPipe.create(); + promise = decltype(promise)(); + + sub->queryRealisation( + id, { [&](std::future<std::shared_ptr<const Realisation>> res) { + try { + Finally updateStats([this]() { outPipe.writeSide.close(); }); + promise.set_value(res.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + } }); + + worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + + state = &DrvOutputSubstitutionGoal::realisationFetched; +} + +void DrvOutputSubstitutionGoal::realisationFetched() +{ + worker.childTerminated(this); + + try { + outputInfo = promise.get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + substituterFailed = true; + } + if (!outputInfo) { - tryNext(); - return; + return tryNext(); } for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { @@ -119,4 +149,10 @@ void DrvOutputSubstitutionGoal::work() (this->*state)(); } +void DrvOutputSubstitutionGoal::handleEOF(int fd) +{ + if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); +} + + } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 63ab53d89..67ae2624a 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -3,6 +3,8 @@ #include "store-api.hh" #include "goal.hh" #include "realisation.hh" +#include <thread> +#include <future> namespace nix { @@ -20,11 +22,18 @@ private: // The realisation corresponding to the given output id. // Will be filled once we can get it. - std::optional<Realisation> outputInfo; + std::shared_ptr<const Realisation> outputInfo; /* The remaining substituters. */ std::list<ref<Store>> subs; + /* The current substituter. */ + std::shared_ptr<Store> sub; + + Pipe outPipe; + std::thread thr; + std::promise<std::shared_ptr<const Realisation>> promise; + /* Whether a substituter failed. */ bool substituterFailed = false; @@ -36,6 +45,7 @@ public: void init(); void tryNext(); + void realisationFetched(); void outPathValid(); void finished(); @@ -44,7 +54,7 @@ public: string key() override; void work() override; - + void handleEOF(int fd) override; }; } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 065efc855..9b4cfd835 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,4 +1,3 @@ -#include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" #include "derivation-goal.hh" diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index fab6c3a08..c9a4a31e7 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -342,7 +342,7 @@ int childEntry(void * arg) return 1; } - +#if __linux__ static void linkOrCopy(const Path & from, const Path & to) { if (link(from.c_str(), to.c_str()) == -1) { @@ -358,6 +358,7 @@ static void linkOrCopy(const Path & from, const Path & to) copyPath(from, to); } } +#endif void LocalDerivationGoal::startBuilder() @@ -917,7 +918,9 @@ void LocalDerivationGoal::startBuilder() } else #endif { +#if __linux__ fallback: +#endif pid = startProcess([&]() { runChild(); }); @@ -1179,7 +1182,8 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1198,9 +1202,10 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo } StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) override { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair); + auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); goal.addDependency(path); return path; } @@ -1224,13 +1229,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // corresponds to an allowed derivation { throw Error("registerDrvOutput"); } - std::optional<const Realisation> queryRealisation(const DrvOutput & id) override + void queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override // XXX: This should probably be allowed if the realisation corresponds to // an allowed derivation { if (!goal.isAllowed(id)) - throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string()); - return next->queryRealisation(id); + callback(nullptr); + next->queryRealisation(id, std::move(callback)); } void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override @@ -1353,7 +1359,7 @@ void LocalDerivationGoal::startDaemon() AutoCloseFD remote = accept(daemonSocket.get(), (struct sockaddr *) &remoteAddr, &remoteAddrLen); if (!remote) { - if (errno == EINTR) continue; + if (errno == EINTR || errno == EAGAIN) continue; if (errno == EINVAL) break; throw SysError("accepting connection"); } @@ -1991,7 +1997,7 @@ void LocalDerivationGoal::runChild() else if (drv->builder == "builtin:unpack-channel") builtinUnpackChannel(drv2); else - throw Error("unsupported builtin function '%1%'", string(drv->builder, 8)); + throw Error("unsupported builtin builder '%1%'", string(drv->builder, 8)); _exit(0); } catch (std::exception & e) { writeFull(STDERR_FILENO, e.what() + std::string("\n")); diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 55afb5cca..f11c5ce68 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -281,11 +281,11 @@ void Worker::run(const Goals & _topGoals) if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); else throw Error("unable to start any build; remote machines may not have " "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); } assert(!awake.empty()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 974d1c471..cf32ccdc4 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -120,8 +120,10 @@ ContentAddress parseContentAddress(std::string_view rawCa) { ContentAddressMethod parseContentAddressMethod(std::string_view caMethod) { - std::string_view asPrefix {std::string{caMethod} + ":"}; - return parseContentAddressMethodPrefix(asPrefix); + std::string asPrefix = std::string{caMethod} + ":"; + // parseContentAddressMethodPrefix takes its argument by reference + std::string_view asPrefixView = asPrefix; + return parseContentAddressMethodPrefix(asPrefixView); } std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4b3465da3..bafab6fd5 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -403,9 +403,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store, return store->queryPathInfo(path); }, [&](FixedOutputHashMethod & fohm) { - if (!refs.empty()) - throw UnimplementedError("cannot yet have refs with flat or nar-hashed data"); - auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair); + auto path = store->addToStoreFromDump(source, name, fohm.fileIngestionMethod, fohm.hashType, repair, refs); return store->queryPathInfo(path); }, }, contentAddressMethod); @@ -433,25 +431,30 @@ static void performOp(TunnelLogger * logger, ref<Store> store, hashAlgo = parseHashType(hashAlgoRaw); } - StringSink saved; - TeeSource savedNARSource(from, saved); - RetrieveRegularNARSink savedRegular { saved }; - - if (method == FileIngestionMethod::Recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNARSource); - } else - parseDump(savedRegular, from); - + auto dumpSource = sinkToSource([&](Sink & saved) { + if (method == FileIngestionMethod::Recursive) { + /* We parse the NAR dump through into `saved` unmodified, + so why all this extra work? We still parse the NAR so + that we aren't sending arbitrary data to `saved` + unwittingly`, and we know when the NAR ends so we don't + consume the rest of `from` and can't parse another + command. (We don't trust `addToStoreFromDump` to not + eagerly consume the entire stream it's given, past the + length of the Nar. */ + TeeSource savedNARSource(from, saved); + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNARSource); + } else { + /* Incrementally parse the NAR file, stripping the + metadata, and streaming the sole file we expect into + `saved`. */ + RetrieveRegularNARSink savedRegular { saved }; + parseDump(savedRegular, from); + if (!savedRegular.regular) throw Error("regular file expected"); + } + }); logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - - // FIXME: try to stream directly from `from`. - StringSource dumpSource { *saved.s }; - auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); + auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -625,9 +628,9 @@ static void performOp(TunnelLogger * logger, ref<Store> store, break; } + // Obsolete. case wopSyncWithGC: { logger->startWork(); - store->syncWithGC(); logger->stopWork(); to << 1; break; @@ -953,7 +956,7 @@ void processConnection( Finally finally([&]() { _isInterrupted = false; - prevLogger->log(lvlDebug, fmt("%d operations", opCount)); + printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { @@ -986,6 +989,8 @@ void processConnection( break; } + printMsgUsing(prevLogger, lvlDebug, "received daemon op %d", op); + opCount++; try { diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 36c6e725c..62dc21c59 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -50,8 +50,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } - std::optional<const Realisation> queryRealisation(const DrvOutput&) override - { unsupported("queryRealisation"); } + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override + { callback(nullptr); } }; static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 37e17b397..4621a8217 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -544,6 +544,14 @@ struct curlFileTransfer : public FileTransfer stopWorkerThread(); }); +#ifdef __linux__ + /* Cause this thread to not share any FS attributes with the main thread, + because this causes setns() in restoreMountNamespace() to fail. + Ideally, this would happen in the std::thread() constructor. */ + if (unshare(CLONE_FS) != 0) + throw SysError("unsharing filesystem state in download thread"); +#endif + std::map<CURL *, std::shared_ptr<TransferItem>> items; bool quit = false; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 5a62c6529..7a414da6b 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -10,48 +10,22 @@ #include <regex> #include <random> -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/statvfs.h> +#include <climits> #include <errno.h> #include <fcntl.h> +#include <poll.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/types.h> +#include <sys/un.h> #include <unistd.h> -#include <climits> namespace nix { -static string gcLockName = "gc.lock"; -static string gcRootsDir = "gcroots"; - - -/* Acquire the global GC lock. This is used to prevent new Nix - processes from starting after the temporary root files have been - read. To be precise: when they try to create a new temporary root - file, they will block until the garbage collector has finished / - yielded the GC lock. */ -AutoCloseFD LocalStore::openGCLock(LockType lockType) -{ - Path fnGCLock = (format("%1%/%2%") - % stateDir % gcLockName).str(); - - debug(format("acquiring global GC lock '%1%'") % fnGCLock); - - AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); - if (!fdGCLock) - throw SysError("opening global GC lock '%1%'", fnGCLock); - - if (!lockFile(fdGCLock.get(), lockType, false)) { - printInfo("waiting for the big garbage collector lock..."); - lockFile(fdGCLock.get(), lockType, true); - } - - /* !!! Restrict read permission on the GC root. Otherwise any - process that can open the file for reading can DoS the - collector. */ - - return fdGCLock; -} +static std::string gcSocketPath = "/gc-socket/socket"; +static std::string gcRootsDir = "gcroots"; static void makeSymlink(const Path & link, const Path & target) @@ -71,12 +45,6 @@ static void makeSymlink(const Path & link, const Path & target) } -void LocalStore::syncWithGC() -{ - AutoCloseFD fdGCLock = openGCLock(ltRead); -} - - void LocalStore::addIndirectRoot(const Path & path) { string hash = hashString(htSHA1, path).to_string(Base32, false); @@ -95,6 +63,12 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot "creating a garbage collector root (%1%) in the Nix store is forbidden " "(are you running nix-build inside the store?)", gcRoot); + /* Register this root with the garbage collector, if it's + running. This should be superfluous since the caller should + have registered this root yet, but let's be on the safe + side. */ + addTempRoot(storePath); + /* Don't clobber the link if it already exists and doesn't point to the Nix store. */ if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) @@ -102,11 +76,6 @@ Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot makeSymlink(gcRoot, printStorePath(storePath)); addIndirectRoot(gcRoot); - /* Grab the global GC root, causing us to block while a GC is in - progress. This prevents the set of permanent roots from - increasing while a GC is in progress. */ - syncWithGC(); - return gcRoot; } @@ -119,8 +88,6 @@ void LocalStore::addTempRoot(const StorePath & path) if (!state->fdTempRoots) { while (1) { - AutoCloseFD fdGCLock = openGCLock(ltRead); - if (pathExists(fnTempRoots)) /* It *must* be stale, since there can be no two processes with the same pid. */ @@ -128,10 +95,8 @@ void LocalStore::addTempRoot(const StorePath & path) state->fdTempRoots = openLockFile(fnTempRoots, true); - fdGCLock = -1; - - debug(format("acquiring read lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltRead, true); + debug("acquiring write lock on '%s'", fnTempRoots); + lockFile(state->fdTempRoots.get(), ltWrite, true); /* Check whether the garbage collector didn't get in our way. */ @@ -147,24 +112,55 @@ void LocalStore::addTempRoot(const StorePath & path) } - /* Upgrade the lock to a write lock. This will cause us to block - if the garbage collector is holding our lock. */ - debug(format("acquiring write lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltWrite, true); + if (!state->fdGCLock) + state->fdGCLock = openGCLock(); + + restart: + FdLock gcLock(state->fdGCLock.get(), ltRead, false, ""); + + if (!gcLock.acquired) { + /* We couldn't get a shared global GC lock, so the garbage + collector is running. So we have to connect to the garbage + collector and inform it about our root. */ + if (!state->fdRootsSocket) { + auto socketPath = stateDir.get() + gcSocketPath; + debug("connecting to '%s'", socketPath); + state->fdRootsSocket = createUnixDomainSocket(); + nix::connect(state->fdRootsSocket.get(), socketPath); + } + + try { + debug("sending GC root '%s'", printStorePath(path)); + writeFull(state->fdRootsSocket.get(), printStorePath(path) + "\n", false); + char c; + readFull(state->fdRootsSocket.get(), &c, 1); + assert(c == '1'); + debug("got ack for GC root '%s'", printStorePath(path)); + } catch (SysError & e) { + /* The garbage collector may have exited, so we need to + restart. */ + if (e.errNo == EPIPE) { + debug("GC socket disconnected"); + state->fdRootsSocket.close(); + goto restart; + } + } catch (EndOfFile & e) { + debug("GC socket disconnected"); + state->fdRootsSocket.close(); + goto restart; + } + } + /* Append the store path to the temporary roots file. */ string s = printStorePath(path) + '\0'; writeFull(state->fdTempRoots.get(), s); - - /* Downgrade to a read lock. */ - debug(format("downgrading to read lock on '%1%'") % fnTempRoots); - lockFile(state->fdTempRoots.get(), ltRead, true); } static std::string censored = "{censored}"; -void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) +void LocalStore::findTempRoots(Roots & tempRoots, bool censor) { /* Read the `temproots' directory for per-process temporary root files. */ @@ -179,35 +175,25 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) pid_t pid = std::stoi(i.name); debug(format("reading temporary root file '%1%'") % path); - FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666))); - if (!*fd) { + AutoCloseFD fd(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666)); + if (!fd) { /* It's okay if the file has disappeared. */ if (errno == ENOENT) continue; throw SysError("opening temporary roots file '%1%'", path); } - /* This should work, but doesn't, for some reason. */ - //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); - //if (*fd == -1) continue; - /* Try to acquire a write lock without blocking. This can only succeed if the owning process has died. In that case we don't care about its temporary roots. */ - if (lockFile(fd->get(), ltWrite, false)) { + if (lockFile(fd.get(), ltWrite, false)) { printInfo("removing stale temporary roots file '%1%'", path); unlink(path.c_str()); - writeFull(fd->get(), "d"); + writeFull(fd.get(), "d"); continue; } - /* Acquire a read lock. This will prevent the owning process - from upgrading to a write lock, therefore it will block in - addTempRoot(). */ - debug(format("waiting for read lock on '%1%'") % path); - lockFile(fd->get(), ltRead, true); - /* Read the entire file. */ - string contents = readFile(fd->get()); + string contents = readFile(fd.get()); /* Extract the roots. */ string::size_type pos = 0, end; @@ -218,8 +204,6 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); pos = end + 1; } - - fds.push_back(fd); /* keep open */ } } @@ -304,8 +288,7 @@ Roots LocalStore::findRoots(bool censor) Roots roots; findRootsNoTemp(roots, censor); - FDs fds; - findTempRoots(fds, roots, censor); + findTempRoots(roots, censor); return roots; } @@ -341,6 +324,7 @@ static string quoteRegexChars(const string & raw) return std::regex_replace(raw, specialRegex, R"(\$&)"); } +#if __linux__ static void readFileRoots(const char * path, UncheckedRoots & roots) { try { @@ -350,6 +334,7 @@ static void readFileRoots(const char * path, UncheckedRoots & roots) throw; } } +#endif void LocalStore::findRuntimeRoots(Roots & roots, bool censor) { @@ -431,7 +416,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) } #endif -#if defined(__linux__) +#if __linux__ readFileRoots("/proc/sys/kernel/modprobe", unchecked); readFileRoots("/proc/sys/kernel/fbsplash", unchecked); readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); @@ -455,391 +440,397 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor) struct GCLimitReached { }; -struct LocalStore::GCState -{ - const GCOptions & options; - GCResults & results; - StorePathSet roots; - StorePathSet tempRoots; - StorePathSet dead; - StorePathSet alive; - bool gcKeepOutputs; - bool gcKeepDerivations; - uint64_t bytesInvalidated; - bool moveToTrash = true; - bool shouldDelete; - GCState(const GCOptions & options, GCResults & results) - : options(options), results(results), bytesInvalidated(0) { } -}; - - -bool LocalStore::isActiveTempFile(const GCState & state, - const Path & path, const string & suffix) +void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) { - return hasSuffix(path, suffix) - && state.tempRoots.count(parseStorePath(string(path, 0, path.size() - suffix.size()))); -} + bool shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + bool gcKeepOutputs = settings.gcKeepOutputs; + bool gcKeepDerivations = settings.gcKeepDerivations; + StorePathSet roots, dead, alive; -void LocalStore::deleteGarbage(GCState & state, const Path & path) -{ - uint64_t bytesFreed; - deletePath(path, bytesFreed); - state.results.bytesFreed += bytesFreed; -} + struct Shared + { + // The temp roots only store the hash part to make it easier to + // ignore suffixes like '.lock', '.chroot' and '.check'. + std::unordered_set<std::string> tempRoots; + // Hash part of the store path currently being deleted, if + // any. + std::optional<std::string> pending; + }; -void LocalStore::deletePathRecursive(GCState & state, const Path & path) -{ - checkInterrupt(); - - uint64_t size = 0; - - auto storePath = maybeParseStorePath(path); - if (storePath && isValidPath(*storePath)) { - StorePathSet referrers; - queryReferrers(*storePath, referrers); - for (auto & i : referrers) - if (printStorePath(i) != path) deletePathRecursive(state, printStorePath(i)); - size = queryPathInfo(*storePath)->narSize; - invalidatePathChecked(*storePath); - } + Sync<Shared> _shared; - Path realPath = realStoreDir + "/" + std::string(baseNameOf(path)); + std::condition_variable wakeup; - struct stat st; - if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT) return; - throw SysError("getting status of %1%", realPath); + /* Using `--ignore-liveness' with `--delete' can have unintended + consequences if `keep-outputs' or `keep-derivations' are true + (the garbage collector will recurse into deleting the outputs + or derivers, respectively). So disable them. */ + if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + gcKeepOutputs = false; + gcKeepDerivations = false; } - printInfo(format("deleting '%1%'") % path); - - state.results.paths.insert(path); + if (shouldDelete) + deletePath(reservedPath); - /* If the path is not a regular file or symlink, move it to the - trash directory. The move is to ensure that later (when we're - not holding the global GC lock) we can delete the path without - being afraid that the path has become alive again. Otherwise - delete it right away. */ - if (state.moveToTrash && S_ISDIR(st.st_mode)) { - // Estimate the amount freed using the narSize field. FIXME: - // if the path was not valid, need to determine the actual - // size. - try { - if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) - throw SysError("making '%1%' writable", realPath); - Path tmp = trashDir + "/" + std::string(baseNameOf(path)); - if (rename(realPath.c_str(), tmp.c_str())) - throw SysError("unable to rename '%1%' to '%2%'", realPath, tmp); - state.bytesInvalidated += size; - } catch (SysError & e) { - if (e.errNo == ENOSPC) { - printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg()); - deleteGarbage(state, realPath); + /* Acquire the global GC root. Note: we don't use fdGCLock + here because then in auto-gc mode, another thread could + downgrade our exclusive lock. */ + auto fdGCLock = openGCLock(); + FdLock gcLock(fdGCLock.get(), ltWrite, true, "waiting for the big garbage collector lock..."); + + /* Start the server for receiving new roots. */ + auto socketPath = stateDir.get() + gcSocketPath; + createDirs(dirOf(socketPath)); + auto fdServer = createUnixDomainSocket(socketPath, 0666); + + if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) | O_NONBLOCK) == -1) + throw SysError("making socket '%1%' non-blocking", socketPath); + + Pipe shutdownPipe; + shutdownPipe.create(); + + std::thread serverThread([&]() { + Sync<std::map<int, std::thread>> connections; + + Finally cleanup([&]() { + debug("GC roots server shutting down"); + while (true) { + auto item = remove_begin(*connections.lock()); + if (!item) break; + auto & [fd, thread] = *item; + shutdown(fd, SHUT_RDWR); + thread.join(); } - } - } else - deleteGarbage(state, realPath); - - if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { - printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); - throw GCLimitReached(); - } -} - - -bool LocalStore::canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path) -{ - if (visited.count(path)) return false; - - if (state.alive.count(path)) return true; - - if (state.dead.count(path)) return false; - - if (state.roots.count(path)) { - debug("cannot delete '%1%' because it's a root", printStorePath(path)); - state.alive.insert(path); - return true; - } - - visited.insert(path); - - if (!isValidPath(path)) return false; - - StorePathSet incoming; - - /* Don't delete this path if any of its referrers are alive. */ - queryReferrers(path, incoming); - - /* If keep-derivations is set and this is a derivation, then - don't delete the derivation if any of the outputs are alive. */ - if (state.gcKeepDerivations && path.isDerivation()) { - for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(path)) - if (maybeOutPath && - isValidPath(*maybeOutPath) && - queryPathInfo(*maybeOutPath)->deriver == path - ) - incoming.insert(*maybeOutPath); - } - - /* If keep-outputs is set, then don't delete this path if there - are derivers of this path that are not garbage. */ - if (state.gcKeepOutputs) { - auto derivers = queryValidDerivers(path); - for (auto & i : derivers) - incoming.insert(i); - } + }); + + while (true) { + std::vector<struct pollfd> fds; + fds.push_back({.fd = shutdownPipe.readSide.get(), .events = POLLIN}); + fds.push_back({.fd = fdServer.get(), .events = POLLIN}); + auto count = poll(fds.data(), fds.size(), -1); + assert(count != -1); + + if (fds[0].revents) + /* Parent is asking us to quit. */ + break; + + if (fds[1].revents) { + /* Accept a new connection. */ + assert(fds[1].revents & POLLIN); + AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr); + if (!fdClient) continue; + + /* Process the connection in a separate thread. */ + auto fdClient_ = fdClient.get(); + std::thread clientThread([&, fdClient = std::move(fdClient)]() { + Finally cleanup([&]() { + auto conn(connections.lock()); + auto i = conn->find(fdClient.get()); + if (i != conn->end()) { + i->second.detach(); + conn->erase(i); + } + }); + + while (true) { + try { + auto path = readLine(fdClient.get()); + auto storePath = maybeParseStorePath(path); + if (storePath) { + debug("got new GC root '%s'", path); + auto hashPart = std::string(storePath->hashPart()); + auto shared(_shared.lock()); + shared->tempRoots.insert(hashPart); + /* If this path is currently being + deleted, then we have to wait until + deletion is finished to ensure that + the client doesn't start + re-creating it before we're + done. FIXME: ideally we would use a + FD for this so we don't block the + poll loop. */ + while (shared->pending == hashPart) { + debug("synchronising with deletion of path '%s'", path); + shared.wait(wakeup); + } + } else + printError("received garbage instead of a root from client"); + writeFull(fdClient.get(), "1", false); + } catch (Error &) { break; } + } + }); - for (auto & i : incoming) - if (i != path) - if (canReachRoot(state, visited, i)) { - state.alive.insert(path); - return true; + connections.lock()->insert({fdClient_, std::move(clientThread)}); } + } + }); - return false; -} - - -void LocalStore::tryToDelete(GCState & state, const Path & path) -{ - checkInterrupt(); - - auto realPath = realStoreDir + "/" + std::string(baseNameOf(path)); - if (realPath == linksDir || realPath == trashDir) return; - - //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); - - auto storePath = maybeParseStorePath(path); - - if (!storePath || !isValidPath(*storePath)) { - /* A lock file belonging to a path that we're building right - now isn't garbage. */ - if (isActiveTempFile(state, path, ".lock")) return; + Finally stopServer([&]() { + writeFull(shutdownPipe.writeSide.get(), "x", false); + wakeup.notify_all(); + if (serverThread.joinable()) serverThread.join(); + }); - /* Don't delete .chroot directories for derivations that are - currently being built. */ - if (isActiveTempFile(state, path, ".chroot")) return; + /* Find the roots. Since we've grabbed the GC lock, the set of + permanent roots cannot increase now. */ + printInfo("finding garbage collector roots..."); + Roots rootMap; + if (!options.ignoreLiveness) + findRootsNoTemp(rootMap, true); - /* Don't delete .check directories for derivations that are - currently being built, because we may need to run - diff-hook. */ - if (isActiveTempFile(state, path, ".check")) return; - } + for (auto & i : rootMap) roots.insert(i.first); - StorePathSet visited; - - if (storePath && canReachRoot(state, visited, *storePath)) { - debug("cannot delete '%s' because it's still reachable", path); - } else { - /* No path we visited was a root, so everything is garbage. - But we only delete ‘path’ and its referrers here so that - ‘nix-store --delete’ doesn't have the unexpected effect of - recursing into derivations and outputs. */ - for (auto & i : visited) - state.dead.insert(i); - if (state.shouldDelete) - deletePathRecursive(state, path); + /* Read the temporary roots created before we acquired the global + GC root. Any new roots will be sent to our socket. */ + Roots tempRoots; + findTempRoots(tempRoots, true); + for (auto & root : tempRoots) { + _shared.lock()->tempRoots.insert(std::string(root.first.hashPart())); + roots.insert(root.first); } -} + /* Helper function that deletes a path from the store and throws + GCLimitReached if we've deleted enough garbage. */ + auto deleteFromStore = [&](std::string_view baseName) + { + Path path = storeDir + "/" + std::string(baseName); + Path realPath = realStoreDir + "/" + std::string(baseName); -/* Unlink all files in /nix/store/.links that have a link count of 1, - which indicates that there are no other links and so they can be - safely deleted. FIXME: race condition with optimisePath(): we - might see a link count of 1 just before optimisePath() increases - the link count. */ -void LocalStore::removeUnusedLinks(const GCState & state) -{ - AutoCloseDir dir(opendir(linksDir.c_str())); - if (!dir) throw SysError("opening directory '%1%'", linksDir); - - int64_t actualSize = 0, unsharedSize = 0; + printInfo("deleting '%1%'", path); - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir.get())) { - checkInterrupt(); - string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = linksDir + "/" + name; + results.paths.insert(path); - auto st = lstat(path); + uint64_t bytesFreed; + deletePath(realPath, bytesFreed); + results.bytesFreed += bytesFreed; - if (st.st_nlink != 1) { - actualSize += st.st_size; - unsharedSize += (st.st_nlink - 1) * st.st_size; - continue; + if (results.bytesFreed > options.maxFreed) { + printInfo("deleted more than %d bytes; stopping", options.maxFreed); + throw GCLimitReached(); } + }; - printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); - - if (unlink(path.c_str()) == -1) - throw SysError("deleting '%1%'", path); + std::map<StorePath, StorePathSet> referrersCache; - state.results.bytesFreed += st.st_size; - } + /* Helper function that visits all paths reachable from `start` + via the referrers edges and optionally derivers and derivation + output edges. If none of those paths are roots, then all + visited paths are garbage and are deleted. */ + auto deleteReferrersClosure = [&](const StorePath & start) { + StorePathSet visited; + std::queue<StorePath> todo; - struct stat st; - if (stat(linksDir.c_str(), &st) == -1) - throw SysError("statting '%1%'", linksDir); - int64_t overhead = st.st_blocks * 512ULL; + /* Wake up any GC client waiting for deletion of the paths in + 'visited' to finish. */ + Finally releasePending([&]() { + auto shared(_shared.lock()); + shared->pending.reset(); + wakeup.notify_all(); + }); - printInfo("note: currently hard linking saves %.2f MiB", - ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); -} + auto enqueue = [&](const StorePath & path) { + if (visited.insert(path).second) + todo.push(path); + }; + enqueue(start); -void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) -{ - GCState state(options, results); - state.gcKeepOutputs = settings.gcKeepOutputs; - state.gcKeepDerivations = settings.gcKeepDerivations; + while (auto path = pop(todo)) { + checkInterrupt(); - /* Using `--ignore-liveness' with `--delete' can have unintended - consequences if `keep-outputs' or `keep-derivations' are true - (the garbage collector will recurse into deleting the outputs - or derivers, respectively). So disable them. */ - if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { - state.gcKeepOutputs = false; - state.gcKeepDerivations = false; - } + /* Bail out if we've previously discovered that this path + is alive. */ + if (alive.count(*path)) { + alive.insert(start); + return; + } - state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + /* If we've previously deleted this path, we don't have to + handle it again. */ + if (dead.count(*path)) continue; - if (state.shouldDelete) - deletePath(reservedPath); + auto markAlive = [&]() + { + alive.insert(*path); + alive.insert(start); + try { + StorePathSet closure; + computeFSClosure(*path, closure); + for (auto & p : closure) + alive.insert(p); + } catch (InvalidPath &) { } + }; + + /* If this is a root, bail out. */ + if (roots.count(*path)) { + debug("cannot delete '%s' because it's a root", printStorePath(*path)); + return markAlive(); + } - /* Acquire the global GC root. This prevents - a) New roots from being added. - b) Processes from creating new temporary root files. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); + if (options.action == GCOptions::gcDeleteSpecific + && !options.pathsToDelete.count(*path)) + return; - /* Find the roots. Since we've grabbed the GC lock, the set of - permanent roots cannot increase now. */ - printInfo("finding garbage collector roots..."); - Roots rootMap; - if (!options.ignoreLiveness) - findRootsNoTemp(rootMap, true); + { + auto hashPart = std::string(path->hashPart()); + auto shared(_shared.lock()); + if (shared->tempRoots.count(hashPart)) { + debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); + return markAlive(); + } + shared->pending = hashPart; + } - for (auto & i : rootMap) state.roots.insert(i.first); + if (isValidPath(*path)) { - /* Read the temporary roots. This acquires read locks on all - per-process temporary root files. So after this point no paths - can be added to the set of temporary roots. */ - FDs fds; - Roots tempRoots; - findTempRoots(fds, tempRoots, true); - for (auto & root : tempRoots) { - state.tempRoots.insert(root.first); - state.roots.insert(root.first); - } + /* Visit the referrers of this path. */ + auto i = referrersCache.find(*path); + if (i == referrersCache.end()) { + StorePathSet referrers; + queryReferrers(*path, referrers); + referrersCache.emplace(*path, std::move(referrers)); + i = referrersCache.find(*path); + } + for (auto & p : i->second) + enqueue(p); + + /* If keep-derivations is set and this is a + derivation, then visit the derivation outputs. */ + if (gcKeepDerivations && path->isDerivation()) { + for (auto & [name, maybeOutPath] : queryPartialDerivationOutputMap(*path)) + if (maybeOutPath && + isValidPath(*maybeOutPath) && + queryPathInfo(*maybeOutPath)->deriver == *path) + enqueue(*maybeOutPath); + } - /* After this point the set of roots or temporary roots cannot - increase, since we hold locks on everything. So everything - that is not reachable from `roots' is garbage. */ + /* If keep-outputs is set, then visit the derivers. */ + if (gcKeepOutputs) { + auto derivers = queryValidDerivers(*path); + for (auto & i : derivers) + enqueue(i); + } + } + } - if (state.shouldDelete) { - if (pathExists(trashDir)) deleteGarbage(state, trashDir); - try { - createDirs(trashDir); - } catch (SysError & e) { - if (e.errNo == ENOSPC) { - printInfo("note: can't create trash directory: %s", e.msg()); - state.moveToTrash = false; + for (auto & path : topoSortPaths(visited)) { + if (!dead.insert(path).second) continue; + if (shouldDelete) { + invalidatePathChecked(path); + deleteFromStore(path.to_string()); + referrersCache.erase(path); } } - } + }; - /* Now either delete all garbage paths, or just the specified - paths (for gcDeleteSpecific). */ + /* Synchronisation point for testing, see tests/gc-concurrent.sh. */ + if (auto p = getEnv("_NIX_TEST_GC_SYNC")) + readFile(*p); + /* Either delete all garbage paths, or just the specified + paths (for gcDeleteSpecific). */ if (options.action == GCOptions::gcDeleteSpecific) { for (auto & i : options.pathsToDelete) { - tryToDelete(state, printStorePath(i)); - if (state.dead.find(i) == state.dead.end()) + deleteReferrersClosure(i); + if (!dead.count(i)) throw Error( - "cannot delete path '%1%' since it is still alive. " - "To find out why use: " + "Cannot delete path '%1%' since it is still alive. " + "To find out why, use: " "nix-store --query --roots", printStorePath(i)); } } else if (options.maxFreed > 0) { - if (state.shouldDelete) + if (shouldDelete) printInfo("deleting garbage..."); else printInfo("determining live/dead paths..."); try { - AutoCloseDir dir(opendir(realStoreDir.get().c_str())); if (!dir) throw SysError("opening directory '%1%'", realStoreDir); - /* Read the store and immediately delete all paths that - aren't valid. When using --max-freed etc., deleting - invalid paths is preferred over deleting unreachable - paths, since unreachable paths could become reachable - again. We don't use readDirectory() here so that GCing - can start faster. */ + /* Read the store and delete all paths that are invalid or + unreachable. We don't use readDirectory() here so that + GCing can start faster. */ + auto linksName = baseNameOf(linksDir); Paths entries; struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { checkInterrupt(); string name = dirent->d_name; - if (name == "." || name == "..") continue; - Path path = storeDir + "/" + name; - auto storePath = maybeParseStorePath(path); - if (storePath && isValidPath(*storePath)) - entries.push_back(path); - else - tryToDelete(state, path); - } - - dir.reset(); - - /* Now delete the unreachable valid paths. Randomise the - order in which we delete entries to make the collector - less biased towards deleting paths that come - alphabetically first (e.g. /nix/store/000...). This - matters when using --max-freed etc. */ - vector<Path> entries_(entries.begin(), entries.end()); - std::mt19937 gen(1); - std::shuffle(entries_.begin(), entries_.end(), gen); + if (name == "." || name == ".." || name == linksName) continue; - for (auto & i : entries_) - tryToDelete(state, i); + if (auto storePath = maybeParseStorePath(storeDir + "/" + name)) + deleteReferrersClosure(*storePath); + else + deleteFromStore(name); + } } catch (GCLimitReached & e) { } } - if (state.options.action == GCOptions::gcReturnLive) { - for (auto & i : state.alive) - state.results.paths.insert(printStorePath(i)); + if (options.action == GCOptions::gcReturnLive) { + for (auto & i : alive) + results.paths.insert(printStorePath(i)); return; } - if (state.options.action == GCOptions::gcReturnDead) { - for (auto & i : state.dead) - state.results.paths.insert(printStorePath(i)); + if (options.action == GCOptions::gcReturnDead) { + for (auto & i : dead) + results.paths.insert(printStorePath(i)); return; } - /* Allow other processes to add to the store from here on. */ - fdGCLock = -1; - fds.clear(); - - /* Delete the trash directory. */ - printInfo(format("deleting '%1%'") % trashDir); - deleteGarbage(state, trashDir); - - /* Clean up the links directory. */ + /* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { printInfo("deleting unused links..."); - removeUnusedLinks(state); + + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError("opening directory '%1%'", linksDir); + + int64_t actualSize = 0, unsharedSize = 0; + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + auto st = lstat(path); + + if (st.st_nlink != 1) { + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; + continue; + } + + printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError("deleting '%1%'", path); + + results.bytesFreed += st.st_size; + } + + struct stat st; + if (stat(linksDir.c_str(), &st) == -1) + throw SysError("statting '%1%'", linksDir); + int64_t overhead = st.st_blocks * 512ULL; + + printInfo("note: currently hard linking saves %.2f MiB", + ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); } /* While we're at it, vacuum the database. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 9f1a88130..81ca9cc0f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -122,7 +122,7 @@ StringSet Settings::getDefaultSystemFeatures() /* For backwards compatibility, accept some "features" that are used in Nixpkgs to route builds to certain machines but don't actually require anything special on the machines. */ - StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"}; + StringSet features{"nixos-test", "benchmark", "big-parallel"}; #if __linux__ if (access("/dev/kvm", R_OK | W_OK) == 0) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 165639261..433deaf0f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -797,6 +797,15 @@ public: may be useful in certain scenarios (e.g. to spin up containers or set up userspace network interfaces in tests). )"}; + + Setting<StringSet> ignoredAcls{ + this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls", + R"( + A list of ACLs that should be ignored, normally Nix attempts to + remove all ACLs from files and directories in the Nix store, but + some ACLs like `security.selinux` or `system.nfs4_acl` can't be + removed even by root. Therefore it's best to just ignore them. + )"}; #endif Setting<Strings> hashedMirrors{ @@ -951,6 +960,9 @@ public: Setting<bool> useRegistries{this, true, "use-registries", "Whether to use flake registries to resolve flake references."}; + + Setting<bool> acceptFlakeConfig{this, false, "accept-flake-config", + "Whether to accept nix configuration from a flake without prompting."}; }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 814960bb5..4861d185e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -227,7 +227,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, - PathFilter & filter, RepairFlag repair) override + PathFilter & filter, RepairFlag repair, const StorePathSet & references) override { unsupported("addToStore"); } StorePath addTextToStore(const string & name, const string & s, @@ -367,7 +367,8 @@ public: return conn->remoteVersion; } - std::optional<const Realisation> queryRealisation(const DrvOutput&) override + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override // TODO: Implement { unsupported("queryRealisation"); } }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1e3b5482e..79011b522 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -8,6 +8,7 @@ #include "references.hh" #include "callback.hh" #include "topo-sort.hh" +#include "finally.hh" #include <iostream> #include <algorithm> @@ -145,7 +146,6 @@ LocalStore::LocalStore(const Params & params) , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") - , trashDir(realStoreDir + "/trash") , tempRootsDir(stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) , locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or(""))) @@ -386,6 +386,16 @@ LocalStore::LocalStore(const Params & params) } +AutoCloseFD LocalStore::openGCLock() +{ + Path fnGCLock = stateDir + "/gc.lock"; + auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fdGCLock) + throw SysError("opening global GC lock '%1%'", fnGCLock); + return fdGCLock; +} + + LocalStore::~LocalStore() { std::shared_future<void> future; @@ -495,9 +505,6 @@ void LocalStore::makeStoreWritable() throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); - if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) throw SysError("remounting %1% writable", realStoreDir); } @@ -583,9 +590,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe throw SysError("querying extended attributes of '%s'", path); for (auto & eaName: tokenizeString<Strings>(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - /* Ignore SELinux security labels since these cannot be - removed even by root. */ - if (eaName == "security.selinux") continue; + if (settings.ignoredAcls.get().count(eaName)) continue; if (lremovexattr(path.c_str(), eaName.c_str()) == -1) throw SysError("removing extended attribute '%s' from '%s'", eaName, path); } @@ -825,7 +830,7 @@ uint64_t LocalStore::addValidPath(State & state, { auto state_(Store::state.lock()); - state_->pathInfoCache.upsert(std::string(info.path.hashPart()), + state_->pathInfoCache.upsert(std::string(info.path.to_string()), PathInfoCacheValue{ .value = std::make_shared<const ValidPathInfo>(info) }); } @@ -1198,7 +1203,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) { auto state_(Store::state.lock()); - state_->pathInfoCache.erase(std::string(path.hashPart())); + state_->pathInfoCache.erase(std::string(path.to_string())); } } @@ -1302,7 +1307,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) { /* For computing the store path. */ auto hashSink = std::make_unique<HashSink>(hashAlgo); @@ -1327,13 +1332,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto want = std::min(chunkSize, settings.narBufferSize - oldSize); dump.resize(oldSize + want); auto got = 0; + Finally cleanup([&]() { + dump.resize(oldSize + got); + }); try { got = source.read(dump.data() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; } - dump.resize(oldSize + got); } std::unique_ptr<AutoDelete> delTempDir; @@ -1358,7 +1365,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto [hash, size] = hashSink->finish(); - auto dstPath = makeFixedOutputPath(method, hash, name); + auto dstPath = makeFixedOutputPath(method, hash, name, references); addTempRoot(dstPath); @@ -1405,6 +1412,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, ValidPathInfo info { dstPath, narHash.first }; info.narSize = narHash.second; + info.references = references; info.ca = FixedOutputHash { .method = method, .hash = hash }; registerValidPath(info); } @@ -1505,7 +1513,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Acquire the global GC lock to get a consistent snapshot of existing and valid paths. */ - AutoCloseFD fdGCLock = openGCLock(ltWrite); + auto fdGCLock = openGCLock(); + FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); StringSet store; for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); @@ -1516,8 +1525,6 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) StorePathSet validPaths; PathSet done; - fdGCLock = -1; - for (auto & i : queryAllValidPaths()) verifyPath(printStorePath(i), store, done, validPaths, repair, errors); @@ -1830,13 +1837,24 @@ std::optional<const Realisation> LocalStore::queryRealisation_( return { res }; } -std::optional<const Realisation> -LocalStore::queryRealisation(const DrvOutput & id) +void LocalStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { - return retrySQLite<std::optional<const Realisation>>([&]() { - auto state(_state.lock()); - return queryRealisation_(*state, id); - }); + try { + auto maybeRealisation + = retrySQLite<std::optional<const Realisation>>([&]() { + auto state(_state.lock()); + return queryRealisation_(*state, id); + }); + if (maybeRealisation) + callback( + std::make_shared<const Realisation>(maybeRealisation.value())); + else + callback(nullptr); + + } catch (...) { + callback.rethrow(); + } } FixedOutputHash LocalStore::hashCAPath( diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a01d48c4b..115ea046a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,9 +58,15 @@ private: struct Stmts; std::unique_ptr<Stmts> stmts; + /* The global GC lock */ + AutoCloseFD fdGCLock; + /* The file to which we write our temporary roots. */ AutoCloseFD fdTempRoots; + /* Connection to the garbage collector. */ + AutoCloseFD fdRootsSocket; + /* The last time we checked whether to do an auto-GC, or an auto-GC finished. */ std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; @@ -87,7 +93,6 @@ public: const Path linksDir; const Path reservedPath; const Path schemaPath; - const Path trashDir; const Path tempRootsDir; const Path fnTempRoots; @@ -140,7 +145,7 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair) override; + FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore(const string & name, const string & s, const StorePathSet & references, RepairFlag repair) override; @@ -149,14 +154,11 @@ public: void addIndirectRoot(const Path & path) override; - void syncWithGC() override; - private: - typedef std::shared_ptr<AutoCloseFD> FDPtr; - typedef list<FDPtr> FDs; + void findTempRoots(Roots & roots, bool censor); - void findTempRoots(FDs & fds, Roots & roots, bool censor); + AutoCloseFD openGCLock(); public: @@ -205,7 +207,8 @@ public: std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id); std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id); - std::optional<const Realisation> queryRealisation(const DrvOutput&) override; + void queryRealisationUncached(const DrvOutput&, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; private: @@ -236,29 +239,12 @@ private: PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); - struct GCState; - - void deleteGarbage(GCState & state, const Path & path); - - void tryToDelete(GCState & state, const Path & path); - - bool canReachRoot(GCState & state, StorePathSet & visited, const StorePath & path); - - void deletePathRecursive(GCState & state, const Path & path); - - bool isActiveTempFile(const GCState & state, - const Path & path, const string & suffix); - - AutoCloseFD openGCLock(LockType lockType); - void findRoots(const Path & path, unsigned char type, Roots & roots); void findRootsNoTemp(Roots & roots, bool censor); void findRuntimeRoots(Roots & roots, bool censor); - void removeUnusedLinks(const GCState & state); - Path createTempDirInStore(); void checkDerivationOutputs(const StorePath & drvPath, const Derivation & drv); diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 9843ccf04..b6270a81b 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -39,7 +39,8 @@ Machine::Machine(decltype(storeUri) storeUri, sshPublicHostKey(sshPublicHostKey) {} -bool Machine::allSupported(const std::set<string> & features) const { +bool Machine::allSupported(const std::set<string> & features) const +{ return std::all_of(features.begin(), features.end(), [&](const string & feature) { return supportedFeatures.count(feature) || @@ -47,14 +48,16 @@ bool Machine::allSupported(const std::set<string> & features) const { }); } -bool Machine::mandatoryMet(const std::set<string> & features) const { +bool Machine::mandatoryMet(const std::set<string> & features) const +{ return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), [&](const string & feature) { return features.count(feature); }); } -ref<Store> Machine::openStore() const { +ref<Store> Machine::openStore() const +{ Store::Params storeParams; if (hasPrefix(storeUri, "ssh://")) { storeParams["max-connections"] = "1"; @@ -83,53 +86,86 @@ ref<Store> Machine::openStore() const { return nix::openStore(storeUri, storeParams); } -void parseMachines(const std::string & s, Machines & machines) +static std::vector<std::string> expandBuilderLines(const std::string & builders) { - for (auto line : tokenizeString<std::vector<string>>(s, "\n;")) { + std::vector<std::string> result; + for (auto line : tokenizeString<std::vector<string>>(builders, "\n;")) { trim(line); line.erase(std::find(line.begin(), line.end(), '#'), line.end()); if (line.empty()) continue; if (line[0] == '@') { - auto file = trim(std::string(line, 1)); + const std::string path = trim(std::string(line, 1)); + std::string text; try { - parseMachines(readFile(file), machines); + text = readFile(path); } catch (const SysError & e) { if (e.errNo != ENOENT) throw; - debug("cannot find machines file '%s'", file); + debug("cannot find machines file '%s'", path); } + + const auto lines = expandBuilderLines(text); + result.insert(end(result), begin(lines), end(lines)); continue; } - auto tokens = tokenizeString<std::vector<string>>(line); - auto sz = tokens.size(); - if (sz < 1) - throw FormatError("bad machine specification '%s'", line); + result.emplace_back(line); + } + return result; +} - auto isSet = [&](size_t n) { - return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; - }; +static Machine parseBuilderLine(const std::string & line) +{ + const auto tokens = tokenizeString<std::vector<string>>(line); - machines.emplace_back(tokens[0], - isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, - isSet(2) ? tokens[2] : "", - isSet(3) ? std::stoull(tokens[3]) : 1LL, - isSet(4) ? std::stoull(tokens[4]) : 1LL, - isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, - isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, - isSet(7) ? tokens[7] : ""); - } + auto isSet = [&](size_t fieldIndex) { + return tokens.size() > fieldIndex && tokens[fieldIndex] != "" && tokens[fieldIndex] != "-"; + }; + + auto parseUnsignedIntField = [&](size_t fieldIndex) { + const auto result = string2Int<unsigned int>(tokens[fieldIndex]); + if (!result) { + throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'unsigned int'", fieldIndex, line); + } + return result.value(); + }; + + auto ensureBase64 = [&](size_t fieldIndex) { + const auto & str = tokens[fieldIndex]; + try { + base64Decode(str); + } catch (const Error & e) { + throw FormatError("bad machine specification: a column #%lu in a row: '%s' is not valid base64 string: %s", fieldIndex, line, e.what()); + } + return str; + }; + + if (!isSet(0)) + throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line); + + return { + tokens[0], + isSet(1) ? tokenizeString<std::vector<string>>(tokens[1], ",") : std::vector<string>{settings.thisSystem}, + isSet(2) ? tokens[2] : "", + isSet(3) ? parseUnsignedIntField(3) : 1U, + isSet(4) ? parseUnsignedIntField(4) : 1U, + isSet(5) ? tokenizeString<std::set<string>>(tokens[5], ",") : std::set<string>{}, + isSet(6) ? tokenizeString<std::set<string>>(tokens[6], ",") : std::set<string>{}, + isSet(7) ? ensureBase64(7) : "" + }; +} + +static Machines parseBuilderLines(const std::vector<std::string>& builders) { + Machines result; + std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine); + return result; } Machines getMachines() { - static auto machines = [&]() { - Machines machines; - parseMachines(settings.builders, machines); - return machines; - }(); - return machines; + const auto builderLines = expandBuilderLines(settings.builders); + return parseBuilderLines(builderLines); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index f184dd857..32786e963 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -239,12 +239,11 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) { return topoSort(paths, {[&](const StorePath & path) { - StorePathSet references; try { - references = queryPathInfo(path)->references; + return queryPathInfo(path)->references; } catch (InvalidPath &) { + return StorePathSet(); } - return references; }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( diff --git a/src/libstore/names.cc b/src/libstore/names.cc index ce808accc..54c95055d 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -42,7 +42,7 @@ DrvName::~DrvName() { } -bool DrvName::matches(DrvName & n) +bool DrvName::matches(const DrvName & n) { if (name != "*") { if (!regex) { diff --git a/src/libstore/names.hh b/src/libstore/names.hh index bc62aac93..3f861bc44 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -19,7 +19,7 @@ struct DrvName DrvName(std::string_view s); ~DrvName(); - bool matches(DrvName & n); + bool matches(const DrvName & n); private: std::unique_ptr<Regex> regex; diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 926f4ea1e..2da74e262 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -176,4 +176,17 @@ void PathLocks::setDeletion(bool deletePaths) } +FdLock::FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg) + : fd(fd) +{ + if (wait) { + if (!lockFile(fd, lockType, false)) { + printInfo("%s", waitMsg); + acquired = lockFile(fd, lockType, true); + } + } else + acquired = lockFile(fd, lockType, false); +} + + } diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 411da0222..919c8904c 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -35,4 +35,18 @@ public: void setDeletion(bool deletePaths); }; +struct FdLock +{ + int fd; + bool acquired = false; + + FdLock(int fd, LockType lockType, bool wait, std::string_view waitMsg); + + ~FdLock() + { + if (acquired) + lockFile(fd, ltNone, false); + } +}; + } diff --git a/src/libstore/references.cc b/src/libstore/references.cc index c369b14ac..91b3fc142 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -54,12 +54,12 @@ void RefScanSink::operator () (std::string_view data) fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ auto s = tail; - s.append(data.data(), refLength); + auto tailLen = std::min(data.size(), refLength); + s.append(data.data(), tailLen); search(s, hashes, seen); search(data, hashes, seen); - auto tailLen = std::min(data.size(), refLength); auto rest = refLength - tailLen; if (rest < tail.size()) tail = tail.substr(tail.size() - rest); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index fa5ea8af7..7f7e973e9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -290,6 +290,10 @@ ConnectionHandle RemoteStore::getConnection() return ConnectionHandle(connections->get()); } +void RemoteStore::setOptions() +{ + setOptions(*(getConnection().handle)); +} bool RemoteStore::isValidPathUncached(const StorePath & path) { @@ -578,9 +582,8 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore( StorePath RemoteStore::addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method, HashType hashType, RepairFlag repair) + FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) { - StorePathSet references; return addCAToStore(dump, name, FixedOutputHashMethod{ .fileIngestionMethod = method, .hashType = hashType }, references, repair)->path; } @@ -677,23 +680,41 @@ void RemoteStore::registerDrvOutput(const Realisation & info) conn.processStderr(); } -std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id) +void RemoteStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); + try { + callback(nullptr); + } catch (...) { return callback.rethrow(); } + } + conn->to << wopQueryRealisation; conn->to << id.to_string(); conn.processStderr(); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { - auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{}); - if (outPaths.empty()) - return std::nullopt; - return {Realisation{.id = id, .outPath = *outPaths.begin()}}; - } else { - auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{}); - if (realisations.empty()) - return std::nullopt; - return *realisations.begin(); - } + + auto real = [&]() -> std::shared_ptr<const Realisation> { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + auto outPaths = worker_proto::read( + *this, conn->from, Phantom<std::set<StorePath>> {}); + if (outPaths.empty()) + return nullptr; + return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() }); + } else { + auto realisations = worker_proto::read( + *this, conn->from, Phantom<std::set<Realisation>> {}); + if (realisations.empty()) + return nullptr; + return std::make_shared<const Realisation>(*realisations.begin()); + } + }(); + + try { + callback(std::shared_ptr<const Realisation>(real)); + } catch (...) { return callback.rethrow(); } } static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs) @@ -797,15 +818,6 @@ void RemoteStore::addIndirectRoot(const Path & path) } -void RemoteStore::syncWithGC() -{ - auto conn(getConnection()); - conn->to << wopSyncWithGC; - conn.processStderr(); - readInt(conn->from); -} - - Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index ac1eaa19e..0fd67f371 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -73,7 +73,7 @@ public: /* Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; @@ -88,7 +88,8 @@ public: void registerDrvOutput(const Realisation & info) override; - std::optional<const Realisation> queryRealisation(const DrvOutput &) override; + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override; @@ -101,8 +102,6 @@ public: void addIndirectRoot(const Path & path) override; - void syncWithGC() override; - Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -149,6 +148,8 @@ protected: virtual void setOptions(Connection & conn); + void setOptions() override; + ConnectionHandle getConnection(); friend struct ConnectionHandle; diff --git a/src/libstore/sandbox-defaults.sb b/src/libstore/sandbox-defaults.sb index 41893e6dd..56b35c3fe 100644 --- a/src/libstore/sandbox-defaults.sb +++ b/src/libstore/sandbox-defaults.sb @@ -100,4 +100,5 @@ ; Allow Rosetta 2 to run x86_64 binaries on aarch64-darwin. (allow file-read* - (subpath "/Library/Apple/usr/libexec/oah")) + (subpath "/Library/Apple/usr/libexec/oah") + (subpath "/System/Library/Apple/usr/libexec/oah")) diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 447b4179b..1d6baf02d 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -1,4 +1,5 @@ #include "sqlite.hh" +#include "globals.hh" #include "util.hh" #include <sqlite3.h> @@ -27,8 +28,12 @@ namespace nix { SQLite::SQLite(const Path & path, bool create) { + // useSQLiteWAL also indicates what virtual file system we need. Using + // `unix-dotfile` is needed on NFS file systems and on Windows' Subsystem + // for Linux (WSL) where useSQLiteWAL should be false by default. + const char *vfs = settings.useSQLiteWAL ? 0 : "unix-dotfile"; if (sqlite3_open_v2(path.c_str(), &db, - SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), vfs) != SQLITE_OK) throw Error("cannot open SQLite database '%s'", path); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 3338cdc1b..aab4ce94c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -237,7 +237,7 @@ StorePath Store::computeStorePathForText(const string & name, const string & s, StorePath Store::addToStore(const string & name, const Path & _srcPath, - FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair) + FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair, const StorePathSet & references) { Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { @@ -246,7 +246,7 @@ StorePath Store::addToStore(const string & name, const Path & _srcPath, else readFile(srcPath, sink); }); - return addToStoreFromDump(*source, name, method, hashAlgo, repair); + return addToStoreFromDump(*source, name, method, hashAlgo, repair, references); } @@ -355,8 +355,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, StringSet StoreConfig::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) res.insert("ca-derivations"); + + if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix)) + res.insert("recursive-nix"); + return res; } @@ -414,11 +419,9 @@ StorePathSet Store::queryDerivationOutputs(const StorePath & path) bool Store::isValidPath(const StorePath & storePath) { - std::string hashPart(storePath.hashPart()); - { auto state_(state.lock()); - auto res = state_->pathInfoCache.get(hashPart); + auto res = state_->pathInfoCache.get(std::string(storePath.to_string())); if (res && res->isKnownNow()) { stats.narInfoReadAverted++; return res->didExist(); @@ -426,11 +429,11 @@ bool Store::isValidPath(const StorePath & storePath) } if (diskCache) { - auto res = diskCache->lookupNarInfo(getUri(), hashPart); + auto res = diskCache->lookupNarInfo(getUri(), std::string(storePath.hashPart())); if (res.first != NarInfoDiskCache::oUnknown) { stats.narInfoReadAverted++; auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, + state_->pathInfoCache.upsert(std::string(storePath.to_string()), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue { .value = res.second }); return res.first == NarInfoDiskCache::oValid; } @@ -440,7 +443,7 @@ bool Store::isValidPath(const StorePath & storePath) if (diskCache && !valid) // FIXME: handle valid = true case. - diskCache->upsertNarInfo(getUri(), hashPart, 0); + diskCache->upsertNarInfo(getUri(), std::string(storePath.hashPart()), 0); return valid; } @@ -487,13 +490,11 @@ static bool goodStorePath(const StorePath & expected, const StorePath & actual) void Store::queryPathInfo(const StorePath & storePath, Callback<ref<const ValidPathInfo>> callback) noexcept { - std::string hashPart; + auto hashPart = std::string(storePath.hashPart()); try { - hashPart = storePath.hashPart(); - { - auto res = state.lock()->pathInfoCache.get(hashPart); + auto res = state.lock()->pathInfoCache.get(std::string(storePath.to_string())); if (res && res->isKnownNow()) { stats.narInfoReadAverted++; if (!res->didExist()) @@ -508,7 +509,7 @@ void Store::queryPathInfo(const StorePath & storePath, stats.narInfoReadAverted++; { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, + state_->pathInfoCache.upsert(std::string(storePath.to_string()), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{ .value = res.second }); if (res.first == NarInfoDiskCache::oInvalid || !goodStorePath(storePath, res.second->path)) @@ -523,7 +524,7 @@ void Store::queryPathInfo(const StorePath & storePath, auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); queryPathInfoUncached(storePath, - {[this, storePathS{printStorePath(storePath)}, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { + {[this, storePath, hashPart, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) { try { auto info = fut.get(); @@ -533,14 +534,12 @@ void Store::queryPathInfo(const StorePath & storePath, { auto state_(state.lock()); - state_->pathInfoCache.upsert(hashPart, PathInfoCacheValue { .value = info }); + state_->pathInfoCache.upsert(std::string(storePath.to_string()), PathInfoCacheValue { .value = info }); } - auto storePath = parseStorePath(storePathS); - if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; - throw InvalidPath("path '%s' is not valid", storePathS); + throw InvalidPath("path '%s' is not valid", printStorePath(storePath)); } (*callbackPtr)(ref<const ValidPathInfo>(info)); @@ -548,6 +547,74 @@ void Store::queryPathInfo(const StorePath & storePath, }}); } +void Store::queryRealisation(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept +{ + + try { + if (diskCache) { + auto [cacheOutcome, maybeCachedRealisation] + = diskCache->lookupRealisation(getUri(), id); + switch (cacheOutcome) { + case NarInfoDiskCache::oValid: + debug("Returning a cached realisation for %s", id.to_string()); + callback(maybeCachedRealisation); + return; + case NarInfoDiskCache::oInvalid: + debug( + "Returning a cached missing realisation for %s", + id.to_string()); + callback(nullptr); + return; + case NarInfoDiskCache::oUnknown: + break; + } + } + } catch (...) { + return callback.rethrow(); + } + + auto callbackPtr + = std::make_shared<decltype(callback)>(std::move(callback)); + + queryRealisationUncached( + id, + { [this, id, callbackPtr]( + std::future<std::shared_ptr<const Realisation>> fut) { + try { + auto info = fut.get(); + + if (diskCache) { + if (info) + diskCache->upsertRealisation(getUri(), *info); + else + diskCache->upsertAbsentRealisation(getUri(), id); + } + + (*callbackPtr)(std::shared_ptr<const Realisation>(info)); + + } catch (...) { + callbackPtr->rethrow(); + } + } }); +} + +std::shared_ptr<const Realisation> Store::queryRealisation(const DrvOutput & id) +{ + using RealPtr = std::shared_ptr<const Realisation>; + std::promise<RealPtr> promise; + + queryRealisation(id, + {[&](std::future<RealPtr> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + + return promise.get_future().get(); +} void Store::substitutePaths(const StorePathSet & paths) { @@ -1012,7 +1079,7 @@ std::map<StorePath, StorePath> copyPaths( nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what())); + printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what()); showProgress(); return; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 54471bdf2..aa44651d4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -232,7 +232,6 @@ protected: struct State { - // FIXME: fix key LRUCache<std::string, PathInfoCacheValue> pathInfoCache; }; @@ -370,6 +369,14 @@ public: void queryPathInfo(const StorePath & path, Callback<ref<const ValidPathInfo>> callback) noexcept; + /* Query the information about a realisation. */ + std::shared_ptr<const Realisation> queryRealisation(const DrvOutput &); + + /* Asynchronous version of queryRealisation(). */ + void queryRealisation(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept; + + /* Check whether the given valid path info is sufficiently attested, by either being signed by a trusted public key or content-addressed, in order to be included in the given store. @@ -394,11 +401,11 @@ protected: virtual void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0; + virtual void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept = 0; public: - virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0; - /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) @@ -453,7 +460,7 @@ public: libutil/archive.hh). */ virtual StorePath addToStore(const string & name, const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair); + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()); /* Copy the contents of a path to the store and register the validity the resulting path, using a constant amount of @@ -469,7 +476,8 @@ public: `dump` may be drained */ // FIXME: remove? virtual StorePath addToStoreFromDump(Source & dump, const string & name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } /* Like addToStore, but the contents written to the output path is @@ -561,26 +569,6 @@ public: virtual void addIndirectRoot(const Path & path) { unsupported("addIndirectRoot"); } - /* Acquire the global GC lock, then immediately release it. This - function must be called after registering a new permanent root, - but before exiting. Otherwise, it is possible that a running - garbage collector doesn't see the new root and deletes the - stuff we've just built. By acquiring the lock briefly, we - ensure that either: - - - The collector is already running, and so we block until the - collector is finished. The collector will know about our - *temporary* locks, which should include whatever it is we - want to register as a permanent lock. - - - The collector isn't running, or it's just started but hasn't - acquired the GC lock yet. In that case we get and release - the lock right away, then exit. The collector scans the - permanent root and sees ours. - - In either case the permanent root is seen by the collector. */ - virtual void syncWithGC() { }; - /* Find the roots of the garbage collector. Each root is a pair (link, storepath) where `link' is the path of the symlink outside of the Nix store that point to `storePath'. If @@ -745,6 +733,11 @@ public: virtual void createUser(const std::string & userName, uid_t userId) { } + /* + * Synchronises the options of the client with those of the daemon + * (a no-op when there’s no daemon) + */ + virtual void setOptions() { } protected: Stats stats; diff --git a/src/libstore/tests/machines.cc b/src/libstore/tests/machines.cc new file mode 100644 index 000000000..f51052b14 --- /dev/null +++ b/src/libstore/tests/machines.cc @@ -0,0 +1,169 @@ +#include "machines.hh" +#include "globals.hh" + +#include <gmock/gmock-matchers.h> + +using testing::Contains; +using testing::ElementsAre; +using testing::EndsWith; +using testing::Eq; +using testing::Field; +using testing::SizeIs; + +using nix::absPath; +using nix::FormatError; +using nix::getMachines; +using nix::Machine; +using nix::Machines; +using nix::pathExists; +using nix::Settings; +using nix::settings; + +class Environment : public ::testing::Environment { + public: + void SetUp() override { settings.thisSystem = "TEST_ARCH-TEST_OS"; } +}; + +testing::Environment* const foo_env = + testing::AddGlobalTestEnvironment(new Environment); + +TEST(machines, getMachinesWithEmptyBuilders) { + settings.builders = ""; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesUriOnly) { + settings.builders = "nix@scratchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); +} + +TEST(machines, getMachinesDefaults) { + settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(1))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, SizeIs(0))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); +} + +TEST(machines, getMachinesWithNewLineSeparator) { + settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(2)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithSemicolonSeparator) { + settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"; + Machines actual = getMachines(); + EXPECT_THAT(actual, SizeIs(2)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { + settings.builders = "nix@scratchy.labs.cs.uu.nl i686-linux " + "/home/nix/.ssh/id_scratchy_auto 8 3 kvm " + "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark"))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, Eq("SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="))); +} + +TEST(machines, + getMachinesWithCorrectCompleteSingleBuilderWithTabColumnDelimiter) { + settings.builders = + "nix@scratchy.labs.cs.uu.nl\ti686-linux\t/home/nix/.ssh/" + "id_scratchy_auto\t8\t3\tkvm\tbenchmark\tSSH+HOST+PUBLIC+" + "KEY+BASE64+ENCODED=="; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); + EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); + EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); + EXPECT_THAT(actual[0], Field(&Machine::speedFactor, Eq(3))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("kvm"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("benchmark"))); + EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, Eq("SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="))); +} + +TEST(machines, getMachinesWithMultiOptions) { + settings.builders = "nix@scratchy.labs.cs.uu.nl Arch1,Arch2 - - - " + "SupportedFeature1,SupportedFeature2 " + "MandatoryFeature1,MandatoryFeature2"; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(1)); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2"))); + EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2"))); + EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2"))); +} + +TEST(machines, getMachinesWithIncorrectFormat) { + settings.builders = "nix@scratchy.labs.cs.uu.nl - - eight"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - -1"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 three"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 -3"; + EXPECT_THROW(getMachines(), FormatError); + settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 3 - - BAD_BASE64"; + EXPECT_THROW(getMachines(), FormatError); +} + +TEST(machines, getMachinesWithCorrectFileReference) { + auto path = absPath("src/libstore/tests/test-data/machines.valid"); + ASSERT_TRUE(pathExists(path)); + + settings.builders = std::string("@") + path; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(3)); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl")))); +} + +TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { + auto path = "/dev/null"; + ASSERT_TRUE(pathExists(path)); + + settings.builders = std::string("@") + path; + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesWithIncorrectFileReference) { + settings.builders = std::string("@") + absPath("/not/a/file"); + Machines actual = getMachines(); + ASSERT_THAT(actual, SizeIs(0)); +} + +TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { + settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); + EXPECT_THROW(getMachines(), FormatError); +} diff --git a/src/libstore/tests/test-data/machines.bad_format b/src/libstore/tests/test-data/machines.bad_format new file mode 100644 index 000000000..7255a1216 --- /dev/null +++ b/src/libstore/tests/test-data/machines.bad_format @@ -0,0 +1 @@ +nix@scratchy.labs.cs.uu.nl - - eight diff --git a/src/libstore/tests/test-data/machines.valid b/src/libstore/tests/test-data/machines.valid new file mode 100644 index 000000000..1a6c8017c --- /dev/null +++ b/src/libstore/tests/test-data/machines.valid @@ -0,0 +1,3 @@ +nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm +nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 +nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFDQVFDWWV5R1laNTNzd1VjMUZNSHBWL1BCcXlKaFR5S1JoRkpWWVRpRHlQN2h5c1JGa0w4VDlLOGdhL2Y2L3c3QjN2SjNHSFRIUFkybENiUEdZbGNLd2h6M2ZRbFNNOEViNi95b3ZLajdvM1FsMEx5Y0dzdGJvRmcwWkZKNldncUxsR0ltS0NobUlxOGZ3TW5ZTWUxbnRQeTBUZFZjSU1tOTV3YzF3SjBMd2c3cEVMRmtHazdkeTVvYnM4a3lGZ0pORDVRSmFwQWJjeWp4Z1QzdzdMcktNZ2xzeWhhd01JNVpkMGZsQTVudW5OZ3pid3plYVhLaUsyTW0vdGJXYTU1YTd4QmNYdHpIZGlPSWdSajJlRWxaMGh5bk10YjBmcklsdmxIcEtLaVFaZ3pQdCtIVXQ2bXpRMkRVME52MGYyYnNSU0krOGpJU2pQcmdlcVVHRldMUzVIUTg2N2xSMlpiaWtyclhZNTdqbVFEZk5DRHY1VFBHZU9UekFEd2pjMDc2aFZ3VFJCd3VTZFhtaWNxTS95b3lrWitkV1dnZ25MenE5QU1tdlNZcDhmZkZDcS9CSDBZNUFXWTFHay9vS3hMVTNaOWt3ZDd2UWNFQWFCQ2dxdnVZRGdTaHE1RlhndDM3OVZESWtEL05ZSTg2QXVvajVDRmVNTzlRM2pJSlRadlh6c1VldjVoSnA2djcxSVh5ODVtbTY5R20zcXdicVE1SjVQZDU1Um56SitpaW5BNjZxTEFSc0Y4amNsSnd5ekFXclBoYU9DRVY2bjVMeVhVazhzMW9EVVR4V1pWN25rVkFTbHJ0MllGcjN5dzdjRTRXQVhsemhHcDhocmdLMVVkMUlyeDVnZWRaSnBWcy9uNWVybmJFMUxmb2x5UHUvRUFIWlh6VGd4dHVDUFNobXc9PQo= diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 02e81b022..5c38323cd 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -56,14 +56,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection() auto conn = make_ref<Connection>(); /* Connect to a daemon that does the privileged work for us. */ - conn->fd = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!conn->fd) - throw SysError("cannot create Unix domain socket"); - closeOnExec(conn->fd.get()); + conn->fd = createUnixDomainSocket(); nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile); diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 96ad69790..ce9c3dfed 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -189,13 +189,14 @@ extern Verbosity verbosity; /* suppress msgs > this */ /* Print a string message if the current log level is at least the specified level. Note that this has to be implemented as a macro to ensure that the arguments are evaluated lazily. */ -#define printMsg(level, args...) \ +#define printMsgUsing(loggerParam, level, args...) \ do { \ auto __lvl = level; \ if (__lvl <= nix::verbosity) { \ - logger->log(__lvl, fmt(args)); \ + loggerParam->log(__lvl, fmt(args)); \ } \ } while (0) +#define printMsg(level, args...) printMsgUsing(logger, level, args) #define printError(args...) printMsg(lvlError, args) #define notice(args...) printMsg(lvlNotice, args) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 50e691a3d..790bc943a 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -93,9 +93,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir) else archive.check(r); - archive_entry_set_pathname(entry, + archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str()); + // Patch hardlink path + const char *original_hardlink = archive_entry_hardlink(entry); + if (original_hardlink) { + archive_entry_copy_hardlink(entry, + (destDir + "/" + original_hardlink).c_str()); + } + archive.check(archive_read_extract(archive.archive, entry, flags)); } diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 58df9c5ac..92972ed14 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -4,6 +4,8 @@ #include <limits.h> #include <gtest/gtest.h> +#include <numeric> + namespace nix { /* ----------- tests for util.hh ------------------------------------------------*/ @@ -282,6 +284,17 @@ namespace nix { ASSERT_EQ(decoded, s); } + TEST(base64Encode, encodeAndDecodeNonPrintable) { + char s[256]; + std::iota(std::rbegin(s), std::rend(s), 0); + + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + EXPECT_EQ(decoded.length(), 255); + ASSERT_EQ(decoded, s); + } + /* ---------------------------------------------------------------------------- * base64Decode * --------------------------------------------------------------------------*/ @@ -294,6 +307,10 @@ namespace nix { ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); } + TEST(base64Decode, decodeThrowsOnInvalidChar) { + ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); + } + /* ---------------------------------------------------------------------------- * toLower * --------------------------------------------------------------------------*/ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 563a72c12..1b6467eb2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -512,6 +512,7 @@ std::pair<AutoCloseFD, Path> createTempFile(const Path & prefix) AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); if (!fd) throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); return {std::move(fd), tmpl}; } @@ -562,7 +563,7 @@ Path getConfigDir() std::vector<Path> getConfigDirs() { Path configHome = getConfigDir(); - string configDirs = getEnv("XDG_CONFIG_DIRS").value_or(""); + string configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":"); result.insert(result.begin(), configHome); return result; @@ -1205,7 +1206,7 @@ void closeOnExec(int fd) ////////////////////////////////////////////////////////////////////// -bool _isInterrupted = false; +std::atomic<bool> _isInterrupted = false; static thread_local bool interruptThrown = false; thread_local std::function<bool()> interruptCheck; @@ -1436,8 +1437,7 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in } -static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -static std::array<char, 256> base64DecodeChars; +constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; string base64Encode(std::string_view s) { @@ -1462,12 +1462,15 @@ string base64Encode(std::string_view s) string base64Decode(std::string_view s) { - static std::once_flag flag; - std::call_once(flag, [](){ - base64DecodeChars = { (char)-1 }; + constexpr char npos = -1; + constexpr std::array<char, 256> base64DecodeChars = [&]() { + std::array<char, 256> result{}; + for (auto& c : result) + c = npos; for (int i = 0; i < 64; i++) - base64DecodeChars[(int) base64Chars[i]] = i; - }); + result[base64Chars[i]] = i; + return result; + }(); string res; unsigned int d = 0, bits = 0; @@ -1477,7 +1480,7 @@ string base64Decode(std::string_view s) if (c == '\n') continue; char digit = base64DecodeChars[(unsigned char) c]; - if (digit == -1) + if (digit == npos) throw Error("invalid character in Base64 string: '%c'", c); bits += 6; @@ -1630,9 +1633,39 @@ void setStackSize(size_t stackSize) #endif } -void restoreProcessContext() +static AutoCloseFD fdSavedMountNamespace; + +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); + if (!fd) + throw SysError("saving parent mount namespace"); + fdSavedMountNamespace = std::move(fd); + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + try { + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + } catch (Error & e) { + debug(e.msg()); + } +#endif +} + +void restoreProcessContext(bool restoreMounts) { restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } restoreAffinity(); @@ -1670,7 +1703,7 @@ std::unique_ptr<InterruptCallback> createInterruptCallback(std::function<void()> } -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +AutoCloseFD createUnixDomainSocket() { AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM #ifdef SOCK_CLOEXEC @@ -1679,8 +1712,14 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) , 0); if (!fdSocket) throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); + return fdSocket; +} + + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); bind(fdSocket.get(), path); @@ -1709,7 +1748,7 @@ void bind(int fd, const std::string & path) std::string base(baseNameOf(path)); if (base.size() + 1 >= sizeof(addr.sun_path)) throw Error("socket path '%s' is too long", base); - strcpy(addr.sun_path, base.c_str()); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError("cannot bind to socket '%s'", path); _exit(0); @@ -1718,7 +1757,7 @@ void bind(int fd, const std::string & path) if (status != 0) throw Error("cannot bind to socket '%s'", path); } else { - strcpy(addr.sun_path, path.c_str()); + memcpy(addr.sun_path, path.c_str(), path.size() + 1); if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError("cannot bind to socket '%s'", path); } @@ -1738,7 +1777,7 @@ void connect(int fd, const std::string & path) std::string base(baseNameOf(path)); if (base.size() + 1 >= sizeof(addr.sun_path)) throw Error("socket path '%s' is too long", base); - strcpy(addr.sun_path, base.c_str()); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError("cannot connect to socket at '%s'", path); _exit(0); @@ -1747,7 +1786,7 @@ void connect(int fd, const std::string & path) if (status != 0) throw Error("cannot connect to socket at '%s'", path); } else { - strcpy(addr.sun_path, path.c_str()); + memcpy(addr.sun_path, path.c_str(), path.size() + 1); if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) throw SysError("cannot connect to socket at '%s'", path); } @@ -1766,7 +1805,7 @@ void commonChildInit(Pipe & logPipe) logger = makeSimpleLogger(); const static string pathNullDevice = "/dev/null"; - restoreProcessContext(); + restoreProcessContext(false); /* Put the child in a separate session (and thus a separate process group) so that it has no controlling terminal (meaning diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 29232453f..bc96bfed1 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -11,6 +11,7 @@ #include <unistd.h> #include <signal.h> +#include <atomic> #include <functional> #include <map> #include <sstream> @@ -300,7 +301,15 @@ void setStackSize(size_t stackSize); /* Restore the original inherited Unix process context (such as signal masks, stack size, CPU affinity). */ -void restoreProcessContext(); +void restoreProcessContext(bool restoreMounts = true); + +/* Save the current mount namespace. Ignored if called more than + once. */ +void saveMountNamespace(); + +/* Restore the mount namespace saved by saveMountNamespace(). Ignored + if saveMountNamespace() was never called. */ +void restoreMountNamespace(); class ExecError : public Error @@ -329,7 +338,7 @@ void closeOnExec(int fd); /* User interruption. */ -extern bool _isInterrupted; +extern std::atomic<bool> _isInterrupted; extern thread_local std::function<bool()> interruptCheck; @@ -511,6 +520,29 @@ std::optional<typename T::mapped_type> get(const T & map, const typename T::key_ } +/* Remove and return the first item from a container. */ +template <class T> +std::optional<typename T::value_type> remove_begin(T & c) +{ + auto i = c.begin(); + if (i == c.end()) return {}; + auto v = std::move(*i); + c.erase(i); + return v; +} + + +/* Remove and return the first item from a container. */ +template <class T> +std::optional<typename T::value_type> pop(T & c) +{ + if (c.empty()) return {}; + auto v = std::move(c.front()); + c.pop(); + return v; +} + + template<typename T> class Callback; @@ -571,6 +603,9 @@ extern PathFilter defaultPathFilter; /* Common initialisation performed in child processes. */ void commonChildInit(Pipe & logPipe); +/* Create a Unix domain socket. */ +AutoCloseFD createUnixDomainSocket(); + /* Create a Unix domain socket in listen mode. */ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 73d93480e..e2325c91f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -105,7 +105,8 @@ static void main_nix_build(int argc, char * * argv) // List of environment variables kept for --pure std::set<string> keepVars{ - "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", + "HOME", "XDG_RUNTIME_DIR", "USER", "LOGNAME", "DISPLAY", + "WAYLAND_DISPLAY", "WAYLAND_SOCKET", "PATH", "TERM", "IN_NIX_SHELL", "NIX_SHELL_PRESERVE_PROMPT", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL", "http_proxy", "https_proxy", "ftp_proxy", "all_proxy", "no_proxy" }; @@ -358,6 +359,7 @@ static void main_nix_build(int argc, char * * argv) is not set, then build bashInteractive from <nixpkgs>. */ auto shell = getEnv("NIX_BUILD_SHELL"); + std::optional<StorePath> shellDrv; if (!shell) { @@ -374,8 +376,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = store->parseStorePath(drv->queryDrvPath()); pathsToBuild.push_back({bashDrv}); pathsToCopy.insert(bashDrv); - - shell = drv->queryOutPath() + "/bin/bash"; + shellDrv = bashDrv; } catch (Error & e) { logError(e.info()); @@ -401,6 +402,11 @@ static void main_nix_build(int argc, char * * argv) if (dryRun) return; + if (shellDrv) { + auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value()); + shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; + } + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { auto resolvedDrv = drv.tryResolve(*store); assert(resolvedDrv && "Successfully resolved the derivation"); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a86f55f84..b9e7be1c6 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -224,6 +224,91 @@ static void checkSelectorUse(DrvNames & selectors) } +namespace { + +std::set<std::string> searchByPrefix(const DrvInfos & allElems, std::string_view prefix) { + constexpr std::size_t maxResults = 3; + std::set<std::string> result; + for (const auto & drvInfo : allElems) { + const auto drvName = DrvName { drvInfo.queryName() }; + if (hasPrefix(drvName.name, prefix)) { + result.emplace(drvName.name); + + if (result.size() >= maxResults) { + break; + } + } + } + return result; +} + +struct Match +{ + DrvInfo drvInfo; + std::size_t index; + + Match(DrvInfo drvInfo_, std::size_t index_) + : drvInfo{std::move(drvInfo_)} + , index{index_} + {} +}; + +/* If a selector matches multiple derivations + with the same name, pick the one matching the current + system. If there are still multiple derivations, pick the + one with the highest priority. If there are still multiple + derivations, pick the one with the highest version. + Finally, if there are still multiple derivations, + arbitrarily pick the first one. */ +std::vector<Match> pickNewestOnly(EvalState & state, std::vector<Match> matches) { + /* Map from package names to derivations. */ + std::map<std::string, Match> newest; + StringSet multiple; + + for (auto & match : matches) { + auto & oneDrv = match.drvInfo; + + const auto drvName = DrvName { oneDrv.queryName() }; + long comparison = 1; + + const auto itOther = newest.find(drvName.name); + + if (itOther != newest.end()) { + auto & newestDrv = itOther->second.drvInfo; + + comparison = + oneDrv.querySystem() == newestDrv.querySystem() ? 0 : + oneDrv.querySystem() == settings.thisSystem ? 1 : + newestDrv.querySystem() == settings.thisSystem ? -1 : 0; + if (comparison == 0) + comparison = comparePriorities(state, oneDrv, newestDrv); + if (comparison == 0) + comparison = compareVersions(drvName.version, DrvName { newestDrv.queryName() }.version); + } + + if (comparison > 0) { + newest.erase(drvName.name); + newest.emplace(drvName.name, match); + multiple.erase(drvName.fullName); + } else if (comparison == 0) { + multiple.insert(drvName.fullName); + } + } + + matches.clear(); + for (auto & [name, match] : newest) { + if (multiple.find(name) != multiple.end()) + warn( + "there are multiple derivations named '%1%'; using the first one", + name); + matches.push_back(match); + } + + return matches; +} + +} // end namespace + static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, const Strings & args, bool newestOnly) { @@ -232,80 +317,43 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, selectors.emplace_back("*"); DrvInfos elems; - set<unsigned int> done; - - for (auto & i : selectors) { - typedef list<std::pair<DrvInfo, unsigned int> > Matches; - Matches matches; - unsigned int n = 0; - for (DrvInfos::const_iterator j = allElems.begin(); - j != allElems.end(); ++j, ++n) - { - DrvName drvName(j->queryName()); - if (i.matches(drvName)) { - i.hits++; - matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n)); + std::set<std::size_t> done; + + for (auto & selector : selectors) { + std::vector<Match> matches; + for (const auto & [index, drvInfo] : enumerate(allElems)) { + const auto drvName = DrvName { drvInfo.queryName() }; + if (selector.matches(drvName)) { + ++selector.hits; + matches.emplace_back(drvInfo, index); } } - /* If `newestOnly', if a selector matches multiple derivations - with the same name, pick the one matching the current - system. If there are still multiple derivations, pick the - one with the highest priority. If there are still multiple - derivations, pick the one with the highest version. - Finally, if there are still multiple derivations, - arbitrarily pick the first one. */ if (newestOnly) { - - /* Map from package names to derivations. */ - typedef map<string, std::pair<DrvInfo, unsigned int> > Newest; - Newest newest; - StringSet multiple; - - for (auto & j : matches) { - DrvName drvName(j.first.queryName()); - long d = 1; - - Newest::iterator k = newest.find(drvName.name); - - if (k != newest.end()) { - d = j.first.querySystem() == k->second.first.querySystem() ? 0 : - j.first.querySystem() == settings.thisSystem ? 1 : - k->second.first.querySystem() == settings.thisSystem ? -1 : 0; - if (d == 0) - d = comparePriorities(state, j.first, k->second.first); - if (d == 0) - d = compareVersions(drvName.version, DrvName(k->second.first.queryName()).version); - } - - if (d > 0) { - newest.erase(drvName.name); - newest.insert(Newest::value_type(drvName.name, j)); - multiple.erase(j.first.queryName()); - } else if (d == 0) { - multiple.insert(j.first.queryName()); - } - } - - matches.clear(); - for (auto & j : newest) { - if (multiple.find(j.second.first.queryName()) != multiple.end()) - printInfo( - "warning: there are multiple derivations named '%1%'; using the first one", - j.second.first.queryName()); - matches.push_back(j.second); - } + matches = pickNewestOnly(state, std::move(matches)); } /* Insert only those elements in the final list that we haven't inserted before. */ - for (auto & j : matches) - if (done.insert(j.second).second) - elems.push_back(j.first); + for (auto & match : matches) + if (done.insert(match.index).second) + elems.push_back(match.drvInfo); + + if (selector.hits == 0 && selector.fullName != "*") { + const auto prefixHits = searchByPrefix(allElems, selector.name); + + if (prefixHits.empty()) { + throw Error("selector '%1%' matches no derivations", selector.fullName); + } else { + std::string suggestionMessage = ", maybe you meant:"; + for (const auto & drvName : prefixHits) { + suggestionMessage += fmt("\n%s", drvName); + } + throw Error("selector '%1%' matches no derivations" + suggestionMessage, selector.fullName); + } + } } - checkSelectorUse(selectors); - return elems; } @@ -879,7 +927,7 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems) placeholder.write(nullptr); } else { PathSet context; - printValueAsJSON(*globals.state, true, *v, placeholder, context); + printValueAsJSON(*globals.state, true, *v, noPos, placeholder, context); } } } @@ -1149,10 +1197,10 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } else if (v->type() == nList) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); - for (unsigned int j = 0; j < v->listSize(); ++j) { - if (v->listElems()[j]->type() != nString) continue; + for (auto elem : v->listItems()) { + if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = v->listElems()[j]->string.s; + attrs3["value"] = elem->string.s; xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 25d0fa3ba..19a954ddd 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -50,9 +50,9 @@ void processExpr(EvalState & state, const Strings & attrPaths, else state.autoCallFunction(autoArgs, v, vRes); if (output == okXML) - printValueAsXML(state, strict, location, vRes, std::cout, context); + printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) - printValueAsJSON(state, strict, vRes, std::cout, context); + printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); else { if (strict) state.forceValueDeep(vRes); std::cout << vRes << std::endl; diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 65d61e005..c7517cf79 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -112,7 +112,7 @@ struct CmdEval : MixJSON, InstallableCommand else if (json) { JSONPlaceholder jsonOut(std::cout); - printValueAsJSON(*state, true, *v, jsonOut, context); + printValueAsJSON(*state, true, *v, pos, jsonOut, context); } else { diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index d995d6274..07031c909 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -31,38 +31,38 @@ at the first error. The following flake output attributes must be derivations: * `checks.`*system*`.`*name* -* `defaultPackage.`*system*` -* `devShell.`*system*` -* `devShells.`*system*`.`*name*` -* `nixosConfigurations.`*name*`.config.system.build.toplevel +* `defaultPackage.`*system* +* `devShell.`*system* +* `devShells.`*system*`.`*name* +* `nixosConfigurations.`*name*`.config.system.build.toplevel` * `packages.`*system*`.`*name* The following flake output attributes must be [app definitions](./nix3-run.md): * `apps.`*system*`.`*name* -* `defaultApp.`*system*` +* `defaultApp.`*system* The following flake output attributes must be [template definitions](./nix3-flake-init.md): * `defaultTemplate` -* `templates`.`*name* +* `templates.`*name* The following flake output attributes must be *Nixpkgs overlays*: * `overlay` -* `overlays`.`*name* +* `overlays.`*name* The following flake output attributes must be *NixOS modules*: * `nixosModule` -* `nixosModules`.`*name* +* `nixosModules.`*name* The following flake output attributes must be [bundlers](./nix3-bundle.md): -* `bundlers`.`*name* +* `bundlers.`*name* * `defaultBundler` In addition, the `hydraJobs` output is evaluated in the same way as diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 68bb76742..97f4d911c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -252,6 +252,14 @@ struct CmdFlakeInfo : CmdFlakeMetadata } }; +static bool argHasName(std::string_view arg, std::string_view expected) +{ + return + arg == expected + || arg == "_" + || (hasPrefix(arg, "_") && arg.substr(1) == expected); +} + struct CmdFlakeCheck : FlakeCommand { bool build = true; @@ -346,10 +354,14 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (!v.isLambda() || v.lambda.fun->hasFormals() || std::string(v.lambda.fun->arg) != "final") + if (!v.isLambda() + || v.lambda.fun->hasFormals() + || !argHasName(v.lambda.fun->arg, "final")) throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body); - if (!body || body->hasFormals() || std::string(body->arg) != "prev") + if (!body + || body->hasFormals() + || !argHasName(body->arg, "prev")) throw Error("overlay does not take an argument named 'prev'"); // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. @@ -1040,7 +1052,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON (attrPath.size() == 1 && attrPath[0] == "overlay") || (attrPath.size() == 2 && attrPath[0] == "overlays") ? std::make_pair("nixpkgs-overlay", "Nixpkgs overlay") : attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? std::make_pair("nixos-configuration", "NixOS configuration") : - attrPath.size() == 2 && attrPath[0] == "nixosModules" ? std::make_pair("nixos-module", "NixOS module") : + (attrPath.size() == 1 && attrPath[0] == "nixosModule") + || (attrPath.size() == 2 && attrPath[0] == "nixosModules") ? std::make_pair("nixos-module", "NixOS module") : std::make_pair("unknown", "unknown"); if (json) { j.emplace("type", type); diff --git a/src/nix/main.cc b/src/nix/main.cc index 1e033f4f2..60b0aa410 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -255,6 +255,16 @@ void mainWrapped(int argc, char * * argv) initNix(); initGC(); + #if __linux__ + if (getuid() == 0) { + try { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + } catch (Error & e) { } + } + #endif + programPath = argv[0]; auto programName = std::string(baseNameOf(programPath)); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 6a92576c7..c496f94f8 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -226,6 +226,7 @@ struct CmdRegistry : virtual NixMultiCommand void run() override { + settings.requireExperimentalFeature(Xp::Flakes); if (!command) throw UsageError("'nix registry' requires a sub-command."); command->second->prepare(); diff --git a/src/nix/registry.md b/src/nix/registry.md index a1674bd2e..d5c9ef442 100644 --- a/src/nix/registry.md +++ b/src/nix/registry.md @@ -2,7 +2,7 @@ R""( # Description -`nix flake` provides subcommands for managing *flake +`nix registry` provides subcommands for managing *flake registries*. Flake registries are a convenience feature that allows you to refer to flakes using symbolic identifiers such as `nixpkgs`, rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9c0d22438..f453343f3 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -279,6 +279,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt) }; setupSignals(); + Finally resetTerminal([&]() { rl_deprep_terminal(); }); char * s = readline(prompt.c_str()); Finally doFree([&]() { free(s); }); restoreSignals(); @@ -356,6 +357,8 @@ StringSet NixRepl::completePrefix(string prefix) // Quietly ignore evaluation errors. } catch (UndefinedVarError & e) { // Quietly ignore undefined variable errors. + } catch (BadURL & e) { + // Quietly ignore BadURL flake-related errors. } } @@ -427,7 +430,8 @@ bool NixRepl::processLine(string line) << " :s <expr> Build dependencies of derivation, then start nix-shell\n" << " :t <expr> Describe result of evaluation\n" << " :u <expr> Build derivation, then start nix-shell\n" - << " :doc <expr> Show documentation of a builtin function\n"; + << " :doc <expr> Show documentation of a builtin function\n" + << " :log <expr> Show logs for a derivation\n"; } else if (command == ":a" || command == ":add") { @@ -471,7 +475,10 @@ bool NixRepl::processLine(string line) auto args = editorFor(pos); auto editor = args.front(); args.pop_front(); - runProgram(editor, true, args); + + // runProgram redirects stdout to a StringSink, + // using runProgram2 to allow editors to display their UI + runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); // Reload right after exiting the editor state->resetFileCache(); @@ -494,7 +501,7 @@ bool NixRepl::processLine(string line) runNix("nix-shell", {state->store->printStorePath(drvPath)}); } - else if (command == ":b" || command == ":i" || command == ":s") { + else if (command == ":b" || command == ":i" || command == ":s" || command == ":log") { Value v; evalString(arg, v); StorePath drvPath = getDerivationPath(v); @@ -504,10 +511,31 @@ bool NixRepl::processLine(string line) state->store->buildPaths({DerivedPath::Built{drvPath}}); auto drv = state->store->readDerivation(drvPath); logger->cout("\nThis derivation produced the following outputs:"); - for (auto & i : drv.outputsAndOptPaths(*state->store)) - logger->cout(" %s -> %s", i.first, state->store->printStorePath(*i.second.second)); + for (auto & [outputName, outputPath] : state->store->queryDerivationOutputMap(drvPath)) + logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath)); } else if (command == ":i") { runNix("nix-env", {"-i", drvPathRaw}); + } else if (command == ":log") { + settings.readOnlyMode = true; + Finally roModeReset([&]() { + settings.readOnlyMode = false; + }); + auto subs = getDefaultSubstituters(); + + subs.push_front(state->store); + + bool foundLog = false; + RunPager pager; + for (auto & sub : subs) { + auto log = sub->getBuildLog(drvPath); + if (log) { + printInfo("got build log for '%s' from '%s'", drvPathRaw, sub->getUri()); + logger->writeToStdout(*log); + foundLog = true; + break; + } + } + if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw); } else { runNix("nix-shell", {drvPathRaw}); } @@ -644,7 +672,10 @@ void NixRepl::addVarToScope(const Symbol & name, Value & v) { if (displ >= envSize) throw Error("environment full; cannot add more variables"); - staticEnv.vars[name] = displ; + if (auto oldVar = staticEnv.find(name); oldVar != staticEnv.vars.end()) + staticEnv.vars.erase(oldVar); + staticEnv.vars.emplace_back(name, displ); + staticEnv.sort(); env->values[displ++] = &v; varNames.insert((string) name); } @@ -767,12 +798,12 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << "[ "; if (maxDepth > 0) - for (unsigned int n = 0; n < v.listSize(); ++n) { - if (seen.find(v.listElems()[n]) != seen.end()) + for (auto elem : v.listItems()) { + if (seen.count(elem)) str << "«repeated»"; else try { - printValue(str, *v.listElems()[n], maxDepth - 1, seen); + printValue(str, *elem, maxDepth - 1, seen); } catch (AssertionError & e) { str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; } diff --git a/src/nix/repl.md b/src/nix/repl.md index bba60f871..9b6f2bee3 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -35,14 +35,17 @@ R""( nix-repl> emacs.drvPath "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv" - nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out" + nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello; hello > $out" - nix-repl> :b x + nix-repl> :b drv this derivation produced the following outputs: out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello nix-repl> builtins.readFile drv "Hello, world!\n" + + nix-repl> :log drv + Hello, world! ``` # Description diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 6a238efbe..3d659d6d2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -218,8 +218,7 @@ struct CmdKey : NixMultiCommand void run() override { if (!command) - throw UsageError("'nix flake' requires a sub-command."); - settings.requireExperimentalFeature(Xp::Flakes); + throw UsageError("'nix key' requires a sub-command."); command->second->prepare(); command->second->run(); } |