diff options
Diffstat (limited to 'src')
29 files changed, 401 insertions, 184 deletions
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9bd27e22d..afee89420 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -4,6 +4,7 @@ #include "primops.hh" #include "print-options.hh" #include "shared.hh" +#include "suggestions.hh" #include "types.hh" #include "store-api.hh" #include "derivations.hh" @@ -1426,11 +1427,13 @@ static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & a void ExprSelect::eval(EvalState & state, Env & env, Value & v) { - Value vTmp; - PosIdx pos2; - Value * vAttrs = &vTmp; + Value vFirst; - e->eval(state, env, vTmp); + // Pointer to the current attrset Value in this select chain. + Value * vCurrent = &vFirst; + // Position for the current attrset Value in this select chain. + PosIdx posCurrent; + e->eval(state, env, vFirst); try { auto dts = state.debugRepl @@ -1443,48 +1446,75 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) showAttrPath(state, env, attrPath)) : nullptr; - for (auto & i : attrPath) { + for (auto const & currentAttrName : attrPath) { state.nrLookups++; - Bindings::iterator j; - auto name = getName(i, state, env); - if (def) { - state.forceValue(*vAttrs, pos); - if (vAttrs->type() != nAttrs || - (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) - { - def->eval(state, env, v); + + Symbol const name = getName(currentAttrName, state, env); + + state.forceValue(*vCurrent, pos); + + if (vCurrent->type() != nAttrs) { + + // If we have an `or` provided default, + // then this is allowed to not be an attrset. + if (def != nullptr) { + this->def->eval(state, env, v); return; } - } else { - state.forceAttrs(*vAttrs, pos, "while selecting an attribute"); - if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { - std::set<std::string> allAttrNames; - for (auto & attr : *vAttrs->attrs) - allAttrNames.insert(state.symbols[attr.name]); - auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); - state.error<EvalError>("attribute '%1%' missing", state.symbols[name]) - .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); + + // Otherwise, we must type error. + state.error<TypeError>( + "expected a set but found %s: %s", + showType(*vCurrent), + ValuePrinter(state, *vCurrent, errorPrintOptions) + ).withTrace(pos, "while selecting an attribute").debugThrow(); + } + + // Now that we know this is actually an attrset, try to find an attr + // with the selected name. + Bindings::iterator attrIt = vCurrent->attrs->find(name); + if (attrIt == vCurrent->attrs->end()) { + + // If we have an `or` provided default, then we'll use that. + if (def != nullptr) { + this->def->eval(state, env, v); + return; } + + // Otherwise, missing attr error. + std::set<std::string> allAttrNames; + for (auto const & attr : *vCurrent->attrs) { + allAttrNames.insert(state.symbols[attr.name]); + } + auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); + state.error<EvalError>("attribute '%s' missing", state.symbols[name]) + .atPos(pos) + .withSuggestions(suggestions) + .withFrame(env, *this) + .debugThrow(); } - vAttrs = j->value; - pos2 = j->pos; - if (state.countCalls) state.attrSelects[pos2]++; + + // If we're here, then we successfully found the attribute. + // Set our currently operated-on attrset to this one, and keep going. + vCurrent = attrIt->value; + posCurrent = attrIt->pos; + if (state.countCalls) state.attrSelects[posCurrent]++; } - state.forceValue(*vAttrs, (pos2 ? pos2 : this->pos ) ); + state.forceValue(*vCurrent, (posCurrent ? posCurrent : this->pos)); } catch (Error & e) { - if (pos2) { - auto pos2r = state.positions[pos2]; + if (posCurrent) { + auto pos2r = state.positions[posCurrent]; auto origin = std::get_if<SourcePath>(&pos2r.origin); if (!(origin && *origin == state.derivationInternal)) - state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'", + state.addErrorTrace(e, posCurrent, "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath)); } throw; } - v = *vAttrs; + v = *vCurrent; } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 45d44d1d1..418f888b3 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -157,8 +157,18 @@ struct ExprInheritFrom : ExprVar struct ExprSelect : Expr { PosIdx pos; - std::unique_ptr<Expr> e, def; + + /** The expression attributes are being selected on. e.g. `foo` in `foo.bar.baz`. */ + std::unique_ptr<Expr> e; + + /** A default value specified with `or`, if the selected attr doesn't exist. + * e.g. `bix` in `foo.bar.baz or bix` + */ + std::unique_ptr<Expr> def; + + /** The path of attributes being selected. e.g. `bar.baz` in `foo.bar.baz.` */ AttrPath attrPath; + ExprSelect(const PosIdx & pos, std::unique_ptr<Expr> e, AttrPath attrPath, std::unique_ptr<Expr> def) : pos(pos), e(std::move(e)), def(std::move(def)), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, std::unique_ptr<Expr> e, Symbol name) : pos(pos), e(std::move(e)) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 8fcb10325..0b756301c 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -58,7 +58,7 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) addFlag({ .longName = "log-format", - .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.", + .description = "Set the format of log output; one of `raw`, `internal-json`, `bar`, `bar-with-logs`, `multiline` or `multiline-with-logs`.", .category = loggingCategory, .labels = {"format"}, .handler = {[](std::string format) { setLogFormat(format); }}, diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 80080d616..8c3c4e355 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -17,6 +17,10 @@ LogFormat parseLogFormat(const std::string & logFormatStr) { return LogFormat::bar; else if (logFormatStr == "bar-with-logs") return LogFormat::barWithLogs; + else if (logFormatStr == "multiline") + return LogFormat::multiline; + else if (logFormatStr == "multiline-with-logs") + return LogFormat::multilineWithLogs; throw Error("option 'log-format' has an invalid value '%s'", logFormatStr); } @@ -35,6 +39,17 @@ Logger * makeDefaultLogger() { logger->setPrintBuildLogs(true); return logger; } + case LogFormat::multiline: { + auto logger = makeProgressBar(); + logger->setPrintMultiline(true); + return logger; + } + case LogFormat::multilineWithLogs: { + auto logger = makeProgressBar(); + logger->setPrintMultiline(true); + logger->setPrintBuildLogs(true); + return logger; + } default: abort(); } diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh index e5721420c..6064b6160 100644 --- a/src/libmain/loggers.hh +++ b/src/libmain/loggers.hh @@ -11,6 +11,8 @@ enum class LogFormat { internalJSON, bar, barWithLogs, + multiline, + multilineWithLogs, }; void setLogFormat(const std::string & logFormatStr); diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index b3466a860..b3b46fc21 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -95,8 +95,7 @@ void ProgressBar::logEI(const ErrorInfo & ei) void ProgressBar::log(State & state, Verbosity lvl, std::string_view s) { if (state.active) { - writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); - draw(state); + draw(state, s); } else { auto s2 = s + ANSI_NORMAL "\n"; if (!isTTY) s2 = filterANSIEscapes(s2, true); @@ -234,10 +233,14 @@ void ProgressBar::result(ActivityId act, ResultType type, const std::vector<Fiel } log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); } else { - state->activities.erase(i->second); - info.lastLine = lastLine; - state->activities.emplace_back(info); - i->second = std::prev(state->activities.end()); + if (!printMultiline) { + state->activities.erase(i->second); + info.lastLine = lastLine; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + } else { + i->second->lastLine = lastLine; + } update(*state); } } @@ -290,60 +293,93 @@ void ProgressBar::update(State & state) updateCV.notify_one(); } -std::chrono::milliseconds ProgressBar::draw(State & state) +std::chrono::milliseconds ProgressBar::draw(State & state, const std::optional<std::string_view> & s) { auto nextWakeup = A_LONG_TIME; state.haveUpdate = false; if (state.paused || !state.active) return nextWakeup; - std::string line; + auto windowSize = getWindowSize(); + auto width = windowSize.second; + if (width <= 0) { + width = std::numeric_limits<decltype(width)>::max(); + } + + if (printMultiline && (state.lastLines >= 1)) { + // FIXME: make sure this works on windows + writeToStderr(fmt("\e[G\e[%dF\e[J", state.lastLines)); + } + state.lastLines = 0; + + if (s != std::nullopt) + writeToStderr("\r\e[K" + filterANSIEscapes(s.value(), !isTTY) + ANSI_NORMAL "\n"); + + std::string line; std::string status = getStatus(state); if (!status.empty()) { line += '['; line += status; line += "]"; } + if (printMultiline && !line.empty()) { + writeToStderr(filterANSIEscapes(line, false, width) + "\n"); + state.lastLines++; + } + auto height = windowSize.first > 0 ? windowSize.first : 25; + auto moreActivities = 0; auto now = std::chrono::steady_clock::now(); + std::string activity_line; if (!state.activities.empty()) { - if (!status.empty()) line += " "; - auto i = state.activities.rbegin(); - - while (i != state.activities.rend()) { - if (i->visible && (!i->s.empty() || !i->lastLine.empty())) { - /* Don't show activities until some time has - passed, to avoid displaying very short - activities. */ - auto delay = std::chrono::milliseconds(10); - if (i->startTime + delay < now) - break; - else - nextWakeup = std::min(nextWakeup, std::chrono::duration_cast<std::chrono::milliseconds>(delay - (now - i->startTime))); + for (auto i = state.activities.begin(); i != state.activities.end(); ++i) { + if (!(i->visible && (!i->s.empty() || !i->lastLine.empty()))) { + continue; } - ++i; - } + /* Don't show activities until some time has + passed, to avoid displaying very short + activities. */ + auto delay = std::chrono::milliseconds(10); + if (i->startTime + delay >= now) { + nextWakeup = std::min( + nextWakeup, + std::chrono::duration_cast<std::chrono::milliseconds>( + delay - (now - i->startTime) + ) + ); + } + + activity_line = i->s; - if (i != state.activities.rend()) { - line += i->s; if (!i->phase.empty()) { - line += " ("; - line += i->phase; - line += ")"; + activity_line += " ("; + activity_line += i->phase; + activity_line += ")"; } if (!i->lastLine.empty()) { - if (!i->s.empty()) line += ": "; - line += i->lastLine; + if (!i->s.empty()) + activity_line += ": "; + activity_line += i->lastLine; + } + + if (printMultiline) { + if (state.lastLines < (height -1)) { + writeToStderr(filterANSIEscapes(activity_line, false, width) + "\n"); + state.lastLines++; + } else moreActivities++; } } } - auto width = getWindowSize().second; - if (width <= 0) width = std::numeric_limits<decltype(width)>::max(); + if (printMultiline && moreActivities) + writeToStderr(fmt("And %d more...", moreActivities)); - writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); + if (!printMultiline) { + line += " " + activity_line; + writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); + } return nextWakeup; } @@ -442,9 +478,8 @@ void ProgressBar::writeToStdout(std::string_view s) { auto state(state_.lock()); if (state->active) { - std::cerr << "\r\e[K"; Logger::writeToStdout(s); - draw(*state); + draw(*state, {}); } else { Logger::writeToStdout(s); } @@ -457,7 +492,7 @@ std::optional<char> ProgressBar::ask(std::string_view msg) std::cerr << fmt("\r\e[K%s ", msg); auto s = trim(readLine(STDIN_FILENO)); if (s.size() != 1) return {}; - draw(*state); + draw(*state, {}); return s[0]; } @@ -466,6 +501,11 @@ void ProgressBar::setPrintBuildLogs(bool printBuildLogs) this->printBuildLogs = printBuildLogs; } +void ProgressBar::setPrintMultiline(bool printMultiline) +{ + this->printMultiline = printMultiline; +} + Logger * makeProgressBar() { return new ProgressBar(shouldANSI()); diff --git a/src/libmain/progress-bar.hh b/src/libmain/progress-bar.hh index 76e2ed4ff..176e941e8 100644 --- a/src/libmain/progress-bar.hh +++ b/src/libmain/progress-bar.hh @@ -47,6 +47,8 @@ struct ProgressBar : public Logger std::map<ActivityType, ActivitiesByType> activitiesByType; + int lastLines = 0; + uint64_t filesLinked = 0, bytesLinked = 0; uint64_t corruptedPaths = 0, untrustedPaths = 0; @@ -63,6 +65,7 @@ struct ProgressBar : public Logger std::condition_variable quitCV, updateCV; bool printBuildLogs = false; + bool printMultiline = false; bool isTTY; ProgressBar(bool isTTY) @@ -75,7 +78,7 @@ struct ProgressBar : public Logger while (state->active) { if (!state->haveUpdate) state.wait_for(updateCV, nextWakeup); - nextWakeup = draw(*state); + nextWakeup = draw(*state, {}); state.wait_for(quitCV, std::chrono::milliseconds(50)); } }); @@ -114,7 +117,7 @@ struct ProgressBar : public Logger void update(State & state); - std::chrono::milliseconds draw(State & state); + std::chrono::milliseconds draw(State & state, const std::optional<std::string_view> & s); std::string getStatus(State & state); @@ -123,6 +126,8 @@ struct ProgressBar : public Logger std::optional<char> ask(std::string_view msg) override; void setPrintBuildLogs(bool printBuildLogs) override; + + void setPrintMultiline(bool printMultiline) override; }; Logger * makeProgressBar(); diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index f99777a20..29538a9ca 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -378,7 +378,7 @@ RunPager::RunPager() RunPager::~RunPager() { try { - if (pid != -1) { + if (pid) { std::cout.flush(); dup2(std_out, STDOUT_FILENO); pid.wait(); diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 86f72486e..9a93646f4 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -79,7 +79,7 @@ HookInstance::~HookInstance() { try { toHook.writeSide.reset(); - if (pid != -1) pid.kill(); + if (pid) pid.kill(); } catch (...) { ignoreException(); } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index bae821266..efba648a4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -51,9 +51,6 @@ #endif #if __APPLE__ -#include <spawn.h> -#include <sys/sysctl.h> - /* This definition is undocumented but depended upon by all major browsers. */ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf); #endif @@ -140,7 +137,7 @@ LocalStore & LocalDerivationGoal::getLocalStore() void LocalDerivationGoal::killChild() { - if (pid != -1) { + if (pid) { worker.childTerminated(this); /* If we're using a build user, then there is a tricky race @@ -148,7 +145,7 @@ void LocalDerivationGoal::killChild() done its setuid() to the build user uid, then it won't be killed, and we'll potentially lock up in pid.wait(). So also send a conventional kill to the child. */ - ::kill(-pid, SIGKILL); /* ignore the result */ + ::kill(-pid.get(), SIGKILL); /* ignore the result */ killSandbox(true); @@ -161,19 +158,7 @@ void LocalDerivationGoal::killChild() void LocalDerivationGoal::killSandbox(bool getStats) { - if (cgroup) { - #if __linux__ - auto stats = destroyCgroup(*cgroup); - if (getStats) { - buildResult.cpuUser = stats.cpuUser; - buildResult.cpuSystem = stats.cpuSystem; - } - #else - abort(); - #endif - } - - else if (buildUser) { + if (buildUser) { auto uid = buildUser->getUID(); assert(uid != 0); killUser(uid); @@ -945,7 +930,7 @@ void LocalDerivationGoal::startBuilder() if (usingUserNamespace) options.cloneFlags |= CLONE_NEWUSER; - pid_t child = startProcess([&]() { runChild(); }, options); + pid_t child = startProcess([&]() { runChild(); }, options).release(); writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); _exit(0); @@ -966,7 +951,7 @@ void LocalDerivationGoal::startBuilder() auto ss = tokenizeString<std::vector<std::string>>(readLine(sendPid.readSide.get())); assert(ss.size() == 1); - pid = string2Int<pid_t>(ss[0]).value(); + pid = Pid{string2Int<pid_t>(ss[0]).value()}; if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace @@ -976,13 +961,13 @@ void LocalDerivationGoal::startBuilder() uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); uid_t nrIds = buildUser ? buildUser->getUIDCount() : 1; - writeFile("/proc/" + std::to_string(pid) + "/uid_map", + writeFile("/proc/" + std::to_string(pid.get()) + "/uid_map", fmt("%d %d %d", sandboxUid(), hostUid, nrIds)); if (!buildUser || buildUser->getUIDCount() == 1) - writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + writeFile("/proc/" + std::to_string(pid.get()) + "/setgroups", "deny"); - writeFile("/proc/" + std::to_string(pid) + "/gid_map", + writeFile("/proc/" + std::to_string(pid.get()) + "/gid_map", fmt("%d %d %d", sandboxGid(), hostGid, nrIds)); } else { debug("note: not using a user namespace"); @@ -1005,19 +990,19 @@ void LocalDerivationGoal::startBuilder() /* Save the mount- and user namespace of the child. We have to do this *before* the child does a chroot. */ - sandboxMountNamespace = AutoCloseFD{open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY)}; + sandboxMountNamespace = AutoCloseFD{open(fmt("/proc/%d/ns/mnt", pid.get()).c_str(), O_RDONLY)}; if (sandboxMountNamespace.get() == -1) throw SysError("getting sandbox mount namespace"); if (usingUserNamespace) { - sandboxUserNamespace = AutoCloseFD{open(fmt("/proc/%d/ns/user", (pid_t) pid).c_str(), O_RDONLY)}; + sandboxUserNamespace = AutoCloseFD{open(fmt("/proc/%d/ns/user", pid.get()).c_str(), O_RDONLY)}; if (sandboxUserNamespace.get() == -1) throw SysError("getting sandbox user namespace"); } /* Move the child into its own cgroup. */ if (cgroup) - writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); + writeFile(*cgroup + "/cgroup.procs", fmt("%d", pid.get())); /* Signal the builder that we've updated its user namespace. */ writeFull(userNamespaceSync.writeSide.get(), "1"); @@ -1585,7 +1570,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path) entering its mount namespace, which is not possible in multithreaded programs. So we do this in a child process.*/ - Pid child(startProcess([&]() { + Pid child = startProcess([&]() { if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) throw SysError("entering sandbox user namespace"); @@ -1596,7 +1581,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path) doBind(source, target); _exit(0); - })); + }); int status = child.wait(); if (status != 0) @@ -2177,31 +2162,8 @@ void LocalDerivationGoal::runChild() } } -#if __APPLE__ - posix_spawnattr_t attrp; - - if (posix_spawnattr_init(&attrp)) - throw SysError("failed to initialize builder"); - - if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) - throw SysError("failed to initialize builder"); - - if (drv->platform == "aarch64-darwin") { - // Unset kern.curproc_arch_affinity so we can escape Rosetta - int affinity = 0; - sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); - - cpu_type_t cpu = CPU_TYPE_ARM64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } else if (drv->platform == "x86_64-darwin") { - cpu_type_t cpu = CPU_TYPE_X86_64; - posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); - } - - posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -#else - execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); -#endif + execBuilder(drv->builder, args, envStrs); + // execBuilder should not return throw SysError("executing '%1%'", drv->builder); @@ -2217,6 +2179,11 @@ void LocalDerivationGoal::runChild() } } +void LocalDerivationGoal::execBuilder(std::string builder, Strings args, Strings envStrs) +{ + execve(builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +} + SingleDrvOutputs LocalDerivationGoal::registerOutputs() { diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index f3a83d42f..91329ca35 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -178,7 +178,28 @@ struct LocalDerivationGoal : public DerivationGoal friend struct RestrictedStore; - using DerivationGoal::DerivationGoal; + /** + * Create a LocalDerivationGoal without an on-disk .drv file, + * possibly a platform-specific subclass + */ + static std::shared_ptr<LocalDerivationGoal> makeLocalDerivationGoal( + const StorePath & drvPath, + const OutputsSpec & wantedOutputs, + Worker & worker, + BuildMode buildMode + ); + + /** + * Create a LocalDerivationGoal for an on-disk .drv file, + * possibly a platform-specific subclass + */ + static std::shared_ptr<LocalDerivationGoal> makeLocalDerivationGoal( + const StorePath & drvPath, + const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, + Worker & worker, + BuildMode buildMode + ); virtual ~LocalDerivationGoal() noexcept(false) override; @@ -282,7 +303,7 @@ struct LocalDerivationGoal : public DerivationGoal * Kill any processes running under the build user UID or in the * cgroup of the build. */ - void killSandbox(bool getStats); + virtual void killSandbox(bool getStats); /** * Create alternative path calculated from but distinct from the @@ -299,6 +320,16 @@ struct LocalDerivationGoal : public DerivationGoal * rewrites caught everything */ StorePath makeFallbackPath(OutputNameView outputName); + +protected: + using DerivationGoal::DerivationGoal; + + /** + * Execute the builder, replacing the current process. + * Generally this means an `execve` call. + */ + virtual void execBuilder(std::string builder, Strings args, Strings envStrs); + }; } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index d8d9ba283..cc4cb3c8c 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -217,6 +217,7 @@ void PathSubstitutionGoal::tryToRun() promise = std::promise<void>(); thr = std::thread([this]() { + auto & fetchPath = subPath ? *subPath : storePath; try { ReceiveInterrupts receiveInterrupts; @@ -226,10 +227,17 @@ void PathSubstitutionGoal::tryToRun() Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); PushActivity pact(act.id); - copyStorePath(*sub, worker.store, - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + copyStorePath( + *sub, worker.store, fetchPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs + ); promise.set_value(); + } catch (const EndOfFile &) { + promise.set_exception(std::make_exception_ptr(EndOfFile( + "NAR for '%s' fetched from '%s' is incomplete", + sub->printStorePath(fetchPath), + sub->getUri() + ))); } catch (...) { promise.set_exception(std::current_exception()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 04be0da99..a7a298c34 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -65,8 +65,8 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> { return !dynamic_cast<LocalStore *>(&store) - ? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode) - : std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode); + ? std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode) + : LocalDerivationGoal::makeLocalDerivationGoal(drvPath, wantedOutputs, *this, buildMode); }); } @@ -76,8 +76,8 @@ std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> { return !dynamic_cast<LocalStore *>(&store) - ? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode) - : std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode); + ? std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode) + : LocalDerivationGoal::makeLocalDerivationGoal(drvPath, drv, wantedOutputs, *this, buildMode); }); } diff --git a/src/libstore/platform.cc b/src/libstore/platform.cc index acdedab99..d10d33f0e 100644 --- a/src/libstore/platform.cc +++ b/src/libstore/platform.cc @@ -1,4 +1,5 @@ #include "local-store.hh" +#include "build/local-derivation-goal.hh" #if __linux__ #include "platform/linux.hh" @@ -19,4 +20,43 @@ std::shared_ptr<LocalStore> LocalStore::makeLocalStore(const Params & params) return std::shared_ptr<LocalStore>(new FallbackLocalStore(params)); #endif } + +std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal( + const StorePath & drvPath, + const OutputsSpec & wantedOutputs, + Worker & worker, + BuildMode buildMode +) +{ +#if __linux__ + return std::make_shared<LinuxLocalDerivationGoal>(drvPath, wantedOutputs, worker, buildMode); +#elif __APPLE__ + return std::make_shared<DarwinLocalDerivationGoal>(drvPath, wantedOutputs, worker, buildMode); +#else + return std::make_shared<FallbackLocalDerivationGoal>(drvPath, wantedOutputs, worker, buildMode); +#endif +} + +std::shared_ptr<LocalDerivationGoal> LocalDerivationGoal::makeLocalDerivationGoal( + const StorePath & drvPath, + const BasicDerivation & drv, + const OutputsSpec & wantedOutputs, + Worker & worker, + BuildMode buildMode +) +{ +#if __linux__ + return std::make_shared<LinuxLocalDerivationGoal>( + drvPath, drv, wantedOutputs, worker, buildMode + ); +#elif __APPLE__ + return std::make_shared<DarwinLocalDerivationGoal>( + drvPath, drv, wantedOutputs, worker, buildMode + ); +#else + return std::make_shared<FallbackLocalDerivationGoal>( + drvPath, drv, wantedOutputs, worker, buildMode + ); +#endif +} } diff --git a/src/libstore/platform/darwin.cc b/src/libstore/platform/darwin.cc index bbb81784c..83b4b4183 100644 --- a/src/libstore/platform/darwin.cc +++ b/src/libstore/platform/darwin.cc @@ -6,6 +6,7 @@ #include <sys/proc_info.h> #include <sys/sysctl.h> #include <libproc.h> +#include <spawn.h> #include <regex> @@ -220,4 +221,29 @@ void DarwinLocalStore::findPlatformRoots(UncheckedRoots & unchecked) } } } + +void DarwinLocalDerivationGoal::execBuilder(std::string builder, Strings args, Strings envStrs) +{ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +} } diff --git a/src/libstore/platform/darwin.hh b/src/libstore/platform/darwin.hh index b7170aa05..0ac7077fb 100644 --- a/src/libstore/platform/darwin.hh +++ b/src/libstore/platform/darwin.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "build/local-derivation-goal.hh" #include "gc-store.hh" #include "local-store.hh" @@ -32,4 +33,19 @@ private: void findPlatformRoots(UncheckedRoots & unchecked) override; }; +/** + * Darwin-specific implementation of LocalDerivationGoal + */ +class DarwinLocalDerivationGoal : public LocalDerivationGoal +{ +public: + using LocalDerivationGoal::LocalDerivationGoal; + +private: + /** + * Set process flags to enter or leave rosetta, then execute the builder + */ + void execBuilder(std::string builder, Strings args, Strings envStrs) override; +}; + } diff --git a/src/libstore/platform/fallback.hh b/src/libstore/platform/fallback.hh index fd27edbe6..3b77536bb 100644 --- a/src/libstore/platform/fallback.hh +++ b/src/libstore/platform/fallback.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "build/local-derivation-goal.hh" #include "local-store.hh" namespace nix { @@ -28,4 +29,14 @@ public: } }; +/** + * Fallback platform implementation of LocalDerivationGoal + * Exists so we can make LocalDerivationGoal constructor protected + */ +class FallbackLocalDerivationGoal : public LocalDerivationGoal +{ +public: + using LocalDerivationGoal::LocalDerivationGoal; +}; + } diff --git a/src/libstore/platform/linux.cc b/src/libstore/platform/linux.cc index a34608894..6b94c01cc 100644 --- a/src/libstore/platform/linux.cc +++ b/src/libstore/platform/linux.cc @@ -1,3 +1,4 @@ +#include "cgroup.hh" #include "gc-store.hh" #include "signals.hh" #include "platform/linux.hh" @@ -114,4 +115,17 @@ void LinuxLocalStore::findPlatformRoots(UncheckedRoots & unchecked) readFileRoots("/proc/sys/kernel/fbsplash", unchecked); readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); } + +void LinuxLocalDerivationGoal::killSandbox(bool getStats) +{ + if (cgroup) { + auto stats = destroyCgroup(*cgroup); + if (getStats) { + buildResult.cpuUser = stats.cpuUser; + buildResult.cpuSystem = stats.cpuSystem; + } + } else { + LocalDerivationGoal::killSandbox(getStats); + } +} } diff --git a/src/libstore/platform/linux.hh b/src/libstore/platform/linux.hh index 8b97e17c5..2cad001ea 100644 --- a/src/libstore/platform/linux.hh +++ b/src/libstore/platform/linux.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "build/local-derivation-goal.hh" #include "gc-store.hh" #include "local-store.hh" @@ -32,4 +33,16 @@ private: void findPlatformRoots(UncheckedRoots & unchecked) override; }; +/** + * Linux-specific implementation of LocalDerivationGoal + */ +class LinuxLocalDerivationGoal : public LocalDerivationGoal +{ +public: + using LocalDerivationGoal::LocalDerivationGoal; + +private: + void killSandbox(bool getStatus) override; +}; + } diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 932ebaa52..0d7bfa01d 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -131,7 +131,7 @@ Path SSHMaster::startMaster() auto state(state_.lock()); - if (state->sshMaster != -1) return state->socketPath; + if (state->sshMaster) return state->socketPath; state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 244ecf256..7c0902978 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1072,8 +1072,6 @@ void copyStorePath( }); TeeSink tee { sink, progressSink }; srcStore.narFromPath(storePath, tee); - }, [&]() { - throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", srcStore.printStorePath(storePath), srcStore.getUri()); }); dstStore.addToStore(*info, *source, repair, checkSigs); diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 64be8bc62..115b979f8 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -114,6 +114,9 @@ public: virtual void setPrintBuildLogs(bool printBuildLogs) { } + + virtual void setPrintMultiline(bool printMultiline) + { } }; /** diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index 98d3cd306..247fba2c4 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -94,12 +94,7 @@ bool userNamespacesSupported() static auto res = [&]() -> bool { try { - Pid pid = startProcess([&]() - { - _exit(0); - }, { - .cloneFlags = CLONE_NEWUSER - }); + Pid pid = startProcess([&]() { _exit(0); }, {.cloneFlags = CLONE_NEWUSER}); auto r = pid.wait(); assert(!r); @@ -120,8 +115,7 @@ bool mountAndPidNamespacesSupported() { try { - Pid pid = startProcess([&]() - { + Pid pid = startProcess([&]() { /* Make sure we don't remount the parent's /proc. */ if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1) _exit(1); diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc index 8639a77f8..e8af12fbd 100644 --- a/src/libutil/processes.cc +++ b/src/libutil/processes.cc @@ -35,29 +35,25 @@ Pid::Pid() } -Pid::Pid(pid_t pid) - : pid(pid) +Pid::Pid(Pid && other) : pid(other.pid), separatePG(other.separatePG), killSignal(other.killSignal) { + other.pid = -1; } -Pid::~Pid() noexcept(false) +Pid & Pid::operator=(Pid && other) { - if (pid != -1) kill(); + Pid tmp(std::move(other)); + std::swap(pid, tmp.pid); + std::swap(separatePG, tmp.separatePG); + std::swap(killSignal, tmp.killSignal); + return *this; } -void Pid::operator =(pid_t pid) -{ - if (this->pid != -1 && this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default -} - - -Pid::operator pid_t() +Pid::~Pid() noexcept(false) { - return pid; + if (pid != -1) kill(); } @@ -131,7 +127,7 @@ void killUser(uid_t uid) users to which the current process can send signals. So we fork a process, switch to uid, and send a mass kill. */ - Pid pid = startProcess([&]() { + Pid pid{startProcess([&]() { if (setuid(uid) == -1) throw SysError("setting uid"); @@ -153,7 +149,7 @@ void killUser(uid_t uid) } _exit(0); - }); + })}; int status = pid.wait(); if (status != 0) @@ -187,7 +183,7 @@ static int childEntry(void * arg) #endif -pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) +Pid startProcess(std::function<void()> fun, const ProcessOptions & options) { std::function<void()> wrapper = [&]() { logger = makeSimpleLogger(); @@ -231,7 +227,7 @@ pid_t startProcess(std::function<void()> fun, const ProcessOptions & options) if (pid == -1) throw SysError("unable to fork"); - return pid; + return Pid{pid}; } std::string runProgram(Path program, bool searchPath, const Strings & args, @@ -294,7 +290,7 @@ void runProgram2(const RunOptions & options) } /* Fork. */ - Pid pid = startProcess([&]() { + Pid pid{startProcess([&]() { if (options.environment) replaceEnv(*options.environment); if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) @@ -328,7 +324,7 @@ void runProgram2(const RunOptions & options) execv(options.program.c_str(), stringsToCharPtrs(args_).data()); throw SysError("executing '%1%'", options.program); - }, processOptions); + }, processOptions)}; out.writeSide.close(); diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh index b84fc7c4b..001e1fa12 100644 --- a/src/libutil/processes.hh +++ b/src/libutil/processes.hh @@ -26,16 +26,18 @@ class Pid int killSignal = SIGKILL; public: Pid(); - Pid(pid_t pid); + explicit Pid(pid_t pid): pid(pid) {} + Pid(Pid && other); + Pid & operator=(Pid && other); ~Pid() noexcept(false); - void operator =(pid_t pid); - operator pid_t(); + explicit operator bool() const { return pid != -1; } int kill(); int wait(); void setSeparatePG(bool separatePG); void setKillSignal(int signal); pid_t release(); + pid_t get() const { return pid; } }; /** @@ -60,7 +62,8 @@ struct ProcessOptions int cloneFlags = 0; }; -pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); +[[nodiscard]] +Pid startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions()); /** diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 3a8a01f16..80b111f08 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -266,20 +266,17 @@ std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun) } -std::unique_ptr<Source> sinkToSource( - std::function<void(Sink &)> fun, - std::function<void()> eof) +std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun) { struct SinkToSource : Source { typedef boost::coroutines2::coroutine<std::string> coro_t; std::function<void(Sink &)> fun; - std::function<void()> eof; std::optional<coro_t::pull_type> coro; - SinkToSource(std::function<void(Sink &)> fun, std::function<void()> eof) - : fun(fun), eof(eof) + SinkToSource(std::function<void(Sink &)> fun) + : fun(fun) { } @@ -298,7 +295,9 @@ std::unique_ptr<Source> sinkToSource( }); } - if (!*coro) { eof(); abort(); } + if (!*coro) { + throw EndOfFile("coroutine has finished"); + } if (pos == cur.size()) { if (!cur.empty()) { @@ -317,7 +316,7 @@ std::unique_ptr<Source> sinkToSource( } }; - return std::make_unique<SinkToSource>(fun, eof); + return std::make_unique<SinkToSource>(fun); } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index c9294ba2d..0632e3109 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -338,11 +338,7 @@ std::unique_ptr<FinishSink> sourceToSink(std::function<void(Source &)> fun); * Convert a function that feeds data into a Sink into a Source. The * Source executes the function as a coroutine. */ -std::unique_ptr<Source> sinkToSource( - std::function<void(Sink &)> fun, - std::function<void()> eof = []() { - throw EndOfFile("coroutine has finished"); - }); +std::unique_ptr<Source> sinkToSource(std::function<void(Sink &)> fun); void writePadding(size_t len, Sink & sink); diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc index a6e46ca50..d4fc37fab 100644 --- a/src/libutil/unix-domain-socket.cc +++ b/src/libutil/unix-domain-socket.cc @@ -65,7 +65,7 @@ static void bindConnectProcHelper( if (path.size() + 1 >= sizeof(addr.sun_path)) { Pipe pipe; pipe.create(); - Pid pid = startProcess([&] { + Pid pid{startProcess([&] { try { pipe.readSide.close(); Path dir = dirOf(path); @@ -83,7 +83,7 @@ static void bindConnectProcHelper( } catch (...) { writeFull(pipe.writeSide.get(), "-1\n"); } - }); + })}; pipe.writeSide.close(); auto errNo = string2Int<int>(chomp(drainFD(pipe.readSide.get()))); if (!errNo || *errNo == -1) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index f1cc1ee9c..a9211d64a 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -369,7 +369,7 @@ static void daemonLoop(std::optional<TrustedFlag> forceTrustClientOpt) processConnection(openUncachedStore(), from, to, trusted, NotRecursive); exit(0); - }, options); + }, options).release(); } catch (Interrupted & e) { return; |