diff options
Diffstat (limited to 'src')
24 files changed, 711 insertions, 211 deletions
diff --git a/src/asan-options/asan-options.cc b/src/asan-options/asan-options.cc new file mode 100644 index 000000000..c4cf360af --- /dev/null +++ b/src/asan-options/asan-options.cc @@ -0,0 +1,17 @@ +/// @file This is very bothersome code that has to be included in every +/// executable to get the correct default ASan options. I am so sorry. + +extern "C" [[gnu::retain]] const char *__asan_default_options() +{ + // We leak a bunch of memory knowingly on purpose. It's not worthwhile to + // diagnose that memory being leaked for now. + // + // Instruction bytes are useful for finding the actual code that + // corresponds to an ASan report. + // + // TODO: setting log_path=asan.log or not: neither works, since you can't + // write to the fs in certain places in the testsuite, but you also cannot + // write arbitrarily to stderr in other places so the reports get eaten. + // pain 🥖 + return "halt_on_error=1:abort_on_error=1:detect_leaks=0:print_summary=1:dump_instruction_bytes=1"; +} diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index eb15fecc3..9891b263c 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -1,23 +1,11 @@ -#include "globals.hh" #include "installable-attr-path.hh" #include "outputs-spec.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" -#include "derivations.hh" -#include "eval-inline.hh" #include "eval.hh" #include "get-drvs.hh" -#include "store-api.hh" -#include "shared.hh" #include "flake/flake.hh" -#include "eval-cache.hh" -#include "url.hh" -#include "registry.hh" -#include "build-result.hh" - -#include <regex> -#include <queue> #include <nlohmann/json.hpp> diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh index 86c2f8219..edc9c2906 100644 --- a/src/libcmd/installable-attr-path.hh +++ b/src/libcmd/installable-attr-path.hh @@ -1,25 +1,11 @@ #pragma once ///@file -#include "globals.hh" #include "installable-value.hh" #include "outputs-spec.hh" #include "command.hh" -#include "attr-path.hh" #include "common-eval-args.hh" -#include "derivations.hh" -#include "eval-inline.hh" #include "eval.hh" -#include "get-drvs.hh" -#include "store-api.hh" -#include "shared.hh" -#include "eval-cache.hh" -#include "url.hh" -#include "registry.hh" -#include "build-result.hh" - -#include <regex> -#include <queue> #include <nlohmann/json.hpp> diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 635b54958..5086e9999 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -652,9 +652,12 @@ ProcessLineResult NixRepl::processLine(std::string line) // using runProgram2 to allow editors to display their UI runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }).wait(); - // Reload right after exiting the editor - state->resetFileCache(); - reloadFiles(); + // Reload right after exiting the editor if path is not in store + // Store is immutable, so there could be no changes, so there's no need to reload + if (!state->store->isInStore(path.resolveSymlinks().path.abs())) { + state->resetFileCache(); + reloadFiles(); + } } else if (command == ":t") { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c0e7a9a2e..a925ce2d8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -494,6 +494,14 @@ std::ostream & operator<<(std::ostream & output, PrimOp & primOp) } +Value::Value(primop_t, PrimOp & primop) + : internalType(tPrimOp) + , primOp(&primop) + , _primop_pad(0) +{ + primop.check(); +} + PrimOp * Value::primOpAppPrimOp() const { Value * left = primOpApp.left; @@ -506,7 +514,6 @@ PrimOp * Value::primOpAppPrimOp() const return left->primOp; } - void Value::mkPrimOp(PrimOp * p) { p->check(); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 8e3969aac..d7869d09b 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,6 +1,7 @@ #include "get-drvs.hh" #include "eval-inline.hh" #include "derivations.hh" +#include "eval.hh" #include "store-api.hh" #include "path-with-outputs.hh" @@ -101,66 +102,134 @@ StorePath DrvInfo::queryOutPath() return *outPath; } +void DrvInfo::fillOutputs(bool withPaths) +{ + auto fillDefault = [&]() { + std::optional<StorePath> outPath = std::nullopt; + if (withPaths) { + outPath.emplace(this->queryOutPath()); + } + this->outputs.emplace("out", outPath); + }; + + // lol. lmao even. + if (this->attrs == nullptr) { + fillDefault(); + return; + } + + Attr * outputs = this->attrs->get(this->state->sOutputs); + if (outputs == nullptr) { + fillDefault(); + return; + } + + // NOTE(Qyriad): I don't think there is any codepath that can cause this to error. + this->state->forceList( + *outputs->value, + outputs->pos, + "while evaluating the 'outputs' attribute of a derivation" + ); + + for (auto [idx, elem] : enumerate(outputs->value->listItems())) { + // NOTE(Qyriad): This error should be *extremely* rare in practice. + // It is impossible to construct with `stdenv.mkDerivation`, + // `builtins.derivation`, or even `derivationStrict`. As far as we can tell, + // it is only possible by overriding a derivation attrset already created by + // one of those with `//` to introduce the failing `outputs` entry. + auto errMsg = fmt("while evaluating output %d of a derivation", idx); + std::string_view outputName = state->forceStringNoCtx( + *elem, + outputs->pos, + errMsg + ); + + if (withPaths) { + // Find the attr with this output's name... + Attr * out = this->attrs->get(this->state->symbols.create(outputName)); + if (out == nullptr) { + // FIXME: throw error? + continue; + } + + // Meanwhile we couldn't figure out any circumstances + // that cause this to error. + state->forceAttrs(*out->value, outputs->pos, errMsg); + + // ...and evaluate its `outPath` attribute. + Attr * outPath = out->value->attrs->get(this->state->sOutPath); + if (outPath == nullptr) { + continue; + // FIXME: throw error? + } + + NixStringContext context; + // And idk what could possibly cause this one to error + // that wouldn't error before here. + auto storePath = state->coerceToStorePath( + outPath->pos, + *outPath->value, + context, + errMsg + ); + this->outputs.emplace(outputName, storePath); + } else { + this->outputs.emplace(outputName, std::nullopt); + } + } +} DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall) { + // If we haven't already cached the outputs set, then do so now. if (outputs.empty()) { - /* Get the ‘outputs’ list. */ - Bindings::iterator i; - if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { - state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation"); - - /* For each output... */ - for (auto elem : i->value->listItems()) { - std::string output(state->forceStringNoCtx(*elem, i->pos, "while evaluating the name of an output of a derivation")); - - if (withPaths) { - /* Evaluate the corresponding set. */ - Bindings::iterator out = attrs->find(state->symbols.create(output)); - if (out == attrs->end()) continue; // FIXME: throw error? - state->forceAttrs(*out->value, i->pos, "while evaluating an output of a derivation"); - - /* And evaluate its ‘outPath’ attribute. */ - Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); - if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - NixStringContext context; - outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation")); - } else - outputs.emplace(output, std::nullopt); - } - } else - outputs.emplace("out", withPaths ? std::optional{queryOutPath()} : std::nullopt); + // FIXME: this behavior seems kind of busted, since whether or not this + // DrvInfo will have paths is forever determined by the *first* call to + // this function?? + fillOutputs(withPaths); } + // Things that operate on derivations like packages, like `nix-env` and `nix build`, + // allow derivations to specify which outputs should be used in those user-facing + // cases if the user didn't specify an output explicitly. + // If the caller just wanted all the outputs for this derivation, though, + // then we're done here. if (!onlyOutputsToInstall || !attrs) return outputs; - Bindings::iterator i; - if (attrs && (i = attrs->find(state->sOutputSpecified)) != attrs->end() && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) { - Outputs result; - auto out = outputs.find(queryOutputName()); - if (out == outputs.end()) - throw Error("derivation does not have output '%s'", queryOutputName()); - result.insert(*out); - return result; + // Regardless of `meta.outputsToInstall`, though, you can select into a derivation + // output by its attribute, e.g. `pkgs.lix.dev`, which (lol?) sets the magic + // attribute `outputSpecified = true`, and changes the `outputName` attr to the + // explicitly selected-into output. + if (Attr * outSpecAttr = attrs->get(state->sOutputSpecified)) { + bool outputSpecified = this->state->forceBool( + *outSpecAttr->value, + outSpecAttr->pos, + "while evaluating the 'outputSpecified' attribute of a derivation" + ); + if (outputSpecified) { + auto maybeOut = outputs.find(queryOutputName()); + if (maybeOut == outputs.end()) { + throw Error("derivation does not have output '%s'", queryOutputName()); + } + return Outputs{*maybeOut}; + } } - else { - /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ - const Value * outTI = queryMeta("outputsToInstall"); - if (!outTI) return outputs; - auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); - /* ^ this shows during `nix-env -i` right under the bad derivation */ - if (!outTI->isList()) throw errMsg; - Outputs result; - for (auto elem : outTI->listItems()) { - if (elem->type() != nString) throw errMsg; - auto out = outputs.find(elem->string.s); - if (out == outputs.end()) throw errMsg; - result.insert(*out); - } - return result; + /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ + const Value * outTI = queryMeta("outputsToInstall"); + if (!outTI) return outputs; + auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); + /* ^ this shows during `nix-env -i` right under the bad derivation */ + if (!outTI->isList()) throw errMsg; + Outputs result; + for (auto elem : outTI->listItems()) { + if (elem->type() != nString) throw errMsg; + auto out = outputs.find(elem->string.s); + if (out == outputs.end()) throw errMsg; + result.insert(*out); } + return result; } @@ -350,56 +419,95 @@ static void getDerivations(EvalState & state, Value & vIn, Value v; state.autoCallFunction(autoArgs, vIn, v); - /* Process the expression. */ - if (!getDerivation(state, v, pathPrefix, drvs, ignoreAssertionFailures)) ; - - else if (v.type() == nAttrs) { - - /* Dont consider sets we've already seen, e.g. y in - `rec { x.d = derivation {...}; y = x; }`. */ - if (!done.insert(v.attrs).second) return; - - /* !!! undocumented hackery to support combining channels in - nix-env.cc. */ - bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end(); - - /* Consider the attributes in sorted order to get more - deterministic behaviour in nix-env operations (e.g. when - there are names clashes between derivations, the derivation - bound to the attribute with the "lower" name should take - precedence). */ - for (auto & i : v.attrs->lexicographicOrder(state.symbols)) { - debug("evaluating attribute '%1%'", state.symbols[i->name]); - if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex)) - continue; - std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]); - if (combineChannels) - getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - else if (getDerivation(state, *i->value, pathPrefix2, drvs, ignoreAssertionFailures)) { - /* If the value of this attribute is itself a set, - should we recurse into it? => Only if it has a - `recurseForDerivations = true' attribute. */ - if (i->value->type() == nAttrs) { - Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); - if (j != i->value->attrs->end() && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) - getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - } - } - } + bool shouldRecurse = getDerivation(state, v, pathPrefix, drvs, ignoreAssertionFailures); + if (!shouldRecurse) { + // We're done here. + return; } - else if (v.type() == nList) { + if (v.type() == nList) { // NOTE we can't really deduplicate here because small lists don't have stable addresses // and can cause spurious duplicate detections due to v being on the stack. for (auto [n, elem] : enumerate(v.listItems())) { - std::string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n)); - if (getDerivation(state, *elem, pathPrefix2, drvs, ignoreAssertionFailures)) - getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + std::string joinedAttrPath = addToPath(pathPrefix, fmt("%d", n)); + bool shouldRecurse = getDerivation(state, *elem, joinedAttrPath, drvs, ignoreAssertionFailures); + if (shouldRecurse) { + getDerivations( + state, + *elem, + joinedAttrPath, + autoArgs, + drvs, + done, + ignoreAssertionFailures + ); + } } + + return; + } else if (v.type() != nAttrs) { + state.error<TypeError>( + "expression was expected to be a derivation or collection of derivations, but instead was %s", + showType(v.type(), true) + ).debugThrow(); + } + + /* Dont consider sets we've already seen, e.g. y in + `rec { x.d = derivation {...}; y = x; }`. */ + auto const &[_, didInsert] = done.insert(v.attrs); + if (!didInsert) { + return; } - else - state.error<TypeError>("expression does not evaluate to a derivation (or a set or list of those)").debugThrow(); + // FIXME: what the fuck??? + /* !!! undocumented hackery to support combining channels in + nix-env.cc. */ + bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end(); + + /* Consider the attributes in sorted order to get more + deterministic behaviour in nix-env operations (e.g. when + there are names clashes between derivations, the derivation + bound to the attribute with the "lower" name should take + precedence). */ + for (auto & attr : v.attrs->lexicographicOrder(state.symbols)) { + debug("evaluating attribute '%1%'", state.symbols[attr->name]); + // FIXME: only consider attrs with identifier-like names?? Why??? + if (!std::regex_match(std::string(state.symbols[attr->name]), attrRegex)) { + continue; + } + std::string joinedAttrPath = addToPath(pathPrefix, state.symbols[attr->name]); + if (combineChannels) { + getDerivations(state, *attr->value, joinedAttrPath, autoArgs, drvs, done, ignoreAssertionFailures); + } else if (getDerivation(state, *attr->value, joinedAttrPath, drvs, ignoreAssertionFailures)) { + /* If the value of this attribute is itself a set, + should we recurse into it? => Only if it has a + `recurseForDerivations = true' attribute. */ + if (attr->value->type() == nAttrs) { + Attr * recurseForDrvs = attr->value->attrs->get(state.sRecurseForDerivations); + if (recurseForDrvs == nullptr) { + continue; + } + bool shouldRecurse = state.forceBool( + *recurseForDrvs->value, + attr->pos, + fmt("while evaluating the '%s' attribute", Magenta("recurseForDerivations")) + ); + if (!shouldRecurse) { + continue; + } + + getDerivations( + state, + *attr->value, + joinedAttrPath, + autoArgs, + drvs, + done, + ignoreAssertionFailures + ); + } + } + } } diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index 44aac3015..fd927b9e5 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -37,13 +37,14 @@ private: bool checkMeta(Value & v); + void fillOutputs(bool withPaths = true); + public: /** * path towards the derivation */ std::string attrPath; - DrvInfo(EvalState & state) : state(&state) { }; DrvInfo(EvalState & state, std::string attrPath, Bindings * attrs); DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index c35f88f8d..57485aa0a 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -3,6 +3,9 @@ #include <cassert> #include <climits> +#include <functional> +#include <ranges> +#include <span> #include "gc-alloc.hh" #include "symbol-table.hh" @@ -11,6 +14,7 @@ #include "source-path.hh" #include "print-options.hh" #include "checked-arithmetic.hh" +#include "concepts.hh" #include <nlohmann/json_fwd.hpp> @@ -132,6 +136,55 @@ class ExternalValueBase std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); +extern ExprBlackHole eBlackHole; + +struct NewValueAs +{ + struct integer_t { }; + constexpr static integer_t integer{}; + + struct floating_t { }; + constexpr static floating_t floating{}; + + struct boolean_t { }; + constexpr static boolean_t boolean{}; + + struct string_t { }; + constexpr static string_t string{}; + + struct path_t { }; + constexpr static path_t path{}; + + struct list_t { }; + constexpr static list_t list{}; + + struct attrs_t { }; + constexpr static attrs_t attrs{}; + + struct thunk_t { }; + constexpr static thunk_t thunk{}; + + struct null_t { }; + constexpr static null_t null{}; + + struct app_t { }; + constexpr static app_t app{}; + + struct primop_t { }; + constexpr static primop_t primop{}; + + struct primOpApp_t { }; + constexpr static primOpApp_t primOpApp{}; + + struct lambda_t { }; + constexpr static lambda_t lambda{}; + + struct external_t { }; + constexpr static external_t external{}; + + struct blackhole_t { }; + constexpr static blackhole_t blackhole{}; +}; struct Value { @@ -142,6 +195,315 @@ private: public: + // Discount `using NewValueAs::*;` +#define USING_VALUETYPE(name) using name = NewValueAs::name + USING_VALUETYPE(integer_t); + USING_VALUETYPE(floating_t); + USING_VALUETYPE(boolean_t); + USING_VALUETYPE(string_t); + USING_VALUETYPE(path_t); + USING_VALUETYPE(list_t); + USING_VALUETYPE(attrs_t); + USING_VALUETYPE(thunk_t); + USING_VALUETYPE(primop_t); + USING_VALUETYPE(app_t); + USING_VALUETYPE(null_t); + USING_VALUETYPE(primOpApp_t); + USING_VALUETYPE(lambda_t); + USING_VALUETYPE(external_t); + USING_VALUETYPE(blackhole_t); +#undef USING_VALUETYPE + + /// Default constructor which is still used in the codebase but should not + /// be used in new code. Zero initializes its members. + [[deprecated]] Value() + : internalType(static_cast<InternalType>(0)) + , _empty{ 0, 0 } + { } + + /// Constructs a nix language value of type "int", with the integral value + /// of @ref i. + Value(integer_t, NixInt i) + : internalType(tInt) + , _empty{ 0, 0 } + { + // the NixInt ctor here is is special because NixInt has a ctor too, so + // we're not allowed to have it as an anonymous aggreagte member. we do + // however still have the option to clear the data members using _empty + // and leaving the second word of data cleared by setting only integer. + integer = i; + } + + /// Constructs a nix language value of type "float", with the floating + /// point value of @ref f. + Value(floating_t, NixFloat f) + : internalType(tFloat) + , fpoint(f) + , _float_pad(0) + { } + + /// Constructs a nix language value of type "bool", with the boolean + /// value of @ref b. + Value(boolean_t, bool b) + : internalType(tBool) + , boolean(b) + , _bool_pad(0) + { } + + /// Constructs a nix language value of type "string", with the value of the + /// C-string pointed to by @ref strPtr, and optionally with an array of + /// string context pointed to by @ref contextPtr. + /// + /// Neither the C-string nor the context array are copied; this constructor + /// assumes suitable memory has already been allocated (with the GC if + /// enabled), and string and context data copied into that memory. + Value(string_t, char const * strPtr, char const ** contextPtr = nullptr) + : internalType(tString) + , string({ .s = strPtr, .context = contextPtr }) + { } + + /// Constructx a nix language value of type "string", with a copy of the + /// string data viewed by @ref copyFrom. + /// + /// The string data *is* copied from @ref copyFrom, and this constructor + /// performs a dynamic (GC) allocation to do so. + Value(string_t, std::string_view copyFrom, NixStringContext const & context = {}) + : internalType(tString) + , string({ .s = gcCopyStringIfNeeded(copyFrom), .context = nullptr }) + { + if (context.empty()) { + // It stays nullptr. + return; + } + + // Copy the context. + this->string.context = gcAllocType<char const *>(context.size() + 1); + + size_t n = 0; + for (NixStringContextElem const & contextElem : context) { + this->string.context[n] = gcCopyStringIfNeeded(contextElem.to_string()); + n += 1; + } + + // Terminator sentinel. + this->string.context[n] = nullptr; + } + + /// Constructx a nix language value of type "string", with the value of the + /// C-string pointed to by @ref strPtr, and optionally with a set of string + /// context @ref context. + /// + /// The C-string is not copied; this constructor assumes suitable memory + /// has already been allocated (with the GC if enabled), and string data + /// has been copied into that memory. The context data *is* copied from + /// @ref context, and this constructor performs a dynamic (GC) allocation + /// to do so. + Value(string_t, char const * strPtr, NixStringContext const & context) + : internalType(tString) + , string({ .s = strPtr, .context = nullptr }) + { + if (context.empty()) { + // It stays nullptr + return; + } + + // Copy the context. + this->string.context = gcAllocType<char const *>(context.size() + 1); + + size_t n = 0; + for (NixStringContextElem const & contextElem : context) { + this->string.context[n] = gcCopyStringIfNeeded(contextElem.to_string()); + n += 1; + } + + // Terminator sentinel. + this->string.context[n] = nullptr; + } + + /// Constructs a nix language value of type "path", with the value of the + /// C-string pointed to by @ref strPtr. + /// + /// The C-string is not copied; this constructor assumes suitable memory + /// has already been allocated (with the GC if enabled), and string data + /// has been copied into that memory. + Value(path_t, char const * strPtr) + : internalType(tPath) + , _path(strPtr) + , _path_pad(0) + { } + + /// Constructs a nix language value of type "path", with the path + /// @ref path. + /// + /// The data from @ref path *is* copied, and this constructor performs a + /// dynamic (GC) allocation to do so. + Value(path_t, SourcePath const & path) + : internalType(tPath) + , _path(gcCopyStringIfNeeded(path.path.abs())) + , _path_pad(0) + { } + + /// Constructs a nix language value of type "list", with element array + /// @ref items. + /// + /// Generally, the data in @ref items is neither deep copied nor shallow + /// copied. This construct assumes the std::span @ref items is a region of + /// memory that has already been allocated (with the GC if enabled), and + /// an array of valid Value pointers has been copied into that memory. + /// + /// Howver, as an implementation detail, if @ref items is only 2 items or + /// smaller, the list is stored inline, and the Value pointers in + /// @ref items are shallow copied into this structure, without dynamically + /// allocating memory. + Value(list_t, std::span<Value *> items) + { + if (items.size() == 1) { + this->internalType = tList1; + this->smallList[0] = items[0]; + this->smallList[1] = nullptr; + } else if (items.size() == 2) { + this->internalType = tList2; + this->smallList[0] = items[0]; + this->smallList[1] = items[1]; + } else { + this->internalType = tListN; + this->bigList.size = items.size(); + this->bigList.elems = items.data(); + } + } + + /// Constructs a nix language value of type "list", with an element array + /// initialized by applying @ref transformer to each element in @ref items. + /// + /// This allows "in-place" construction of a nix list when some logic is + /// needed to get each Value pointer. This constructor dynamically (GC) + /// allocates memory for the size of @ref items, and the Value pointers + /// returned by @ref transformer are shallow copied into it. + template< + std::ranges::sized_range SizedIterableT, + InvocableR<Value *, typename SizedIterableT::value_type const &> TransformerT + > + Value(list_t, SizedIterableT & items, TransformerT const & transformer) + { + if (items.size() == 1) { + this->internalType = tList1; + this->smallList[0] = transformer(*items.begin()); + this->smallList[1] = nullptr; + } else if (items.size() == 2) { + this->internalType = tList2; + auto it = items.begin(); + this->smallList[0] = transformer(*it); + it++; + this->smallList[1] = transformer(*it); + } else { + this->internalType = tListN; + this->bigList.size = items.size(); + this->bigList.elems = gcAllocType<Value *>(items.size()); + auto it = items.begin(); + for (size_t i = 0; i < items.size(); i++, it++) { + this->bigList.elems[i] = transformer(*it); + } + } + } + + /// Constructs a nix language value of the singleton type "null". + Value(null_t) + : internalType(tNull) + , _empty{0, 0} + { } + + /// Constructs a nix language value of type "set", with the attribute + /// bindings pointed to by @ref bindings. + /// + /// The bindings are not not copied; this constructor assumes @ref bindings + /// has already been suitably allocated by something like nix::buildBindings. + Value(attrs_t, Bindings * bindings) + : internalType(tAttrs) + , attrs(bindings) + , _attrs_pad(0) + { } + + /// Constructs a nix language lazy delayed computation, or "thunk". + /// + /// The thunk stores the environment it will be computed in @ref env, and + /// the expression that will need to be evaluated @ref expr. + Value(thunk_t, Env & env, Expr & expr) + : internalType(tThunk) + , thunk({ .env = &env, .expr = &expr }) + { } + + /// Constructs a nix language value of type "lambda", which represents + /// a builtin, primitive operation ("primop"), from the primop + /// implemented by @ref primop. + Value(primop_t, PrimOp & primop); + + /// Constructs a nix language value of type "lambda", which represents a + /// partially applied primop. + Value(primOpApp_t, Value & lhs, Value & rhs) + : internalType(tPrimOpApp) + , primOpApp({ .left = &lhs, .right = &rhs }) + { } + + /// Constructs a nix language value of type "lambda", which represents a + /// lazy partial application of another lambda. + Value(app_t, Value & lhs, Value & rhs) + : internalType(tApp) + , app({ .left = &lhs, .right = &rhs }) + { } + + /// Constructs a nix language value of type "external", which is only used + /// by plugins. Do any existing plugins even use this mechanism? + Value(external_t, ExternalValueBase & external) + : internalType(tExternal) + , external(&external) + , _external_pad(0) + { } + + /// Constructs a nix language value of type "lambda", which represents a + /// run of the mill lambda defined in nix code. + /// + /// This takes the environment the lambda is closed over @ref env, and + /// the lambda expression itself @ref lambda, which will not be evaluated + /// until it is applied. + Value(lambda_t, Env & env, ExprLambda & lambda) + : internalType(tLambda) + , lambda({ .env = &env, .fun = &lambda }) + { } + + /// Constructs an evil thunk, whose evaluation represents infinite recursion. + explicit Value(blackhole_t) + : internalType(tThunk) + , thunk({ .env = nullptr, .expr = reinterpret_cast<Expr *>(&eBlackHole) }) + { } + + Value(Value const & rhs) = default; + + /// Move constructor. Does the same thing as the copy constructor, but + /// also zeroes out the other Value. + Value(Value && rhs) + : internalType(rhs.internalType) + , _empty{ 0, 0 } + { + *this = std::move(rhs); + } + + Value & operator=(Value const & rhs) = default; + + /// Move assignment operator. + /// Does the same thing as the copy assignment operator, but also zeroes out + /// the rhs. + inline Value & operator=(Value && rhs) + { + *this = static_cast<const Value &>(rhs); + if (this != &rhs) { + // Kill `rhs`, because non-destructive move lol. + rhs.internalType = static_cast<InternalType>(0); + rhs._empty[0] = 0; + rhs._empty[1] = 0; + } + return *this; + } + void print(EvalState &state, std::ostream &str, PrintOptions options = PrintOptions {}); // Functions needed to distinguish the type @@ -160,8 +522,15 @@ public: union { + /// Dummy field, which takes up as much space as the largest union variants + /// to set the union's memory to zeroed memory. + uintptr_t _empty[2]; + NixInt integer; - bool boolean; + struct { + bool boolean; + uintptr_t _bool_pad; + }; /** * Strings in the evaluator carry a so-called `context` which @@ -190,8 +559,14 @@ public: const char * * context; // must be in sorted order } string; - const char * _path; - Bindings * attrs; + struct { + const char * _path; + uintptr_t _path_pad; + }; + struct { + Bindings * attrs; + uintptr_t _attrs_pad; + }; struct { size_t size; Value * * elems; @@ -208,12 +583,21 @@ public: Env * env; ExprLambda * fun; } lambda; - PrimOp * primOp; + struct { + PrimOp * primOp; + uintptr_t _primop_pad; + }; struct { Value * left, * right; } primOpApp; - ExternalValueBase * external; - NixFloat fpoint; + struct { + ExternalValueBase * external; + uintptr_t _external_pad; + }; + struct { + NixFloat fpoint; + uintptr_t _float_pad; + }; }; /** @@ -449,8 +833,6 @@ public: }; -extern ExprBlackHole eBlackHole; - bool Value::isBlackhole() const { return internalType == tThunk && thunk.expr == (Expr*) &eBlackHole; diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index 18f519c5c..de280c13f 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -15,4 +15,27 @@ GENERATE_CMP_EXT( me->cpuUser, me->cpuSystem); +KeyedBuildResult BuildResult::restrictTo(DerivedPath path) const +{ + KeyedBuildResult res{*this, std::move(path)}; + + if (auto pbp = std::get_if<DerivedPath::Built>(&res.path)) { + auto & bp = *pbp; + + /* Because goals are in general shared between derived paths + that share the same derivation, we need to filter their + results to get back just the results we care about. + */ + + for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { + if (bp.outputs.contains(it->first)) + ++it; + else + it = res.builtOutputs.erase(it); + } + } + + return res; +} + } diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 8840fa7e3..9634fb944 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -11,6 +11,8 @@ namespace nix { +struct KeyedBuildResult; + struct BuildResult { /** @@ -112,6 +114,18 @@ struct BuildResult { throw Error("%s", errorMsg); } + + /** + * Project a BuildResult with just the information that pertains to + * the given path. + * + * A `BuildResult` may hold information for multiple derived paths; + * this function discards information about outputs not relevant in + * `path`. Build `Goal`s in particular may contain more outputs for + * a single build result than asked for directly, it's necessary to + * remove any such additional result to not leak other build infos. + */ + KeyedBuildResult restrictTo(DerivedPath path) const; }; /** diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d0152355f..38c54e854 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -79,7 +79,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); - worker.updateProgress(); } @@ -100,7 +99,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation trace("created"); mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); - worker.updateProgress(); /* Prevent the .chroot directory from being garbage-collected. (See isActiveTempFile() in gc.cc.) */ @@ -670,7 +668,6 @@ void DerivationGoal::started() act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg, Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1}); mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); - worker.updateProgress(); } void DerivationGoal::tryToBuild() @@ -1366,7 +1363,6 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) void DerivationGoal::handleEOF(int fd) { if (!currentLogLine.empty()) flushLine(); - worker.wakeUp(shared_from_this()); } @@ -1537,8 +1533,6 @@ void DerivationGoal::done( worker.failedBuilds++; } - worker.updateProgress(); - auto traceBuiltOutputsFile = getEnv("_NIX_TRACE_BUILT_OUTPUTS").value_or(""); if (traceBuiltOutputsFile != "") { std::fstream fs; @@ -1566,7 +1560,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) auto & outputs = nodeP->value; for (auto & outputName : outputs) { - auto buildResult = dg->getBuildResult(DerivedPath::Built { + auto buildResult = dg->buildResult.restrictTo(DerivedPath::Built { .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 735a07f96..94c9206a3 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -41,14 +41,13 @@ void DrvOutputSubstitutionGoal::tryNext() /* Make sure that we are allowed to start a substitution. Note that even if maxSubstitutionJobs == 0, we still allow a substituter to run. This prevents infinite waiting. */ - if (worker.runningCASubstitutions >= std::max(1U, settings.maxSubstitutionJobs.get())) { + if (worker.runningSubstitutions >= std::max(1U, settings.maxSubstitutionJobs.get())) { worker.waitForBuildSlot(shared_from_this()); return; } maintainRunningSubstitutions = - std::make_unique<MaintainCount<uint64_t>>(worker.runningCASubstitutions); - worker.updateProgress(); + std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); if (subs.size() == 0) { /* None left. Terminate this goal and let someone else deal @@ -62,7 +61,6 @@ void DrvOutputSubstitutionGoal::tryNext() if (substituterFailed) { worker.failedSubstitutions++; - worker.updateProgress(); } return; @@ -164,10 +162,5 @@ void DrvOutputSubstitutionGoal::work() (this->*state)(); } -void DrvOutputSubstitutionGoal::handleEOF(int fd) -{ - if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 47b9ecc49..598b119dc 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -72,7 +72,6 @@ public: std::string key() override; void work() override; - void handleEOF(int fd) override; JobCategory jobCategory() const override { return JobCategory::Substitution; diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 67236bb39..c6955600e 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -62,10 +62,7 @@ std::vector<KeyedBuildResult> Store::buildPathsWithResults( std::vector<KeyedBuildResult> results; for (auto & [req, goalPtr] : state) - results.emplace_back(KeyedBuildResult { - goalPtr->getBuildResult(req), - /* .path = */ req, - }); + results.emplace_back(goalPtr->buildResult.restrictTo(req)); return results; } @@ -78,7 +75,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); - return goal->getBuildResult(DerivedPath::Built { + return goal->buildResult.restrictTo(DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index f4973efc9..4db6af6e6 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -11,29 +11,6 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { } -BuildResult Goal::getBuildResult(const DerivedPath & req) const { - BuildResult res { buildResult }; - - if (auto pbp = std::get_if<DerivedPath::Built>(&req)) { - auto & bp = *pbp; - - /* Because goals are in general shared between derived paths - that share the same derivation, we need to filter their - results to get back just the results we care about. - */ - - for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { - if (bp.outputs.contains(it->first)) - ++it; - else - it = res.builtOutputs.erase(it); - } - } - - return res; -} - - void Goal::addWaitee(GoalPtr waitee) { waitees.insert(waitee); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index d105c53cd..575621037 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -98,7 +98,6 @@ struct Goal : public std::enable_shared_from_this<Goal> */ std::optional<ExitCode> exitCode; -protected: /** * Build result. */ @@ -107,18 +106,6 @@ protected: public: /** - * Project a `BuildResult` with just the information that pertains - * to the given request. - * - * In general, goals may be aliased between multiple requests, and - * the stored `BuildResult` has information for the union of all - * requests. We don't want to leak what the other request are for - * sake of both privacy and determinism, and this "safe accessor" - * ensures we don't. - */ - BuildResult getBuildResult(const DerivedPath &) const; - - /** * Exception containing an error message, if any. */ std::unique_ptr<Error> ex; @@ -145,7 +132,6 @@ public: virtual void handleEOF(int fd) { - abort(); } void trace(std::string_view s); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 1330f061c..027a7e161 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -86,7 +86,6 @@ void PathSubstitutionGoal::tryNext() if (substituterFailed) { worker.failedSubstitutions++; - worker.updateProgress(); } return; @@ -150,8 +149,6 @@ void PathSubstitutionGoal::tryNext() ? std::make_unique<MaintainCount<uint64_t>>(worker.expectedDownloadSize, narInfo->fileSize) : nullptr; - worker.updateProgress(); - /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ @@ -210,7 +207,6 @@ void PathSubstitutionGoal::tryToRun() } maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); - worker.updateProgress(); outPipe.create(); @@ -289,8 +285,6 @@ void PathSubstitutionGoal::finished() worker.doneNarSize += maintainExpectedNar->delta; maintainExpectedNar.reset(); - worker.updateProgress(); - done(ecSuccess, BuildResult::Substituted); } @@ -300,12 +294,6 @@ void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data) } -void PathSubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - - void PathSubstitutionGoal::cleanup() { try { diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 52780a967..d85b3beb3 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -110,7 +110,6 @@ public: * Callback used by the worker to write to the log. */ void handleChildOutput(int fd, std::string_view data) override; - void handleEOF(int fd) override; /* Called by destructor, can't be overridden */ void cleanup() override final; diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 402cdc5b4..5b2e36acb 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -300,6 +300,19 @@ void Worker::run(const Goals & _topGoals) for (auto & goal : awake2) { checkInterrupt(); goal->work(); + + actDerivations.progress( + doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds + ); + actSubstitutions.progress( + doneSubstitutions, + expectedSubstitutions + doneSubstitutions, + runningSubstitutions, + failedSubstitutions + ); + act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); + act.setExpected(actCopyPath, expectedNarSize + doneNarSize); + if (topGoals.empty()) break; // stuff may have been cancelled } } @@ -446,6 +459,7 @@ void Worker::waitForInput() if (rd == 0 || (rd == -1 && errno == EIO)) { debug("%1%: got EOF", goal->getName()); goal->handleEOF(k); + wakeUp(goal); j->fds.erase(k); } else if (rd == -1) { if (errno != EINTR) diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index bbebeeb9c..3984c9c1c 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -146,7 +146,6 @@ public: uint64_t doneSubstitutions = 0; uint64_t failedSubstitutions = 0; uint64_t runningSubstitutions = 0; - uint64_t runningCASubstitutions = 0; uint64_t expectedDownloadSize = 0; uint64_t doneDownloadSize = 0; uint64_t expectedNarSize = 0; @@ -281,14 +280,6 @@ public: bool pathContentsGood(const StorePath & path); void markContentsGood(const StorePath & path); - - void updateProgress() - { - actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); - actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); - act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize); - act.setExpected(actCopyPath, expectedNarSize + doneNarSize); - } }; } diff --git a/src/libutil/concepts.hh b/src/libutil/concepts.hh new file mode 100644 index 000000000..48bd1dbe1 --- /dev/null +++ b/src/libutil/concepts.hh @@ -0,0 +1,22 @@ +#pragma once +/// @file Defines C++ 20 concepts that std doesn't have. + +#include <type_traits> + +namespace nix +{ + +/// Like std::invocable<>, but also constrains the return type as well. +/// +/// Somehow, there is no std concept to do this, even though there is a type trait +/// for it. +/// +/// @tparam CallableT The type you want to constrain to be callable, and to return +/// @p ReturnT when called with @p Args as arguments. +/// +/// @tparam ReturnT The type the callable should return when called. +/// @tparam Args The arguments the callable should accept to return @p ReturnT. +template<typename CallableT, typename ReturnT, typename ...Args> +concept InvocableR = std::is_invocable_r_v<ReturnT, CallableT, Args...>; + +} diff --git a/src/libutil/meson.build b/src/libutil/meson.build index c860e7e00..01fe65207 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -58,6 +58,7 @@ libutil_headers = files( 'comparator.hh', 'compression.hh', 'compute-levels.hh', + 'concepts.hh', 'config-impl.hh', 'config.hh', 'current-process.hh', diff --git a/src/meson.build b/src/meson.build index 3fc5595b8..e918ae392 100644 --- a/src/meson.build +++ b/src/meson.build @@ -12,10 +12,19 @@ subdir('libmain') # libcmd depends on everything subdir('libcmd') - # The rest of the subdirectories aren't separate components, # just source files in another directory, so we process them here. +# Static library that just sets default ASan options. It needs to be included +# in every executable. +asanoptions = static_library( + 'libasanoptions', + files('asan-options/asan-options.cc'), +) +libasanoptions = declare_dependency( + link_whole: asanoptions +) + build_remote_sources = files( 'build-remote/build-remote.cc', ) diff --git a/src/nix/meson.build b/src/nix/meson.build index 22f148fcb..97387e402 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -80,6 +80,7 @@ nix = executable( profiles_md_gen, nix2_commands_sources, dependencies : [ + libasanoptions, liblixcmd, liblixutil_mstatic, liblixstore_mstatic, |